---
# Source: victoria-metrics-k8s-stack/charts/grafana/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: false
metadata:
  labels:
    helm.sh/chart: grafana-11.3.2
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "12.4.1"
  name: release-name-grafana
  namespace: default
---
# Source: victoria-metrics-k8s-stack/charts/kube-state-metrics/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: true
metadata:
  labels:    
    helm.sh/chart: kube-state-metrics-7.2.1
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: kube-state-metrics
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.18.0"
  name: release-name-kube-state-metrics
  namespace: default
---
# Source: victoria-metrics-k8s-stack/charts/prometheus-node-exporter/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: release-name-prometheus-node-exporter
  namespace: default
  labels:
    helm.sh/chart: prometheus-node-exporter-4.52.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: prometheus-node-exporter
    app.kubernetes.io/name: prometheus-node-exporter
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.10.2"
automountServiceAccountToken: false
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: release-name-victoria-metrics-operator
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
automountServiceAccountToken: true
---
# Source: victoria-metrics-k8s-stack/charts/grafana/templates/secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: release-name-grafana
  namespace: default
  labels:
    helm.sh/chart: grafana-11.3.2
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "12.4.1"
    app.kubernetes.io/component: admin-secret
type: Opaque
data:
  
  admin-user: "YWRtaW4="
  admin-password: "ZDZHYUJobW81YllKTk5lU1U5d0t4dlo1UE9LS1NrWWVYSDhUV3owcg=="
  ldap-toml: ""
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/webhook.yaml
apiVersion: v1
kind: Secret
metadata:
  name: release-name-victoria-metrics-operator-validation
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
type: kubernetes.io/tls
data:
  ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ3akNDQXRhZ0F3SUJBZ0lSQUlxdzVSSkl4YmdQQlpTbWRiem5mSFV3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakF4TVM4d0xRWURWUVFERXlaeVpXeGxZWE5sTFc1aGJXVXRkbWxqZEc5eWFXRXRiV1YwCmNtbGpjeTF2Y0dWeVlYUnZjakNDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFPbVAKMEJqVXJiRmhQOFBmWUcwcmczREtlQTZ6eG9GTDlETlVsS1pjZHhOcHdpWnRuOHVGanhBeGcrY2ZrZUZJNExOZgpYb2p5bEZ6ZzgrWncvbmllWlRkVGtXUXkyQ1RFTGdSMFlHWnVIWDhQOWJuL2VCUDFFZVQyQXdWUnUyZ2hDSW5PCkVOUi9RTDJNSFk4cDNqWWhxVGViWTJuN2NMbmxpQjBHbWhPanIwTjltT09uZzdreEVnVVMxdW01OXc5Vm9PMUwKb2F5SGlXakZ5YjlDL1JKZEYxb3FHbDVvL3VWNlZncDllSXFoVnRNQWlvd0lTU1FPaGZpYlhSNXM3MStOSTVkUwpXQUtuNWhGdTduaHptSllydHlOZ2dsUkhLbVhLZjdJRUduUlFVSVRHZkNvYy9CdTNlaUdUQ3JjTDNib0o5M1BMCks4MGNTWHB0V25NeFIxaG1IQmNDQXdFQUFhT0NBUmN3Z2dFVE1BNEdBMVVkRHdFQi93UUVBd0lGb0RBZEJnTlYKSFNVRUZqQVVCZ2dyQmdFRkJRY0RBUVlJS3dZQkJRVUhBd0l3REFZRFZSMFRBUUgvQkFJd0FEQWZCZ05WSFNNRQpHREFXZ0JTN3dLdVVkU2YxczhiQURPc2pJdW9MR2lhZEJqQ0JzZ1lEVlIwUkJJR3FNSUduZ2k1eVpXeGxZWE5sCkxXNWhiV1V0ZG1samRHOXlhV0V0YldWMGNtbGpjeTF2Y0dWeVlYUnZjaTVrWldaaGRXeDBnakp5Wld4bFlYTmwKTFc1aGJXVXRkbWxqZEc5eWFXRXRiV1YwY21samN5MXZjR1Z5WVhSdmNpNWtaV1poZFd4MExuTjJZNEpCY21WcwpaV0Z6WlMxdVlXMWxMWFpwWTNSdmNtbGhMVzFsZEhKcFkzTXRiM0JsY21GMGIzSXVaR1ZtWVhWc2RDNXpkbU11ClkyeDFjM1JsY2k1c2IyTmhiQzR3RFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUtOVFBGNmNMYUdVcWdhVTl6UmQKbndYQUpaTjRwWDhoR2lVVGNTNUx0L0pJRVJpWjNhUkZXOGtSenpiV2haMXl6cnJYSlJYZ0hSajlRRnhXYkFTdApQZ25lNWlXMGJlT0NGbWRtMkQrQjdDenphV2VKQUR5a2NCcEh4T0k1VDdqRDRUbHc5TXZWTER4RlN1SmU3MUtnClA3K1k1SkUwZ3l0a1hCZlFCVXlEMjcrcFY5Q1dOamhzdFpybzlGRWpaYjlhSWhERkp0NVc4anlOTmQvbVBDNzkKbmI3UWR1WmhtY2VNcHhNWndMZDRjbm5iVVpmU0ZCcHJjbVNXKzdPSHVBSDhkZjhLaWFkT0VGYjA0U1A1TUNMcgpaNjVBelZabXJ5ZzNqTFVhNkhzcGdYNEZucHNBR1hHL2tLR29xTCtFQVJJK3dHUUhWeXlhZkJXSUxLdUQ4bmRDCkQrST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
  tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcFFJQkFBS0NBUUVBNlkvUUdOU3RzV0Uvdzk5Z2JTdURjTXA0RHJQR2dVdjBNMVNVcGx4M0UybkNKbTJmCnk0V1BFREdENXgrUjRVamdzMTllaVBLVVhPRHo1bkQrZUo1bE4xT1JaRExZSk1RdUJIUmdabTRkZncvMXVmOTQKRS9VUjVQWURCVkc3YUNFSWljNFExSDlBdll3ZGp5bmVOaUdwTjV0amFmdHd1ZVdJSFFhYUU2T3ZRMzJZNDZlRAp1VEVTQlJMVzZibjNEMVdnN1V1aHJJZUphTVhKdjBMOUVsMFhXaW9hWG1qKzVYcFdDbjE0aXFGVzB3Q0tqQWhKCkpBNkYrSnRkSG16dlg0MGpsMUpZQXFmbUVXN3VlSE9ZbGl1M0kyQ0NWRWNxWmNwL3NnUWFkRkJRaE1aOEtoejgKRzdkNklaTUt0d3ZkdWduM2M4c3J6UnhKZW0xYWN6RkhXR1ljRndJREFRQUJBb0lCQVFEYnRZOWFiR2JnWDgwdQpJdW1BaU5oaXQvUHNVSzFCWUFQZWJZaFFYdGNkQ2ZXN1ZKQm5VY0pjeDFkL2liRWZMZ0ZDZTBwZ3BXRnZaYVhLCmR5ektwcVlzUDQ2YnBVQ3loV0dNaXpWTDVmWVZXeHdhMWZsTlBOS1NnZUlobE5zb2Z6cWE5NS9aWmNPdkN5bWIKVFJWZDhwaE9XUHhSN0ZnRVo2bDlOSUtvU3VoS0lRWEUrVmk1VS9Cb0UyM1ZzYW9EVVpDYVJ3WmUrSmttUEtxSwp5QU9vd3k0T1ZzME43Q1kxNUVhVHVtL202MjJmaEdhbXJkVUJycTh3cis0VFJNYStUbmlZOE1aaHpUMkJUa2lmCjVjUzQ5NlN3c1VCMmhObE5zSWJLY3RpR1FmekRHWXBiWlArRklzcjVRM3BuVWp1Z1BXQ01nVThKMFZzYjVHTC8KRzZvREJCUnhBb0dCQVA4akc4aHk1Y1JnVUpHUzdWZjJsK1V1eXR5emRkSHN0UXF5R2xrY0xtNlVDSVAzZ3l4NApMc25xbm14NTNkQlZkVjcxSXJxZytSOFRwaFc5WUpDV0wzNENJTWE5T3cxRVVIK0VURGNCNjdNYkhyVFFFUXVpCmNoRGxtNGZUdEZXbFFXK3hMYVZiTHVwd0cwWFlFRWFsNXlwNCtETm9jRWQxK0d4SmtjOHFCRkNqQW9HQkFPcGEKQm1GWWgzREdBUmNBUjhzWnVRN2FOOWxyY3dBWmdNajhoWW1zV3lsK2pCdjV5RWZkOVQrclRIL2praTd6ekRTagpLVDVKekIrdW14dmRUOHhPeDUwNi93YkFKaUlVOHdIVGNvTnhrU3hSY3VOM3BoejFhcXNoUHJwNFRhS2FoMzRkCkNQanllRVE3OTdNbk83RXQzRW5XTzZ3eThMUGF0OXN3R1JsY1c1bjlBb0dCQU5saDhZY2RsblYyVnM5aDc0VzIKSG45d0dmcG5rYWgyVlY2eWwwNTdlM2hOK3Q5dDRUN2ZlNFlndUREV3VNNk1NYi9oZjJDaVRRQlVmbnNoYXdobQpYanF2MTN0ZHVkVzQ4aGxrQUlnUncrQkw2Q0xJOE8wSDBNWml5OXl6M1gvbm1OUndrdUxEeVRxSW9FUW5MdDd5CjlmNHpLYzZyV3BjWWhiekdCazBMa0xlZkFvR0FKMFJBVDIxSy9GMWc3MTZ2cU5zOWc3eVNvUTU4N1daWWl0U3MKOTExMnhPUWNTOCsxZkQwM2k1eURLK2FRcHVkWXJ2VXJ3dkVRajdwczZPVlRLcE9FcE5hdXhvM0psZXBXc1ZhaAptRjZlQ1lQNE1SN3FLVzRCbW56dWlzWC9sQ0E2VmFvOXg2Q3hBQlhnTWhkL3kvYzRNdkwvZXFIcWJPaFg2QlVPCkQ0N0hGbTBDZ1lFQXRyc1VOUkRtZXNqQk9jODFzZE9tNTl2d3RKRFVOcytlSmEzRGtvWVpmSUZsQURVdjRSNysKZmIyVll3bXdBTEV6RVJ3c2JBWDNIcHltYjErNEgwR0tqRXAzejJZdlF6RFhwYndjWXZETVRDZmhXVDlyQVFwWQpsWUxISEJadnZZQ1hFOEtFSTQrb1lkd3F0YWc0ZSttcG1IZmlKK3U4S0dQZko5WnRCY0RncjljPQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalertmanager/vmalertmanager.yaml
apiVersion: v1
kind: Secret
metadata:
  name: vmalertmanager-vmks
  namespace: default
  labels: 
    app.kubernetes.io/component: vmks-alertmanager
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
stringData:
  alertmanager.yaml: |
    receivers:
    - name: blackhole
    route:
      receiver: blackhole
    templates:
    - /etc/vm/configs/**/*.tmpl
---
# Source: victoria-metrics-k8s-stack/charts/grafana/templates/configmap-dashboard-provider.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    helm.sh/chart: grafana-11.3.2
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "12.4.1"
  name: release-name-grafana-config-dashboards
  namespace: default
data:
  provider.yaml: |-
    apiVersion: 1
    providers:
      - name: 'default'
        orgId: 1
        folder: ''
        folderUid: ''
        type: file
        disableDeletion: false
        allowUiUpdates: false
        updateIntervalSeconds: 30
        options:
          foldersFromFilesStructure: false
          path: /var/lib/grafana/dashboards/default
---
# Source: victoria-metrics-k8s-stack/charts/grafana/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: release-name-grafana
  namespace: default
  labels:
    helm.sh/chart: grafana-11.3.2
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "12.4.1"
data:
  
  grafana.ini: |
    [analytics]
    check_for_updates = false
    [log]
    mode = console
    [paths]
    data = /var/lib/grafana/
    logs = /var/log/grafana
    plugins = /var/lib/grafana/plugins
    provisioning = /etc/grafana/provisioning
    [server]
    domain = ''
    root_url = %(protocol)s://%(domain)s/grafana/
    serve_from_sub_path = true
    [unified_storage]
    index_path = /var/lib/grafana-search/bleve
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-alertmanager-overview
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  alertmanager-overview.json: | 
    {"editable":false,"graphTooltip":1,"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":1,"title":"Alerts","type":"row"},{"datasource":{"type":"prometheus","uid":"$datasource"},"description":"current set of alerts stored in the Alertmanager","fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","stacking":{"mode":"normal"}},"unit":"none"}},"gridPos":{"h":7,"w":12,"x":0,"y":1},"id":2,"options":{"legend":{"showLegend":false},"tooltip":{"mode":"multi"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\",cluster=~\"$cluster\"}) by(namespace,service,instance,cluster)","intervalFactor":2,"legendFormat":"{{instance}}"}],"title":"Alerts","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$datasource"},"description":"rate of successful and invalid alerts received by the Alertmanager","fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","stacking":{"mode":"normal"}},"unit":"ops"}},"gridPos":{"h":7,"w":12,"x":12,"y":1},"id":3,"options":{"legend":{"showLegend":false},"tooltip":{"mode":"multi"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\",cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,service,instance,cluster)","intervalFactor":2,"legendFormat":"{{instance}} Received"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\",cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,service,instance,cluster)","intervalFactor":2,"legendFormat":"{{instance}} Invalid"}],"title":"Alerts receive rate","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":8},"id":4,"title":"Notifications","type":"row"},{"datasource":{"type":"prometheus","uid":"$datasource"},"description":"rate of successful and invalid notifications sent by the Alertmanager","fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","stacking":{"mode":"normal"}},"unit":"ops"}},"gridPos":{"h":7,"w":12,"x":0,"y":9},"id":5,"options":{"legend":{"showLegend":false},"tooltip":{"mode":"multi"}},"pluginVersion":"v11.4.0","repeat":"integration","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",cluster=~\"$cluster\"}[$__rate_interval])) by(integration,namespace,service,instance,cluster)","intervalFactor":2,"legendFormat":"{{instance}} Total"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",cluster=~\"$cluster\"}[$__rate_interval])) by(integration,namespace,service,instance,cluster)","intervalFactor":2,"legendFormat":"{{instance}} Failed"}],"title":"$integration: Notifications Send Rate","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$datasource"},"description":"latency of notifications sent by the Alertmanager","fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","stacking":{"mode":"normal"}},"unit":"s"}},"gridPos":{"h":7,"w":12,"x":12,"y":9},"id":6,"options":{"legend":{"showLegend":false},"tooltip":{"mode":"multi"}},"pluginVersion":"v11.4.0","repeat":"integration","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"histogram_quantile(0.99, sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",cluster=~\"$cluster\"}[$__rate_interval])) by(le,namespace,service,instance,cluster))","intervalFactor":2,"legendFormat":"{{instance}} 99th Percentile"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"histogram_quantile(0.50, sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",cluster=~\"$cluster\"}[$__rate_interval])) by(le,namespace,service,instance,cluster))","intervalFactor":2,"legendFormat":"{{instance}} Median"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,service,instance,cluster) / sum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,service,instance,cluster)","intervalFactor":2,"legendFormat":"{{instance}} Average"}],"title":"$integration: Notification Duration","type":"timeseries"}],"schemaVersion":39,"tags":["alertmanager-mixin","vm-k8s-stack"],"templating":{"list":[{"current":{"selected":false,"text":"Prometheus","value":"Prometheus"},"hide":0,"label":"Data Source","name":"datasource","query":"prometheus","type":"datasource"},{"current":{"selected":false,"text":"","value":""},"datasource":{"type":"prometheus","uid":"${datasource}"},"label":"namespace","name":"namespace","query":"label_values(alertmanager_alerts, namespace)","refresh":2,"sort":1,"type":"query"},{"current":{"selected":false,"text":"","value":""},"datasource":{"type":"prometheus","uid":"${datasource}"},"label":"service","name":"service","query":"label_values(alertmanager_alerts, service)","refresh":2,"sort":1,"type":"query"},{"current":{"selected":false,"text":"$__all","value":"$__all"},"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":2,"includeAll":true,"name":"integration","query":"label_values(alertmanager_notifications_total{integration=~\".*\",cluster=~\"$cluster\"}, integration)","refresh":2,"sort":1,"type":"query"},{"allValue":".*","datasource":{"type":"prometheus"},"hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","query":".*","type":"constant"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["30s"]},"timezone":"utc","title":"Alertmanager / Overview","uid":"alertmanager-overview"}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-etcd
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  etcd.json: | 
    {"description":"etcd sample Grafana dashboard with Prometheus","editable":false,"panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"gridPos":{"h":7,"w":6,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none","graphMode":"none","reduceOptions":{"calcs":["lastNotNull"]}},"pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(etcd_server_has_leader{job=~\".*etcd.*\",cluster=~\"$cluster\"}) by(cluster)","legendFormat":"{{cluster}} - {{namespace}}\n"}],"title":"Up","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"ops"}},"gridPos":{"h":7,"w":10,"x":6,"y":0},"id":2,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(grpc_server_started_total{job=~\".*etcd.*\",cluster=~\"$cluster\",grpc_type=\"unary\"}[$__rate_interval])) by(cluster)","legendFormat":"RPC rate"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(grpc_server_handled_total{job=~\".*etcd.*\",cluster=~\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval])) by(cluster)","legendFormat":"RPC failed rate"}],"title":"RPC rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"}}},"gridPos":{"h":7,"w":8,"x":16,"y":0},"id":3,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(grpc_server_started_total{job=~\".*etcd.*\",cluster=~\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by(cluster) - sum(grpc_server_handled_total{cluster=~\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by(cluster)","legendFormat":"Watch streams"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(grpc_server_started_total{job=~\".*etcd.*\",cluster=~\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by(cluster) - sum(grpc_server_handled_total{cluster=~\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by(cluster)","legendFormat":"Lease streams"}],"title":"Active streams","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":0,"y":25},"id":4,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"etcd_mvcc_db_total_size_in_bytes{job=~\".*etcd.*\",cluster=~\"$cluster\"}","legendFormat":"{{instance}} DB size"}],"title":"DB size","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"s"}},"gridPos":{"h":7,"w":8,"x":8,"y":25},"id":5,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~\".*etcd.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(instance,le,cluster))","legendFormat":"{{instance}} WAL fsync"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~\".*etcd.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(instance,le,cluster))","legendFormat":"{{instance}} DB fsync"}],"title":"Disk sync duration","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":16,"y":25},"id":6,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"process_resident_memory_bytes{job=~\".*etcd.*\",cluster=~\"$cluster\"}","legendFormat":"{{instance}} resident memory"}],"title":"Memory","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":0,"y":50},"id":7,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"rate(etcd_network_client_grpc_received_bytes_total{job=~\".*etcd.*\",cluster=~\"$cluster\"}[$__rate_interval])","legendFormat":"{{instance}} client traffic in"}],"title":"Client traffic in","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":6,"y":50},"id":8,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"rate(etcd_network_client_grpc_sent_bytes_total{job=~\".*etcd.*\",cluster=~\"$cluster\"}[$__rate_interval])","legendFormat":"{{instance}} client traffic out"}],"title":"Client traffic out","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":12,"y":50},"id":9,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(etcd_network_peer_received_bytes_total{job=~\".*etcd.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(instance,cluster)","legendFormat":"{{instance}} peer traffic in"}],"title":"Peer traffic in","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":18,"y":50},"id":10,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(etcd_network_peer_sent_bytes_total{job=~\".*etcd.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(instance,cluster)","legendFormat":"{{instance}} peer traffic out"}],"title":"Peer traffic out","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"}}},"gridPos":{"h":7,"w":8,"x":0,"y":75},"id":11,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\",cluster=~\"$cluster\"}[1d])","legendFormat":"{{instance}} total leader elections per day"}],"title":"Raft proposals","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"}}},"gridPos":{"h":7,"w":8,"x":8,"y":75},"id":12,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\",cluster=~\"$cluster\"}[1d])","legendFormat":"{{instance}} total leader elections per day"}],"title":"Total leader elections per day","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"s"}},"gridPos":{"h":7,"w":8,"x":16,"y":75},"id":13,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~\".*etcd.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(instance,le,cluster))","legendFormat":"{{instance}} peer round trip time"}],"title":"Peer round trip time","type":"timeseries"}],"refresh":"10s","schemaVersion":36,"tags":["etcd-mixin","vm-k8s-stack"],"templating":{"list":[{"label":"Data Source","name":"datasource","query":"prometheus","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":2,"label":"cluster","name":"cluster","query":".*","refresh":2,"type":"constant"}]},"time":{"from":"now-15m","to":"now"},"timezone":"utc","title":"etcd","uid":"c2f4e12cdf69feb95caa41a5a1b423d9"}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-grafana-overview
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  grafana-overview.json: | 
    {"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":false,"fiscalYearStartMonth":0,"graphTooltip":0,"id":23,"links":[],"panels":[{"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":6,"x":0,"y":0},"id":6,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.0.2","targets":[{"datasource":{"uid":"$datasource"},"expr":"grafana_alerting_result_total{job=~\"$job\",instance=~\"$instance\",state=\"alerting\",cluster=~\"$cluster\"}","instant":true,"interval":"1m","legendFormat":"","refId":"A"}],"title":"Firing Alerts","type":"stat"},{"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":6,"x":6,"y":0},"id":8,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.0.2","targets":[{"datasource":{"uid":"$datasource"},"expr":"sum(grafana_stat_totals_dashboard{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","interval":"1m","legendFormat":"","refId":"A"}],"title":"Dashboards","type":"stat"},{"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"custom":{"cellOptions":{"type":"auto"},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":12,"x":12,"y":0},"id":10,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"12.0.2","targets":[{"datasource":{"uid":"$datasource"},"expr":"grafana_build_info{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}","instant":true,"interval":"1m","legendFormat":"","refId":"A"}],"title":"Build Info","transformations":[{"id":"labelsToFields","options":{}},{"id":"merge","options":{}},{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"branch":true,"container":true,"goversion":true,"namespace":true,"pod":true,"revision":true},"indexByName":{"Time":7,"Value":11,"branch":4,"container":8,"edition":2,"goversion":6,"instance":1,"job":0,"namespace":9,"pod":10,"revision":5,"version":3},"renameByName":{}}}],"type":"table"},{"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"reqps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":5},"id":2,"options":{"alertThreshold":true,"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.0.2","targets":[{"datasource":{"uid":"$datasource"},"expr":"sum(irate(grafana_http_request_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1m])) by(status_code,cluster)","interval":"1m","legendFormat":"{{status_code}}","refId":"A"}],"title":"RPS","type":"timeseries"},{"datasource":{"uid":"$datasource"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":5},"id":4,"options":{"alertThreshold":true,"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.0.2","targets":[{"datasource":{"uid":"$datasource"},"exemplar":true,"expr":"histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\",job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(le,cluster)) * 1","interval":"1m","legendFormat":"99th Percentile","refId":"A"},{"datasource":{"uid":"$datasource"},"exemplar":true,"expr":"histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\",job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(le,cluster)) * 1","interval":"1m","legendFormat":"50th Percentile","refId":"B"},{"datasource":{"uid":"$datasource"},"exemplar":true,"expr":"(sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\",job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) * 1) / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\",job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"1m","legendFormat":"Average","refId":"C"}],"title":"Request Latency","type":"timeseries"}],"preload":false,"refresh":"","schemaVersion":41,"tags":["vm-k8s-stack"],"templating":{"list":[{"current":{"text":"Prometheus","value":"prometheus"},"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":".*","current":{"text":"All","value":["$__all"]},"datasource":"$datasource","definition":"label_values(grafana_build_info, job)","includeAll":true,"multi":true,"name":"job","options":[],"query":{"query":"label_values(grafana_build_info, job)","refId":"Billing Admin-job-Variable-Query"},"refresh":1,"regex":"","type":"query"},{"allValue":".*","current":{"text":"All","value":"$__all"},"datasource":"$datasource","definition":"label_values(grafana_build_info, instance)","includeAll":true,"multi":true,"name":"instance","options":[],"query":{"query":"label_values(grafana_build_info, instance)","refId":"Billing Admin-instance-Variable-Query"},"refresh":1,"regex":"","type":"query"},{"allValue":".*","datasource":{"type":"prometheus"},"hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","query":".*","type":"constant"}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone":"utc","title":"Grafana Overview","uid":"6be0s85Mk","version":1}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-kubernetes-controller-manager
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  kubernetes-controller-manager.json: | 
    {"editable":false,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(up{cluster=~\"$cluster\",job=\"kube-controller-manager\"}) by(cluster)","instant":true}],"title":"Up","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":20,"x":4,"y":0},"id":2,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(workqueue_adds_total{cluster=~\"$cluster\",job=\"kube-controller-manager\",instance=~\"$instance\"}[$__rate_interval])) by(cluster,instance,name)","legendFormat":"{{cluster}} {{instance}} {{name}}"}],"title":"Work Queue Add Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":24,"x":0,"y":7},"id":3,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(workqueue_depth{cluster=~\"$cluster\",job=\"kube-controller-manager\",instance=~\"$instance\"}[$__rate_interval])) by(cluster,instance,name)","legendFormat":"{{cluster}} {{instance}} {{name}}"}],"title":"Work Queue Depth","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":4,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kube-controller-manager\",instance=~\"$instance\"}[$__rate_interval])) by(cluster,instance,name,le))","legendFormat":"{{cluster}} {{instance}} {{name}}"}],"title":"Work Queue Latency","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":8,"x":0,"y":21},"id":5,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\",instance=~\"$instance\",code=~\"2..\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","legendFormat":"2xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\",instance=~\"$instance\",code=~\"3..\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","legendFormat":"3xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\",instance=~\"$instance\",code=~\"4..\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","legendFormat":"4xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\",instance=~\"$instance\",code=~\"5..\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","legendFormat":"5xx"}],"title":"Kube API Request Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":16,"x":8,"y":21},"id":6,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kube-controller-manager\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by(verb,le,cluster))","legendFormat":"{{verb}}"}],"title":"Post Request Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":28},"id":7,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kube-controller-manager\",instance=~\"$instance\",verb=\"GET\"}[$__rate_interval])) by(verb,le,cluster))","legendFormat":"{{verb}}"}],"title":"Get Request Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":0,"y":35},"id":8,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"process_resident_memory_bytes{cluster=~\"$cluster\",job=\"kube-controller-manager\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Memory","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":8,"y":35},"id":9,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"rate(process_cpu_seconds_total{cluster=~\"$cluster\",job=\"kube-controller-manager\",instance=~\"$instance\"}[$__rate_interval])","legendFormat":"{{instance}}"}],"title":"CPU usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":16,"y":35},"id":10,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"go_goroutines{cluster=~\"$cluster\",job=\"kube-controller-manager\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Goroutines","type":"timeseries"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin","vm-k8s-stack"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":2,"label":"cluster","name":"cluster","query":".*","refresh":2,"sort":1,"type":"constant"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":0,"includeAll":true,"label":"instance","name":"instance","query":"label_values(up{cluster=~\"$cluster\",job=\"kube-controller-manager\"}, instance)","refresh":2,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timezone":"utc","title":"Kubernetes / Controller Manager","uid":"72e0e05bef5099e5f049b05fdc429ed4"}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-kubernetes-kubelet
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  kubernetes-kubelet.json: | 
    {"editable":false,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kubelet_node_name{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\"}) by(cluster)","instant":true}],"title":"Running Kubelets","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":4,"y":0},"id":2,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kubelet_running_pods{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}) by(cluster)","instant":true}],"title":"Running Pods","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":8,"y":0},"id":3,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kubelet_running_containers{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}) by(cluster)","instant":true}],"title":"Running Containers","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":12,"y":0},"id":4,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(volume_manager_total_volumes{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\",state=\"actual_state_of_world\"}) by(cluster)","instant":true}],"title":"Actual Volume Count","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":16,"y":0},"id":5,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(volume_manager_total_volumes{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\",state=\"desired_state_of_world\"}) by(cluster)","instant":true}],"title":"Desired Volume Count","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":20,"y":0},"id":6,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(kubelet_node_config_error{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(cluster)","instant":true}],"title":"Config Error Count","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":12,"x":0,"y":7},"id":7,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(kubelet_runtime_operations_total{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(operation_type,instance,cluster)","legendFormat":"{{instance}} {{operation_type}}"}],"title":"Operation Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":12,"x":12,"y":7},"id":8,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(kubelet_runtime_operations_errors_total{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,operation_type,cluster)","legendFormat":"{{instance}} {{operation_type}}"}],"title":"Operation Error Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":9,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,operation_type,le,cluster))","legendFormat":"{{instance}} {{operation_type}}"}],"title":"Operation Duration 99th quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":12,"x":0,"y":21},"id":10,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(kubelet_pod_start_duration_seconds_count{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,cluster)","legendFormat":"{{instance}} pod"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,cluster)","legendFormat":"{{instance}} worker"}],"title":"Pod Start Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":12,"x":12,"y":21},"id":11,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,le,cluster))","legendFormat":"{{instance}} pod"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,le,cluster))","legendFormat":"{{instance}} worker"}],"title":"Pod Start Duration","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":12,"x":0,"y":28},"id":12,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(storage_operation_duration_seconds_count{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,operation_name,volume_plugin,cluster)","legendFormat":"{{instance}} {{operation_name}} {{volume_plugin}}"}],"title":"Storage Operation Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":12,"x":12,"y":28},"id":13,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(storage_operation_errors_total{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,operation_name,volume_plugin,cluster)","legendFormat":"{{instance}} {{operation_name}} {{volume_plugin}}"}],"title":"Storage Operation Error Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":35},"id":14,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,operation_name,volume_plugin,le,cluster))","legendFormat":"{{instance}} {{operation_name}} {{volume_plugin}}"}],"title":"Storage Operation Duration 99th quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":12,"x":0,"y":42},"id":15,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,operation_type,cluster)","legendFormat":"{{operation_type}}"}],"title":"Cgroup manager operation rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":12,"x":12,"y":42},"id":16,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,operation_type,le,cluster))","legendFormat":"{{instance}} {{operation_type}}"}],"title":"Cgroup manager 99th quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":12,"x":0,"y":49},"id":17,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,cluster)","legendFormat":"{{instance}}"}],"title":"PLEG relist rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":12,"x":12,"y":49},"id":18,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,le,cluster))","legendFormat":"{{instance}}"}],"title":"PLEG relist interval","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":56},"id":19,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,le,cluster))","legendFormat":"{{instance}}"}],"title":"PLEG relist duration","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":24,"x":0,"y":63},"id":20,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\",code=~\"2..\"}[$__rate_interval])) by(cluster)","legendFormat":"2xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\",code=~\"3..\"}[$__rate_interval])) by(cluster)","legendFormat":"3xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\",code=~\"4..\"}[$__rate_interval])) by(cluster)","legendFormat":"4xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\",code=~\"5..\"}[$__rate_interval])) by(cluster)","legendFormat":"5xx"}],"title":"RPC rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":70},"id":21,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by(instance,verb,le,cluster))","legendFormat":"{{instance}} {{verb}}"}],"title":"Request duration 99th quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":0,"y":77},"id":22,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"process_resident_memory_bytes{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Memory","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":8,"y":77},"id":23,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"rate(process_cpu_seconds_total{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])","legendFormat":"{{instance}}"}],"title":"CPU usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":16,"y":77},"id":24,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"go_goroutines{cluster=~\"$cluster\",job=\"kubelet\",metrics_path=\"/metrics\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Goroutines","type":"timeseries"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin","vm-k8s-stack"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":2,"label":"cluster","name":"cluster","query":".*","refresh":2,"sort":1,"type":"constant"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":0,"includeAll":true,"label":"instance","name":"instance","query":"label_values(up{job=\"kubelet\",metrics_path=\"/metrics\",cluster=~\"$cluster\"}, instance)","refresh":2,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timezone":"utc","title":"Kubernetes / Kubelet","uid":"3138fa155d5915769fbded898ac09fd9"}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-kubernetes-scheduler
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  kubernetes-scheduler.json: | 
    {"editable":false,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(up{cluster=~\"$cluster\",job=\"kube-scheduler\"}) by(cluster)","instant":true}],"title":"Up","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":10,"x":4,"y":0},"id":2,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(scheduler_scheduling_attempt_duration_seconds_count{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by(cluster,instance)","legendFormat":"{{cluster}} {{instance}} e2e"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(scheduler_pod_scheduling_sli_duration_seconds_count{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by(cluster,instance)","legendFormat":"{{cluster}} {{instance}} binding"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by(cluster,instance)","legendFormat":"{{cluster}} {{instance}} scheduling algorithm"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by(cluster,instance)","legendFormat":"{{cluster}} {{instance}} volume"}],"title":"Scheduling Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":10,"x":14,"y":0},"id":3,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(scheduler_scheduling_attempt_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by(cluster,instance,le))","legendFormat":"{{cluster}} {{instance}} e2e"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(scheduler_pod_scheduling_sli_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by(cluster,instance,le))","legendFormat":"{{cluster}} {{instance}} binding"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by(cluster,instance,le))","legendFormat":"{{cluster}} {{instance}} scheduling algorithm"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by(cluster,instance,le))","legendFormat":"{{cluster}} {{instance}} volume"}],"title":"Scheduling latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":8,"x":0,"y":7},"id":4,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\",code=~\"2..\"}[$__rate_interval])) by(cluster)","legendFormat":"2xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\",code=~\"3..\"}[$__rate_interval])) by(cluster)","legendFormat":"3xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\",code=~\"4..\"}[$__rate_interval])) by(cluster)","legendFormat":"4xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\",code=~\"5..\"}[$__rate_interval])) by(cluster)","legendFormat":"5xx"}],"title":"Kube API Request Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":16,"x":8,"y":7},"id":5,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by(verb,le,cluster))","legendFormat":"{{verb}}"}],"title":"Post Request Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":6,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\",verb=\"GET\"}[$__rate_interval])) by(verb,le,cluster))","legendFormat":"{{verb}}"}],"title":"Get Request Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":0,"y":21},"id":7,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"process_resident_memory_bytes{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Memory","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":8,"y":21},"id":8,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"rate(process_cpu_seconds_total{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])","legendFormat":"{{instance}}"}],"title":"CPU usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":16,"y":21},"id":9,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"go_goroutines{cluster=~\"$cluster\",job=\"kube-scheduler\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Goroutines","type":"timeseries"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin","vm-k8s-stack"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":2,"label":"cluster","name":"cluster","query":".*","refresh":2,"sort":1,"type":"constant"},{"allValue":".+","datasource":{"type":"prometheus","uid":"${datasource}"},"hide":0,"includeAll":true,"label":"instance","name":"instance","query":"label_values(up{job=\"kube-scheduler\",cluster=~\"$cluster\"}, instance)","refresh":2,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timezone":"utc","title":"Kubernetes / Scheduler","uid":"2e6b6a3b4bddf1427b3a55aa1311c656"}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-kubernetes-system-api-server
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  kubernetes-system-api-server.json: | 
    {"__elements":[],"__inputs":[{"description":"","label":"Prometheus","name":"DS_PROMETHEUS","pluginId":"prometheus","pluginName":"Prometheus","type":"datasource"}],"__requires":[{"id":"grafana","name":"Grafana","type":"grafana","version":"8.4.4"},{"id":"prometheus","name":"Prometheus","type":"datasource","version":"5.0.0"},{"id":"timeseries","name":"Time series","type":"panel","version":""},{"id":"stat","name":"Stat","type":"panel","version":""}],"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"#5c4ee5","name":"terraform","target":{"limit":100,"matchAny":false,"tags":["terraform"],"type":"tags"}},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"red","name":"oncall","target":{"limit":100,"matchAny":false,"tags":["oncall"],"type":"tags"}}]},"description":"This is a modern API Server dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes","editable":false,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[{"options":{"0":{"text":"DOWN"},"1":{"text":"UP"}},"type":"value"}],"thresholds":{"mode":"absolute","steps":[{"color":"red","value":null},{"color":"green","value":1}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"id":42,"options":{"colorMode":"background","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"value_and_name"},"pluginVersion":"10.0.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"up{cluster=~\"$cluster\",job=~\"$job\"}","interval":"","legendFormat":"{{ instance }}","refId":"A"}],"title":"API Server - Health Status","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"custom":{"align":"auto","cellOptions":{"type":"auto"},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"__name__"},"properties":[{"id":"custom.width","value":188}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"id":60,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true,"sortBy":[{"desc":false,"displayName":"removed_release"}]},"pluginVersion":"10.0.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"apiserver_requested_deprecated_apis{cluster=~\"$cluster\"}","interval":"","legendFormat":"","refId":"A"}],"title":"Deprecated Kubernetes Resources","transformations":[{"id":"labelsToFields","options":{"keepLabels":["group","job","removed_release","resource","version","name"],"mode":"columns"}},{"id":"merge","options":{}},{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"job":true},"indexByName":{"Time":6,"Value":7,"group":1,"job":5,"namespace":0,"removed_release":4,"resource":3,"version":2},"renameByName":{}}},{"id":"groupBy","options":{"fields":{"group":{"aggregations":["lastNotNull"],"operation":"groupby"},"job":{"aggregations":[],"operation":"groupby"},"namespace":{"aggregations":["lastNotNull"],"operation":"groupby"},"removed_release":{"aggregations":[],"operation":"groupby"},"resource":{"aggregations":["lastNotNull"],"operation":"groupby"},"version":{"aggregations":[],"operation":"groupby"}}}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"id":38,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(apiserver_request_total{cluster=~\"$cluster\"}[$__rate_interval])) by(code,cluster)","interval":"$resolution","legendFormat":"{{ code }}","refId":"A"}],"title":"API Server - HTTP Requests by code","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"id":39,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(apiserver_request_total{cluster=~\"$cluster\"}[$__rate_interval])) by(verb,cluster)","interval":"$resolution","legendFormat":"{{ verb}}","refId":"A"}],"title":"API Server - HTTP Requests by verb","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"id":53,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(apiserver_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(instance,cluster) / sum(rate(apiserver_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(instance,cluster)","interval":"$resolution","legendFormat":"{{ instance }}","refId":"A"}],"title":"API Server - HTTP Requests Latency by instance","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"id":54,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(apiserver_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(verb,cluster) / sum(rate(apiserver_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(verb,cluster)","interval":"$resolution","legendFormat":"{{ verb }}","refId":"A"}],"title":"API Server - HTTP Requests Latency by verb","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"id":50,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(apiserver_request_total{code=~\"5..\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(instance,cluster) / sum(rate(apiserver_request_total{cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(instance,cluster)","interval":"$resolution","legendFormat":"{{ instance }}","refId":"A"}],"title":"API Server - Errors by Instance","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"id":51,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(apiserver_request_total{code=~\"5..\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(verb,cluster) / sum(rate(apiserver_request_total{cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(verb,cluster)","interval":"$resolution","legendFormat":"{{ verb }}","refId":"A"}],"title":"API Server - Errors by verb","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"id":40,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(apiserver_request_total{cluster=~\"$cluster\"}[$__rate_interval])) by(instance,cluster)","interval":"$resolution","legendFormat":"{{ instance }}","refId":"A"}],"title":"API Server - Stacked HTTP Requests by instance","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"id":56,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(workqueue_depth{cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(instance,cluster)","interval":"$resolution","legendFormat":"{{ instance }}","refId":"A"}],"title":"API Server - Work Queue by instance","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"id":47,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"rate(process_cpu_seconds_total{cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])","interval":"$resolution","legendFormat":"{{ instance }}","refId":"A"}],"title":"API Server - CPU Usage by instance","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"id":48,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"process_resident_memory_bytes{cluster=~\"$cluster\",job=~\"$job\"}","interval":"$resolution","legendFormat":"{{ instance }}","refId":"A"}],"title":"API Server - Memory Usage by instance","type":"timeseries"}],"refresh":"30s","schemaVersion":38,"style":"dark","tags":["Kubernetes","Prometheus","vm-k8s-stack"],"templating":{"list":[{"current":{"selected":false,"text":"Prometheus","value":"Prometheus"},"hide":0,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{"isNone":true,"selected":false,"text":"None","value":""},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":".*","hide":2,"name":"cluster","options":[],"query":".*","refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"constant"},{"current":{"selected":true,"text":"30s","value":"30s"},"hide":0,"name":"resolution","options":[{"selected":false,"text":"1s","value":"1s"},{"selected":false,"text":"15s","value":"15s"},{"selected":true,"text":"30s","value":"30s"},{"selected":false,"text":"1m","value":"1m"},{"selected":false,"text":"3m","value":"3m"},{"selected":false,"text":"5m","value":"5m"}],"query":"1s, 15s, 30s, 1m, 3m, 5m","queryValue":"","skipUrlSync":false,"type":"custom"},{"current":{},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(apiserver_request_total{cluster=~\"$cluster\"}, job)","hide":0,"multi":true,"name":"job","options":[],"query":{"qryType":1,"query":"label_values(apiserver_request_total{cluster=~\"$cluster\"}, job)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{},"timezone":"utc","title":"Kubernetes / System / API Server","uid":"k8s_system_apisrv","version":20,"weekStart":""}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-kubernetes-system-coredns
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  kubernetes-system-coredns.json: | 
    {"__elements":[],"__inputs":[{"description":"","label":"Prometheus","name":"DS_PROMETHEUS","pluginId":"prometheus","pluginName":"Prometheus","type":"datasource"}],"__requires":[{"id":"grafana","name":"Grafana","type":"grafana","version":"8.4.4"},{"id":"prometheus","name":"Prometheus","type":"datasource","version":"5.0.0"},{"id":"timeseries","name":"Time series","type":"panel","version":""},{"id":"stat","name":"Stat","type":"panel","version":""}],"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"#5c4ee5","name":"terraform","target":{"limit":100,"matchAny":false,"tags":["terraform"],"type":"tags"}},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"red","name":"oncall","target":{"limit":100,"matchAny":false,"tags":["oncall"],"type":"tags"}}]},"description":"This is a modern CoreDNS dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes","editable":false,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[{"options":{"0":{"text":"DOWN"},"1":{"text":"UP"}},"type":"value"}],"thresholds":{"mode":"absolute","steps":[{"color":"red","value":null},{"color":"green","value":1}]}},"overrides":[]},"gridPos":{"h":3,"w":24,"x":0,"y":0},"id":25,"options":{"colorMode":"background","graphMode":"none","justifyMode":"auto","orientation":"vertical","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"value_and_name","wideLayout":true},"pluginVersion":"10.4.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"up{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}","interval":"","legendFormat":"{{ instance }}","refId":"A"}],"title":"CoreDNS - Health Status","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":3},"id":19,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"$resolution","legendFormat":"{{ instance }}","refId":"A"}],"title":"CoreDNS - CPU Usage by instance","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":3},"id":21,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}","interval":"","legendFormat":"{{ instance }}","refId":"A"}],"title":"CoreDNS - Memory Usage by instance","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":11},"id":9,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(coredns_dns_requests_total{instance=~\"$instance\",proto=\"$protocol\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"total $protocol requests","refId":"A"}],"title":"CoreDNS - Total DNS Requests ($protocol)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":11},"id":7,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(coredns_dns_request_size_bytes_sum{instance=~\"$instance\",proto=\"$protocol\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(proto,cluster) / sum(rate(coredns_dns_request_size_bytes_count{instance=~\"$instance\",proto=\"$protocol\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(proto,cluster)","interval":"$resolution","legendFormat":"average $protocol packet size","refId":"A"}],"title":"CoreDNS - Average Packet Size  ($protocol)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":19},"id":2,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(coredns_dns_requests_total{instance=~\"$instance\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(type,cluster)","interval":"$resolution","legendFormat":"{{ type }}","refId":"A"}],"title":"CoreDNS - Requests by type","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":19},"id":4,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(coredns_dns_responses_total{instance=~\"$instance\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(rcode,cluster)","interval":"$resolution","legendFormat":"{{ rcode }}","refId":"A"}],"title":"CoreDNS - Requests by return code","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":27},"id":23,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(coredns_proxy_request_duration_seconds_count{proxy_name=\"forward\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"total forward requests","refId":"A"}],"title":"CoreDNS - Total Forward Requests","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":27},"id":13,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(coredns_proxy_request_duration_seconds_count{proxy_name=\"forward\",rcode=~\"SERVFAIL|REFUSED\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(rcode,cluster)","interval":"$resolution","legendFormat":"{{ rcode }}","refId":"A"}],"title":"CoreDNS - DNS Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":35},"id":17,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(coredns_cache_hits_total{instance=~\"$instance\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(type,cluster)","interval":"$resolution","legendFormat":"{{ type }}","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(coredns_cache_misses_total{instance=~\"$instance\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(type,cluster)","interval":"$resolution","legendFormat":"misses","refId":"B"}],"title":"CoreDNS - Cache Hits / Misses","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":35},"id":15,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(coredns_cache_entries{cluster=~\"$cluster\",job=~\"$job\"}) by(type,cluster)","interval":"","legendFormat":"{{ type }}","refId":"A"}],"title":"CoreDNS - Cache Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false},"scaleDistribution":{"type":"linear"}}},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":43},"id":27,"options":{"calculate":false,"cellGap":1,"color":{"exponent":0.5,"fill":"dark-orange","mode":"scheme","reverse":false,"scale":"exponential","scheme":"RdYlBu","steps":64},"exemplars":{"color":"rgba(255,0,255,0.7)"},"filterValues":{"le":1e-9},"legend":{"show":true},"rowsFrame":{"layout":"auto"},"tooltip":{"mode":"single","showColorScale":false,"yHistogram":false},"yAxis":{"axisPlacement":"left","reverse":false,"unit":"s"}},"pluginVersion":"10.4.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(increase(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(le,cluster)","format":"heatmap","legendFormat":"{{le}}","range":true,"refId":"A"}],"title":"CoreDNS - DNS request duration","type":"heatmap"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false},"scaleDistribution":{"type":"linear"}}},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":43},"id":28,"options":{"calculate":false,"cellGap":1,"color":{"exponent":0.5,"fill":"dark-orange","mode":"scheme","reverse":false,"scale":"exponential","scheme":"RdYlBu","steps":64},"exemplars":{"color":"rgba(255,0,255,0.7)"},"filterValues":{"le":1e-9},"legend":{"show":true},"rowsFrame":{"layout":"auto"},"tooltip":{"mode":"single","showColorScale":false,"yHistogram":false},"yAxis":{"axisPlacement":"left","reverse":false,"unit":"decbytes"}},"pluginVersion":"10.4.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(increase(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",le!=\"0\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(le,cluster)","format":"heatmap","legendFormat":"{{le}}","range":true,"refId":"A"}],"title":"CoreDNS - DNS request size","type":"heatmap"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false},"scaleDistribution":{"type":"linear"}}},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":53},"id":29,"options":{"calculate":false,"cellGap":1,"color":{"exponent":0.5,"fill":"dark-orange","mode":"scheme","reverse":false,"scale":"exponential","scheme":"RdYlBu","steps":64},"exemplars":{"color":"rgba(255,0,255,0.7)"},"filterValues":{"le":1e-9},"legend":{"show":true},"rowsFrame":{"layout":"auto"},"tooltip":{"mode":"single","showColorScale":false,"yHistogram":false},"yAxis":{"axisPlacement":"left","reverse":false,"unit":"decbytes"}},"pluginVersion":"10.4.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(increase(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",le!=\"0\",cluster=~\"$cluster\",job=~\"$job\"}[$__rate_interval])) by(le,cluster)","format":"heatmap","legendFormat":"{{le}}","range":true,"refId":"A"}],"title":"CoreDNS - DNS response size","type":"heatmap"}],"refresh":"30s","schemaVersion":39,"tags":["Kubernetes","Prometheus","vm-k8s-stack"],"templating":{"list":[{"current":{"selected":false,"text":"Prometheus","value":"Prometheus"},"hide":0,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{"isNone":true,"selected":false,"text":"None","value":""},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":".*","hide":2,"name":"cluster","options":[],"query":".*","refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"constant"},{"allValue":".*","current":{"selected":false,"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(up{job=\"$job\",cluster=~\"$cluster\"}, instance)","hide":0,"includeAll":true,"name":"instance","options":[],"query":{"qryType":1,"query":"label_values(up{job=\"$job\",cluster=~\"$cluster\"}, instance)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"allValue":"udp,tcp","current":{"selected":false,"text":"udp","value":"udp"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(coredns_dns_requests_total{cluster=~\"$cluster\"}, proto)","hide":0,"name":"protocol","options":[],"query":{"query":"label_values(coredns_dns_requests_total{cluster=~\"$cluster\"}, proto)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"30s","value":"30s"},"hide":0,"name":"resolution","options":[{"selected":false,"text":"1s","value":"1s"},{"selected":false,"text":"15s","value":"15s"},{"selected":true,"text":"30s","value":"30s"},{"selected":false,"text":"1m","value":"1m"},{"selected":false,"text":"3m","value":"3m"},{"selected":false,"text":"5m","value":"5m"}],"query":"1s, 15s, 30s, 1m, 3m, 5m","queryValue":"","skipUrlSync":false,"type":"custom"},{"current":{"selected":true,"text":["coredns"],"value":["coredns"]},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(coredns_build_info{cluster=~\"$cluster\"}, job)","hide":0,"multi":true,"name":"job","options":[],"query":{"qryType":1,"query":"label_values(coredns_build_info{cluster=~\"$cluster\"}, job)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{},"timezone":"utc","title":"Kubernetes / System / CoreDNS","uid":"k8s_system_coredns","version":22,"weekStart":""}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-kubernetes-views-global
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  kubernetes-views-global.json: | 
    {"__elements":{},"__inputs":[{"description":"","label":"Prometheus","name":"DS_PROMETHEUS","pluginId":"prometheus","pluginName":"Prometheus","type":"datasource"}],"__requires":[{"id":"bargauge","name":"Bar gauge","type":"panel","version":""},{"id":"grafana","name":"Grafana","type":"grafana","version":"10.3.1"},{"id":"prometheus","name":"Prometheus","type":"datasource","version":"1.0.0"},{"id":"stat","name":"Stat","type":"panel","version":""},{"id":"timeseries","name":"Time series","type":"panel","version":""}],"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"#5c4ee5","name":"terraform","target":{"limit":100,"matchAny":false,"tags":["terraform"],"type":"tags"}},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"red","name":"oncall","target":{"limit":100,"matchAny":false,"tags":["oncall"],"type":"tags"}}]},"description":"This is a modern 'Global View' dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes","editable":false,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"prometheus","uid":"${datasource}"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":67,"title":"Overview","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"continuous-GrYlRd"},"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":0,"y":1},"id":77,"options":{"displayMode":"lcd","maxVizHeight":300,"minVizHeight":10,"minVizWidth":0,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"avg(sum(rate(node_cpu_seconds_total{mode!~\"idle|iowait|steal\",cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(instance,cpu,cluster)) by(cluster)","interval":"","legendFormat":"Real","range":true,"refId":"Real"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_requests{resource=\"cpu\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster) / sum(machine_cpu_cores{cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"Requests","range":true,"refId":"Requests"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_limits{resource=\"cpu\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster) / sum(machine_cpu_cores{cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"Limits","range":true,"refId":"Limits"}],"title":"Global CPU  Usage","type":"bargauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"continuous-GrYlRd"},"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":6,"y":1},"id":78,"options":{"displayMode":"lcd","maxVizHeight":300,"minVizHeight":10,"minVizWidth":0,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","text":{},"valueMode":"color"},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(node_memory_MemTotal_bytes{cluster=~\"$cluster\",job=\"$job\"} - node_memory_MemAvailable_bytes{cluster=~\"$cluster\",job=\"$job\"}) by(cluster) / sum(node_memory_MemTotal_bytes{cluster=~\"$cluster\",job=\"$job\"}) by(cluster)","hide":false,"interval":"","legendFormat":"Real","range":true,"refId":"Real"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_requests{resource=\"memory\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster) / sum(machine_memory_bytes{cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"Requests","range":true,"refId":"Requests"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_limits{resource=\"memory\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster) / sum(machine_memory_bytes{cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"Limits","range":true,"refId":"Limits"}],"title":"Global RAM Usage","type":"bargauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":2,"x":12,"y":1},"id":63,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"value","wideLayout":true},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"count(count(kube_node_info{cluster=~\"$cluster\"}) by(node,cluster)) by(cluster)","interval":"","legendFormat":"","range":true,"refId":"A"}],"title":"Nodes","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":12,"w":10,"x":14,"y":1},"id":52,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(kube_namespace_labels{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Namespaces","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_status_running{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Running Containers","refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_status_phase{phase=\"Running\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Running Pods","refId":"O"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_service_info{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Services","refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_endpoint_info{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Endpoints","refId":"D"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_ingress_info{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Ingresses","refId":"E"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_deployment_labels{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Deployments","refId":"F"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_statefulset_labels{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Statefulsets","refId":"G"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_daemonset_labels{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Daemonsets","refId":"H"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_persistentvolumeclaim_info{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Persistent Volume Claims","refId":"I"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_hpa_labels{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Horizontal Pod Autoscalers","refId":"J"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_configmap_info{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Configmaps","refId":"K"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_secret_info{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Secrets","refId":"L"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_networkpolicy_labels{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Network Policies","refId":"M"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"count(count(kube_node_info{cluster=~\"$cluster\"}) by(node,cluster)) by(cluster)","hide":false,"interval":"","legendFormat":"Nodes","refId":"N"}],"title":"Kubernetes Resource Count","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":2,"x":12,"y":5},"id":59,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"value","wideLayout":true},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"count(kube_namespace_created{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"","refId":"A"}],"title":"Namespaces","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"none"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":0,"y":9},"id":37,"options":{"colorMode":"none","graphMode":"none","justifyMode":"center","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(node_cpu_seconds_total{mode!~\"idle|iowait|steal\",cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(cluster)","interval":"","legendFormat":"Real","range":true,"refId":"Real"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_requests{resource=\"cpu\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","hide":false,"legendFormat":"Requests","range":true,"refId":"Requests"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_limits{resource=\"cpu\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","hide":false,"legendFormat":"Limits","range":true,"refId":"Limits"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(machine_cpu_cores{cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"Total","range":true,"refId":"Total"}],"title":"CPU Usage","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":6,"y":9},"id":39,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(node_memory_MemTotal_bytes{cluster=~\"$cluster\",job=\"$job\"} - node_memory_MemAvailable_bytes{cluster=~\"$cluster\",job=\"$job\"}) by(cluster)","interval":"","legendFormat":"Real","range":true,"refId":"Real"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_requests{resource=\"memory\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","hide":false,"legendFormat":"Requests","range":true,"refId":"Requests"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_limits{resource=\"memory\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","hide":false,"legendFormat":"Limits","range":true,"refId":"Limits"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(machine_memory_bytes{cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"Total","range":true,"refId":"Total"}],"title":"RAM Usage","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":2,"x":12,"y":9},"id":62,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"value","wideLayout":true},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_status_phase{phase=\"Running\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"","refId":"A"}],"title":"Running Pods","type":"stat"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"${datasource}"},"gridPos":{"h":1,"w":24,"x":0,"y":13},"id":71,"title":"Resources","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"continuous-GrYlRd","seriesBy":"last"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"CPU %","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"scheme","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":0.5},{"color":"red","value":0.7}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":14},"id":72,"options":{"legend":{"calcs":[],"displayMode":"hidden","placement":"right","showLegend":false},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"avg(sum(rate(node_cpu_seconds_total{mode!~\"idle|iowait|steal\",cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(instance,cpu,cluster)) by(cluster)","interval":"$resolution","legendFormat":"CPU usage in %","range":true,"refId":"A"}],"title":"Cluster CPU Utilization","transformations":[{"id":"calculateField","options":{"alias":"CPU usage in %","mode":"reduceRow","reduce":{"reducer":"mean"},"replaceFields":true}}],"type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"continuous-GrYlRd"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"MEMORY","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"scheme","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"#EAB839","value":0.5},{"color":"red","value":0.7}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":14},"id":55,"options":{"legend":{"calcs":["mean","lastNotNull","max","min"],"displayMode":"hidden","placement":"right","showLegend":false},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(node_memory_MemTotal_bytes{cluster=~\"$cluster\",job=\"$job\"} - node_memory_MemAvailable_bytes{cluster=~\"$cluster\",job=\"$job\"}) by(cluster) / sum(node_memory_MemTotal_bytes{cluster=~\"$cluster\",job=\"$job\"}) by(cluster)","interval":"$resolution","legendFormat":"Memory usage in %","range":true,"refId":"A"}],"title":"Cluster Memory Utilization","transformations":[{"id":"calculateField","options":{"alias":"Memory usage in %","mode":"reduceRow","reduce":{"reducer":"mean"},"replaceFields":true}}],"type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"CPU CORES","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":22},"id":46,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{image!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,cluster)","format":"time_series","hide":false,"interval":"$resolution","legendFormat":"{{ namespace }}","range":true,"refId":"A"}],"title":"CPU Utilization by namespace","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":22},"id":50,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(container_memory_working_set_bytes{image!=\"\",cluster=~\"$cluster\"}) by(namespace,cluster)","interval":"$resolution","legendFormat":"{{ namespace }}","range":true,"refId":"A"}],"title":"Memory Utilization by namespace","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"CPU %","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":30},"id":54,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"avg(sum(rate(node_cpu_seconds_total{mode!~\"idle|iowait|steal\",cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(instance,cpu,cluster)) by(instance,cluster)","interval":"$resolution","legendFormat":"{{ node }}","range":true,"refId":"A"}],"title":"CPU Utilization by instance","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"MEMORY","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":30},"id":73,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(node_memory_MemTotal_bytes{cluster=~\"$cluster\",job=\"$job\"} - node_memory_MemAvailable_bytes{cluster=~\"$cluster\",job=\"$job\"}) by(instance,cluster)","hide":false,"interval":"$resolution","legendFormat":"{{ instance }}","range":true,"refId":"A"}],"title":"Memory Utilization by instance","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"No data is generally a good thing here.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"SECONDS","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":38},"id":82,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_cpu_cfs_throttled_seconds_total{image!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,cluster) > 0","interval":"$resolution","legendFormat":"{{ namespace }}","range":true,"refId":"A"}],"title":"CPU Throttled seconds by namespace","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"No data is generally a good thing here.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"NB","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":38},"id":83,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(node_cpu_core_throttles_total{cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(instance,cluster)","interval":"$resolution","legendFormat":"{{ instance }}","range":true,"refId":"A"}],"title":"CPU Core Throttled by instance","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":46},"id":86,"title":"Kubernetes","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":0,"y":47},"id":84,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(kube_pod_status_qos_class{cluster=~\"$cluster\"}) by(qos_class,cluster)","interval":"","legendFormat":"{{ qos_class }} pods","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_info{cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"Total pods","range":true,"refId":"B"}],"title":"Kubernetes Pods QoS classes","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":12,"y":47},"id":85,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(kube_pod_status_reason{cluster=~\"$cluster\"}) by(reason,cluster)","interval":"","legendFormat":"{{ reason }}","range":true,"refId":"A"}],"title":"Kubernetes Pods Status Reason","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"No data is generally a good thing here.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"points","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":0,"y":56},"id":87,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(increase(container_oom_events_total{cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,cluster) > 0","interval":"","legendFormat":"{{ namespace }}","range":true,"refId":"A"}],"title":"OOM Events by namespace","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"No data is generally a good thing here.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"points","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":12,"y":56},"id":88,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(increase(kube_pod_container_status_restarts_total{cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,cluster) > 0","interval":"","legendFormat":"{{ namespace }}","range":true,"refId":"A"}],"title":"Container Restarts by namespace","type":"timeseries"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"${datasource}"},"gridPos":{"h":1,"w":24,"x":0,"y":65},"id":69,"title":"Network","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"Dropped noisy virtual devices for readability.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"BANDWIDTH","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"binBps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":66},"id":44,"options":{"legend":{"calcs":[],"displayMode":"hidden","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(node_network_receive_bytes_total{device!~\"(veth|azv|lxc).*\",cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(device,cluster)","interval":"$resolution","legendFormat":"Received : {{ device }}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"0 - sum(rate(node_network_transmit_bytes_total{device!~\"(veth|azv|lxc).*\",cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(device,cluster)","interval":"$resolution","legendFormat":"Transmitted : {{ device }}","range":true,"refId":"B"}],"title":"Global Network Utilization by device","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"DROPPED PACKETS","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":66},"id":53,"options":{"legend":{"calcs":[],"displayMode":"hidden","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(node_network_receive_drop_total{cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Packets dropped (receive)","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"0 - sum(rate(node_network_transmit_drop_total{cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Packets dropped (transmit)","range":true,"refId":"B"}],"title":"Network Saturation - Packets dropped","transformations":[],"type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"BANDWIDTH","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"binBps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":74},"id":79,"options":{"legend":{"calcs":[],"displayMode":"hidden","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_network_receive_bytes_total{cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,cluster)","interval":"$resolution","legendFormat":"Received : {{ namespace }}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"0 - sum(rate(container_network_transmit_bytes_total{cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,cluster)","hide":false,"interval":"$resolution","legendFormat":"Transmitted : {{ namespace }}","range":true,"refId":"B"}],"title":"Network Received by namespace","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"BANDWIDTH","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"binBps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":74},"id":80,"options":{"legend":{"calcs":[],"displayMode":"hidden","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(node_network_receive_bytes_total{cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(instance,cluster)","interval":"$resolution","legendFormat":"Received bytes in {{ instance }}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"0 - sum(rate(node_network_transmit_bytes_total{cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(instance,cluster)","hide":false,"interval":"$resolution","legendFormat":"Transmitted bytes in {{ instance }}","range":true,"refId":"B"}],"title":"Total Network Received (with all virtual devices) by instance","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"Dropped noisy virtual devices for readability.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"BANDWIDTH","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"binBps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":82},"id":56,"options":{"legend":{"calcs":[],"displayMode":"hidden","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(node_network_receive_bytes_total{device!~\"(veth|azv|lxc|lo).*\",cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(instance,cluster)","interval":"$resolution","legendFormat":"Received bytes in {{ instance }}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"0 - sum(rate(node_network_transmit_bytes_total{device!~\"(veth|azv|lxc|lo).*\",cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(instance,cluster)","hide":false,"interval":"$resolution","legendFormat":"Transmitted bytes in {{ instance }}","range":true,"refId":"B"}],"title":"Network Received (without loopback)  by instance","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"Dropped noisy virtual devices for readability.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"BANDWIDTH","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"binBps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":82},"id":81,"options":{"legend":{"calcs":[],"displayMode":"hidden","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(node_network_receive_bytes_total{device=\"lo\",cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(instance,cluster)","interval":"$resolution","legendFormat":"Received bytes in {{ instance }}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"0 - sum(rate(node_network_transmit_bytes_total{device=\"lo\",cluster=~\"$cluster\",job=\"$job\"}[$__rate_interval])) by(instance,cluster)","hide":false,"interval":"$resolution","legendFormat":"Transmitted bytes in {{ instance }}","range":true,"refId":"B"}],"title":"Network Received (loopback only) by instance","type":"timeseries"}],"refresh":"30s","schemaVersion":39,"tags":["Kubernetes","Prometheus","vm-k8s-stack"],"templating":{"list":[{"current":{"selected":false,"text":"","value":""},"hide":0,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":".*","hide":2,"name":"cluster","options":[],"query":".*","refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"constant"},{"current":{"selected":false,"text":"30s","value":"30s"},"hide":0,"name":"resolution","options":[{"selected":false,"text":"1s","value":"1s"},{"selected":false,"text":"15s","value":"15s"},{"selected":true,"text":"30s","value":"30s"},{"selected":false,"text":"1m","value":"1m"},{"selected":false,"text":"3m","value":"3m"},{"selected":false,"text":"5m","value":"5m"}],"query":"1s, 15s, 30s, 1m, 3m, 5m","queryValue":"","skipUrlSync":false,"type":"custom"},{"current":{"selected":false,"text":"","value":""},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(node_cpu_seconds_total{cluster=~\"$cluster\"}, job)","hide":0,"multi":true,"name":"job","options":[],"query":{"qryType":1,"query":"label_values(node_cpu_seconds_total{cluster=~\"$cluster\"}, job)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{},"timezone":"utc","title":"Kubernetes / Views / Global","uid":"k8s_views_global","version":45,"weekStart":""}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-kubernetes-views-namespaces
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  kubernetes-views-namespaces.json: | 
    {"__elements":{},"__inputs":[{"description":"","label":"Prometheus","name":"DS_PROMETHEUS","pluginId":"prometheus","pluginName":"Prometheus","type":"datasource"}],"__requires":[{"id":"gauge","name":"Gauge","type":"panel","version":""},{"id":"grafana","name":"Grafana","type":"grafana","version":"10.3.1"},{"id":"prometheus","name":"Prometheus","type":"datasource","version":"1.0.0"},{"id":"stat","name":"Stat","type":"panel","version":""},{"id":"timeseries","name":"Time series","type":"panel","version":""}],"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"#5c4ee5","name":"terraform","target":{"limit":100,"matchAny":false,"tags":["terraform"],"type":"tags"}},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"red","name":"oncall","target":{"limit":100,"matchAny":false,"tags":["oncall"],"type":"tags"}}]},"description":"This is a modern 'Namespaces View' dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes","editable":false,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":38,"title":"Overview","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null},{"color":"orange","value":50},{"color":"red","value":70}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":1},"id":46,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto","text":{}},"pluginVersion":"11.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",image!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / sum(machine_cpu_cores{cluster=~\"$cluster\"}) by(cluster)","instant":true,"interval":"","legendFormat":"","range":false,"refId":"A"}],"title":"Namespace(s) usage on total cluster CPU in %","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null},{"color":"orange","value":50},{"color":"red","value":70}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":1},"id":48,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto","text":{}},"pluginVersion":"11.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(container_memory_working_set_bytes{namespace=~\"$namespace\",image!=\"\",cluster=~\"$cluster\"}) by(cluster) / sum(machine_memory_bytes{cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"","refId":"A"}],"title":"Namespace(s) usage on total cluster RAM in %","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":11,"w":12,"x":12,"y":1},"id":32,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_info{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Running Pods","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_service_info{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Services","refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_ingress_info{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Ingresses","refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_deployment_labels{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Deployments","refId":"D"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_statefulset_labels{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Statefulsets","refId":"E"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_daemonset_labels{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Daemonsets","refId":"F"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_persistentvolumeclaim_info{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Persistent Volume Claims","refId":"G"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_hpa_labels{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Horizontal Pod Autoscalers","refId":"H"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_configmap_info{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Configmaps","refId":"I"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_secret_info{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Secrets","refId":"J"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_networkpolicy_labels{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Network Policies","refId":"K"}],"title":"Kubernetes Resource Count","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"none"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":0,"y":8},"id":62,"options":{"colorMode":"none","graphMode":"none","justifyMode":"center","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"11.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",image!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"","legendFormat":"Real","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_requests{namespace=~\"$namespace\",resource=\"cpu\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=~\"$namespace\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","hide":false,"legendFormat":"Requests","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",resource=\"cpu\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=~\"$namespace\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","hide":false,"legendFormat":"Limits","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(machine_cpu_cores{cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"Cluster Total","range":true,"refId":"D"}],"title":"Namespace(s) CPU Usage in cores","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":6,"y":8},"id":64,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"11.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(container_memory_working_set_bytes{namespace=~\"$namespace\",image!=\"\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Real","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_requests{namespace=~\"$namespace\",resource=\"memory\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=~\"$namespace\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","hide":false,"legendFormat":"Requests","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",resource=\"memory\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=~\"$namespace\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","hide":false,"legendFormat":"Limits","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(machine_memory_bytes{cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"Cluster Total","range":true,"refId":"D"}],"title":"Namespace(s) RAM Usage in bytes","type":"stat"},{"collapsed":false,"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":1,"w":24,"x":0,"y":12},"id":40,"title":"Resources","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"CPU CORES","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":13},"id":29,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",image!=\"\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(pod,cluster)","interval":"$resolution","legendFormat":"{{ pod }}","range":true,"refId":"A"}],"title":"CPU usage by Pod","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":13},"id":30,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(container_memory_working_set_bytes{namespace=~\"$namespace\",image!=\"\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}) by(pod,cluster)","interval":"$resolution","legendFormat":"{{ pod }}","range":true,"refId":"A"}],"title":"Memory usage by Pod","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"SECONDS","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":21},"id":68,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\",image!=\"\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(pod,cluster) > 0","interval":"$resolution","legendFormat":"{{ pod }}","range":true,"refId":"A"}],"title":"CPU Throttled seconds by pod","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":29},"id":73,"title":"Kubernetes","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":0,"y":30},"id":70,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(kube_pod_status_qos_class{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(qos_class,cluster)","interval":"","legendFormat":"{{ qos_class }} pods","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_info{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"Total pods","range":true,"refId":"B"}],"title":"Kubernetes Pods QoS classes","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":12,"y":30},"id":72,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(kube_pod_status_reason{cluster=~\"$cluster\"}) by(reason,cluster)","interval":"","legendFormat":"{{ reason }}","range":true,"refId":"A"}],"title":"Kubernetes Pods Status Reason","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"No data is generally a good thing here.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"points","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":0,"y":39},"id":74,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(increase(container_oom_events_total{namespace=~\"${namespace}\",cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,pod,cluster) > 0","interval":"","legendFormat":"namespace: {{ namespace }} - pod: {{ pod }}","range":true,"refId":"A"}],"title":"OOM Events by namespace, pod","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"No data is generally a good thing here.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"points","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":12,"y":39},"id":75,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(increase(kube_pod_container_status_restarts_total{namespace=~\"${namespace}\",cluster=~\"$cluster\"}[$__rate_interval])) by(namespace,pod,cluster) > 0","interval":"","legendFormat":"namespace: {{ namespace }} - pod: {{ pod }}","range":true,"refId":"A"}],"title":"Container Restarts by namespace, pod","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"id":5,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_status_ready{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Ready","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_status_running{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Running","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_status_waiting{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Waiting","refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_status_restarts_total{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Restarts Total","refId":"D"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_status_terminated{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"Terminated","refId":"E"}],"title":"Nb of pods by state","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"id":2,"options":{"legend":{"calcs":[],"displayMode":"hidden","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_info{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}) by(pod,cluster)","interval":"","legendFormat":"{{ pod }}","range":true,"refId":"A"}],"title":"Nb of containers by pod","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"id":7,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(kube_deployment_status_replicas_available{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(deployment,cluster)","interval":"","legendFormat":"{{ deployment }}","range":true,"refId":"A"}],"title":"Replicas available by deployment","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"id":8,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_deployment_status_replicas_unavailable{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}) by(deployment,cluster)","interval":"","legendFormat":"{{ deployment }}","range":true,"refId":"A"}],"title":"Replicas unavailable by deployment","type":"timeseries"},{"datasource":{"default":false,"type":"prometheus","uid":"${datasource}"},"description":"List of pods that are not in Running or Succeeded status.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"id":83,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_status_phase{phase!~\"Running|Succeeded\",namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(pod,cluster) > 0","interval":"","legendFormat":"{{ deployment }}","range":true,"refId":"A"}],"title":"Pods with unexpected status","type":"timeseries"},{"datasource":{"default":false,"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"bars","fillOpacity":25,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"id":82,"options":{"legend":{"calcs":["min","max","last"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"count(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",image!=\"\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(image,cluster)","interval":"","legendFormat":"{{ image }}","range":true,"refId":"A"}],"title":"Container Image Used","type":"timeseries"},{"collapsed":false,"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":1,"w":24,"x":0,"y":72},"id":42,"title":"Kubernetes Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":73},"id":65,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"max(kubelet_volume_stats_used_bytes{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,namespace,cluster) / max(kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,namespace,cluster)","interval":"","legendFormat":"{{ namespace }} - {{ persistentvolumeclaim }}","refId":"A"}],"title":"Persistent Volumes - Capacity and usage in %","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":73},"id":66,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"max(kubelet_volume_stats_used_bytes{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,namespace,cluster)","interval":"","legendFormat":"{{ namespace }} - {{ persistentvolumeclaim }} - Used","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"max(kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,namespace,cluster)","hide":false,"interval":"","legendFormat":"{{ namespace }} - {{ persistentvolumeclaim }} - Capacity","refId":"B"}],"title":"Persistent Volumes - Capacity and usage in bytes","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":81},"id":27,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"1 - (sum(kubelet_volume_stats_inodes_used{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,cluster) / sum(kubelet_volume_stats_inodes{namespace=~\"$namespace\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,cluster))","interval":"","legendFormat":"{{ persistentvolumeclaim }}","refId":"A"}],"title":"Persistent Volumes - Inodes","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":89},"id":76,"title":"Network","type":"row"},{"datasource":{"default":false,"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"binBps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":90},"id":78,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(pod,cluster)","interval":"$resolution","legendFormat":"Received - {{ pod }}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"0 - sum(rate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(pod,cluster)","interval":"$resolution","legendFormat":"Transmitted - {{ pod }}","range":true,"refId":"B"}],"title":"Network - Bandwidth by pod","type":"timeseries"},{"datasource":{"default":false,"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":90},"id":79,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(pod,cluster)","interval":"$resolution","legendFormat":"Received - {{ pod }}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"0 - sum(rate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(pod,cluster)","interval":"$resolution","legendFormat":"Transmitted - {{ pod }}","range":true,"refId":"B"}],"title":"Network - Packets Rate by pod","type":"timeseries"},{"datasource":{"default":false,"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":98},"id":80,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(pod,cluster)","interval":"$resolution","legendFormat":"Received - {{ pod }}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"0 - sum(rate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(pod,cluster)","interval":"$resolution","legendFormat":"Transmitted - {{ pod }}","range":true,"refId":"B"}],"title":"Network - Packets Dropped by pod","type":"timeseries"},{"datasource":{"default":false,"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":98},"id":81,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(pod,cluster)","interval":"$resolution","legendFormat":"Received - {{ pod }}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"0 - sum(rate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"${created_by}.*\",cluster=~\"$cluster\"}[$__rate_interval])) by(pod,cluster)","interval":"$resolution","legendFormat":"Transmitted - {{ pod }}","range":true,"refId":"B"}],"title":"Network - Errors by pod","type":"timeseries"}],"refresh":"30s","schemaVersion":39,"tags":["Kubernetes","Prometheus","vm-k8s-stack"],"templating":{"list":[{"current":{},"hide":0,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":".*","hide":2,"name":"cluster","options":[],"query":".*","refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"constant"},{"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(kube_pod_info{cluster=~\"$cluster\"}, namespace)","hide":0,"includeAll":true,"multi":true,"name":"namespace","options":[],"query":{"query":"label_values(kube_pod_info{cluster=~\"$cluster\"}, namespace)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"30s","value":"30s"},"hide":0,"name":"resolution","options":[{"selected":false,"text":"1s","value":"1s"},{"selected":false,"text":"15s","value":"15s"},{"selected":true,"text":"30s","value":"30s"},{"selected":false,"text":"1m","value":"1m"},{"selected":false,"text":"3m","value":"3m"},{"selected":false,"text":"5m","value":"5m"}],"query":"1s, 15s, 30s, 1m, 3m, 5m","queryValue":"","skipUrlSync":false,"type":"custom"},{"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(kube_pod_info{namespace=~\"$namespace\",cluster=~\"$cluster\"}, created_by_name)","description":"Can be used to filter on a specific deployment, statefulset or deamonset (only relevant panels).","hide":0,"includeAll":true,"multi":true,"name":"created_by","options":[],"query":{"query":"label_values(kube_pod_info{namespace=~\"$namespace\",cluster=~\"$cluster\"}, created_by_name)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{},"timezone":"utc","title":"Kubernetes / Views / Namespaces","uid":"k8s_views_ns","version":45,"weekStart":""}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-kubernetes-views-nodes
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  kubernetes-views-nodes.json: | 
    {"__elements":{},"__inputs":[{"description":"","label":"Prometheus","name":"DS_PROMETHEUS","pluginId":"prometheus","pluginName":"Prometheus","type":"datasource"}],"__requires":[{"id":"gauge","name":"Gauge","type":"panel","version":""},{"id":"grafana","name":"Grafana","type":"grafana","version":"10.3.1"},{"id":"prometheus","name":"Prometheus","type":"datasource","version":"1.0.0"},{"id":"stat","name":"Stat","type":"panel","version":""},{"id":"table","name":"Table","type":"panel","version":""},{"id":"timeseries","name":"Time series","type":"panel","version":""}],"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"#5c4ee5","name":"terraform","target":{"limit":100,"matchAny":false,"tags":["terraform"],"type":"tags"}},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"red","name":"oncall","target":{"limit":100,"matchAny":false,"tags":["oncall"],"type":"tags"}}]},"description":"This is a modern 'Nodes View' dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes","editable":false,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"prometheus","uid":"${datasource}"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":40,"title":"Overview","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null},{"color":"orange","value":50},{"color":"red","value":70}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":4,"x":0,"y":1},"id":7,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto","text":{}},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"avg(sum(rate(node_cpu_seconds_total{mode!~\"idle|iowait|steal\",instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cpu,cluster)) by(cluster)","instant":true,"interval":"$resolution","legendFormat":"","refId":"A"}],"title":"CPU  Usage","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null},{"color":"orange","value":50},{"color":"red","value":70}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":4,"x":4,"y":1},"id":13,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto","text":{}},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(node_memory_MemTotal_bytes{instance=\"$instance\",cluster=~\"$cluster\"} - node_memory_MemAvailable_bytes{instance=\"$instance\",cluster=~\"$cluster\"}) by(cluster) / sum(node_memory_MemTotal_bytes{instance=\"$instance\",cluster=~\"$cluster\"}) by(cluster)","instant":true,"interval":"$resolution","legendFormat":"","refId":"A"}],"title":"RAM Usage","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":4,"x":8,"y":1},"id":24,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"value","wideLayout":true},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(kube_pod_info{node=\"$node\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"","refId":"A"}],"title":"Pods on node","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"inspect":false},"links":[{"targetBlank":true,"title":"Pod details","url":"/d/k8s_views_pods/kubernetes-views-pods?${datasource:queryparam}&var-namespace=${__data.fields.namespace}&${cluster:queryparam}&var-pod=${__data.fields.pod}&${resolution:queryparam}&${__url_time_range}"}],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"pod"},"properties":[{"id":"custom.width","value":416}]},{"matcher":{"id":"byName","options":"priority_class"},"properties":[{"id":"custom.width","value":176}]},{"matcher":{"id":"byName","options":"pod_ip"},"properties":[{"id":"custom.width","value":157}]},{"matcher":{"id":"byName","options":"created_by_kind"},"properties":[{"id":"custom.width","value":205}]},{"matcher":{"id":"byName","options":"namespace"},"properties":[{"id":"custom.width","value":263}]}]},"gridPos":{"h":11,"w":12,"x":12,"y":1},"id":5,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true,"sortBy":[]},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"kube_pod_info{node=\"$node\",cluster=~\"$cluster\"}","format":"table","interval":"","legendFormat":"","refId":"A"}],"title":"List of pods on node ($node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"container":true,"created_by_kind":false,"created_by_name":true,"endpoint":true,"env":true,"host_ip":true,"host_network":true,"instance":true,"job":true,"node":true,"project":true,"prometheus_replica":true,"service":true,"uid":true},"indexByName":{"Time":6,"Value":20,"__name__":7,"container":8,"created_by_kind":2,"created_by_name":9,"endpoint":10,"env":11,"host_ip":5,"host_network":12,"instance":13,"job":14,"namespace":1,"node":15,"pod":0,"pod_ip":3,"priority_class":4,"project":16,"prometheus_replica":17,"service":18,"uid":19},"renameByName":{}}},{"id":"groupBy","options":{"fields":{"created_by_kind":{"aggregations":[],"operation":"groupby"},"host_ip":{"aggregations":[],"operation":"groupby"},"namespace":{"aggregations":["last"],"operation":"groupby"},"pod":{"aggregations":[],"operation":"groupby"},"pod_ip":{"aggregations":[],"operation":"groupby"},"priority_class":{"aggregations":[],"operation":"groupby"}}}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"decimals":3,"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"none"},"overrides":[]},"gridPos":{"h":3,"w":2,"x":0,"y":9},"id":9,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(rate(node_cpu_seconds_total{mode!~\"idle|iowait|steal\",instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","instant":true,"interval":"$resolution","legendFormat":"","refId":"A"}],"title":"CPU Used","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"none"},"overrides":[]},"gridPos":{"h":3,"w":2,"x":2,"y":9},"id":11,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(machine_cpu_cores{node=\"$node\",cluster=~\"$cluster\"}) by(cluster)","interval":"$resolution","legendFormat":"","refId":"A"}],"title":"CPU Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":2,"x":4,"y":9},"id":15,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(node_memory_MemTotal_bytes{instance=\"$instance\",cluster=~\"$cluster\"} - node_memory_MemAvailable_bytes{instance=\"$instance\",cluster=~\"$cluster\"}) by(cluster)","instant":true,"interval":"","legendFormat":"","refId":"A"}],"title":"RAM Used","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":2,"x":6,"y":9},"id":17,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(machine_memory_bytes{node=\"$node\",cluster=~\"$cluster\"}) by(cluster)","instant":false,"interval":"","legendFormat":"","refId":"A"}],"title":"RAM Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"#EAB839","value":25228800},{"color":"red","value":31536000}]},"unit":"s"},"overrides":[]},"gridPos":{"h":3,"w":4,"x":8,"y":9},"id":18,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(node_time_seconds{instance=\"$instance\",cluster=~\"$cluster\"} - node_boot_time_seconds{instance=\"$instance\",cluster=~\"$cluster\"}) by(cluster)","instant":false,"interval":"","legendFormat":"","refId":"A"}],"title":"uptime","type":"stat"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"${datasource}"},"gridPos":{"h":1,"w":24,"x":0,"y":12},"id":38,"title":"Resources","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"max":100,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":0,"y":13},"id":2,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"avg(rate(node_cpu_seconds_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) * 100) by(mode,cluster)","hide":false,"instant":false,"interval":"$resolution","legendFormat":"{{ mode }}","refId":"A"}],"title":"CPU Usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":12,"y":13},"id":3,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"(node_memory_MemTotal_bytes{instance=\"$instance\",cluster=~\"$cluster\"} - node_memory_MemFree_bytes{instance=\"$instance\",cluster=~\"$cluster\"}) - (node_memory_Cached_bytes{instance=\"$instance\",cluster=~\"$cluster\"} + node_memory_Buffers_bytes{instance=\"$instance\",cluster=~\"$cluster\"})","instant":false,"interval":"$resolution","legendFormat":"RAM Used","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"node_memory_MemTotal_bytes{instance=\"$instance\",cluster=~\"$cluster\"}","hide":false,"interval":"$resolution","legendFormat":"RAM Total","refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"node_memory_Cached_bytes{instance=\"$instance\",cluster=~\"$cluster\"}","hide":false,"interval":"$resolution","legendFormat":"RAM Cache","refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"node_memory_Buffers_bytes{instance=\"$instance\",cluster=~\"$cluster\"}","hide":false,"interval":"$resolution","legendFormat":"RAM Buffer","refId":"D"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"node_memory_MemFree_bytes{instance=\"$instance\",cluster=~\"$cluster\"}","hide":false,"interval":"$resolution","legendFormat":"RAM Free","refId":"E"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"node_memory_SwapTotal_bytes{instance=\"$instance\",cluster=~\"$cluster\"} - node_memory_SwapFree_bytes{instance=\"$instance\",cluster=~\"$cluster\"}","hide":false,"interval":"$resolution","legendFormat":"SWAP Used","refId":"F"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"node_memory_SwapTotal_bytes{instance=\"$instance\",cluster=~\"$cluster\"}","hide":false,"interval":"$resolution","legendFormat":"SWAP Total","refId":"G"}],"title":"Memory Usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"CPU Cores","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":22},"id":26,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{node=\"$node\",image!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(pod,cluster)","interval":"$resolution","legendFormat":"{{ pod }}","refId":"A"}],"title":"CPU usage by Pod","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":22},"id":28,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(container_memory_working_set_bytes{node=\"$node\",image!=\"\",cluster=~\"$cluster\"}) by(pod,cluster)","interval":"$resolution","legendFormat":"{{ pod }}","refId":"A"}],"title":"Memory usage by Pod","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"Number of times a CPU core has been throttled on an instance.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"CPU CORES","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":30},"id":66,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(node_cpu_core_throttles_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Nb of cpu core throttles","range":true,"refId":"A"}],"title":"Number of CPU Core Throttled","type":"timeseries"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"${datasource}"},"gridPos":{"h":1,"w":24,"x":0,"y":38},"id":44,"title":"System","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":39},"id":48,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"node_load1{instance=\"$instance\",cluster=~\"$cluster\"}","interval":"$resolution","legendFormat":"1m","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"node_load5{instance=\"$instance\",cluster=~\"$cluster\"}","hide":false,"interval":"$resolution","legendFormat":"5m","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"node_load15{instance=\"$instance\",cluster=~\"$cluster\"}","hide":false,"interval":"$resolution","legendFormat":"15m","range":true,"refId":"C"}],"title":"System Load","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":39},"id":46,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"rate(node_context_switches_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"$resolution","intervalFactor":1,"legendFormat":"Context switches","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"rate(node_intr_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","hide":false,"interval":"$resolution","legendFormat":"Interrupts","range":true,"refId":"B"}],"title":"Context Switches & Interrupts","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":47},"id":49,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"node_filefd_maximum{instance=\"$instance\",cluster=~\"$cluster\"}","instant":false,"interval":"$resolution","legendFormat":"Maximum file descriptors","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"node_filefd_allocated{instance=\"$instance\",cluster=~\"$cluster\"}","hide":false,"instant":false,"interval":"$resolution","legendFormat":"Allocated file descriptors","refId":"B"}],"title":"File Descriptors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":47},"id":50,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"node_timex_estimated_error_seconds{instance=\"$instance\",cluster=~\"$cluster\"}","instant":false,"interval":"$resolution","intervalFactor":1,"legendFormat":"Estimated error in seconds","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"node_timex_maxerror_seconds{instance=\"$instance\",cluster=~\"$cluster\"}","hide":false,"interval":"$resolution","intervalFactor":1,"legendFormat":"Maximum error in seconds","range":true,"refId":"B"}],"title":"Time Sync","type":"timeseries"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"${datasource}"},"gridPos":{"h":1,"w":24,"x":0,"y":55},"id":36,"title":"Network","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"BANDWIDTH","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"binBps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"id":20,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(node_network_receive_bytes_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"In","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"0 - sum(rate(node_network_transmit_bytes_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Out","range":true,"refId":"B"}],"title":"Network usage (bytes/s)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"id":61,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(node_network_receive_errs_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"In","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"0 - sum(rate(node_network_transmit_errs_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Out","range":true,"refId":"B"}],"title":"Network errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"id":62,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(node_network_receive_packets_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"In","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"0 - sum(rate(node_network_transmit_packets_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Out","range":true,"refId":"B"}],"title":"Network usage (packet/s)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"id":64,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(node_network_receive_drop_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","hide":false,"interval":"$resolution","legendFormat":"In","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"0 - sum(rate(node_network_transmit_drop_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","hide":false,"interval":"$resolution","legendFormat":"Out","refId":"B"}],"title":"Network total drops","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"id":60,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"node_netstat_Tcp_CurrEstab{instance=\"$instance\",cluster=~\"$cluster\"}","instant":false,"interval":"$resolution","legendFormat":"TCP Currently Established","refId":"A"}],"title":"TCP Currently Established","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[{"matcher":{"id":"byName","options":"NF Conntrack limit"},"properties":[{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"id":63,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"node_nf_conntrack_entries{instance=\"$instance\",cluster=~\"$cluster\"}","instant":false,"interval":"$resolution","legendFormat":"NF Conntrack entries","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"node_nf_conntrack_entries_limit{instance=\"$instance\",cluster=~\"$cluster\"}","hide":false,"interval":"$resolution","legendFormat":"NF Conntrack limit","range":true,"refId":"B"}],"title":"NF Conntrack","type":"timeseries"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"${datasource}"},"gridPos":{"h":1,"w":24,"x":0,"y":80},"id":54,"title":"Kubernetes Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":81},"id":30,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"max(kubelet_volume_stats_used_bytes{node=\"$node\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,namespace,cluster) / max(kubelet_volume_stats_capacity_bytes{node=\"$node\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,namespace,cluster)","interval":"$resolution","legendFormat":"{{ namespace }} - {{ persistentvolumeclaim }}","range":true,"refId":"A"}],"title":"Persistent Volumes - Usage in %","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"continuous-GrYlRd"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"filterable":false,"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Used"},"properties":[{"id":"custom.width","value":146}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"custom.width","value":167}]}]},"gridPos":{"h":16,"w":12,"x":12,"y":81},"id":34,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true,"sortBy":[]},"pluginVersion":"11.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"max(kubelet_volume_stats_used_bytes{node=\"$node\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,namespace,cluster)","format":"table","hide":false,"interval":"","legendFormat":"","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"max(kubelet_volume_stats_capacity_bytes{node=\"$node\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,namespace,cluster)","format":"table","hide":false,"interval":"","legendFormat":"","refId":"B"}],"title":"Persistent Volumes - Usage in GB","transformations":[{"id":"groupBy","options":{"fields":{"Value":{"aggregations":["lastNotNull"],"operation":"aggregate"},"Value #A":{"aggregations":["lastNotNull"],"operation":"aggregate"},"Value #B":{"aggregations":["lastNotNull"],"operation":"aggregate"},"persistentvolumeclaim":{"aggregations":[],"operation":"groupby"}}}},{"id":"seriesToColumns","options":{"byField":"persistentvolumeclaim"}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Value #A (lastNotNull)":"Used","Value #B (lastNotNull)":"Total","persistentvolumeclaim":"Persistent Volume Claim"}}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":89},"id":32,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"(sum(kubelet_volume_stats_inodes_used{node=\"$node\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,cluster) / sum(kubelet_volume_stats_inodes{node=\"$node\",cluster=~\"$cluster\"}) by(persistentvolumeclaim,cluster)) * 100","interval":"$resolution","legendFormat":"{{ persistentvolumeclaim }}","range":true,"refId":"A"}],"title":"Persistent Volumes - Inodes","type":"timeseries"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"${datasource}"},"gridPos":{"h":1,"w":24,"x":0,"y":97},"id":42,"title":"Node Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":98},"id":33,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"100 - ((node_filesystem_avail_bytes{instance=\"$instance\",cluster=~\"$cluster\"} * 100) / node_filesystem_size_bytes{instance=\"$instance\",cluster=~\"$cluster\"})","hide":false,"interval":"$resolution","legendFormat":"{{ mountpoint }}","range":true,"refId":"A"}],"title":"FS usage in %","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":98},"id":59,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"100 - ((node_filesystem_files_free{instance=\"$instance\",cluster=~\"$cluster\"} / node_filesystem_files{instance=\"$instance\",cluster=~\"$cluster\"}) * 100)","hide":false,"interval":"$resolution","legendFormat":"{{ mountpoint }}","range":true,"refId":"A"}],"title":"FS inode usage in %","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":106},"id":52,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"rate(node_disk_read_bytes_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"$resolution","legendFormat":"{{device}}","range":true,"refId":"A"}],"title":"Reads by disk (bytes)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":106},"id":57,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"rate(node_disk_written_bytes_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","hide":false,"interval":"$resolution","legendFormat":"{{device}}","range":true,"refId":"A"}],"title":"Writes by disk (bytes)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"read/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":114},"id":51,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"rate(node_disk_reads_completed_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"$resolution","legendFormat":"{{device}}","range":true,"refId":"A"}],"title":"Completed reads by disk","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"write/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":114},"id":56,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"rate(node_disk_writes_completed_total{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","hide":false,"interval":"$resolution","legendFormat":"{{device}}","range":true,"refId":"A"}],"title":"Completed writes by disk","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"io/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":122},"id":58,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"rate(node_disk_io_now{instance=\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"$resolution","legendFormat":"{{device}}","range":true,"refId":"A"}],"title":"Disk(s) io/s","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":122},"id":55,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"right","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(node_filesystem_device_error{instance=\"$instance\",cluster=~\"$cluster\"}) by(mountpoint,cluster)","interval":"$resolution","legendFormat":"{{ mountpoint }}","range":true,"refId":"A"}],"title":"FS - Device Errors","type":"timeseries"}],"refresh":"30s","schemaVersion":39,"tags":["Kubernetes","Prometheus","vm-k8s-stack"],"templating":{"list":[{"current":{},"hide":0,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":".*","hide":2,"name":"cluster","options":[],"query":".*","refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"constant"},{"current":{"selected":false,"text":"30s","value":"30s"},"hide":0,"name":"resolution","options":[{"selected":false,"text":"1s","value":"1s"},{"selected":false,"text":"15s","value":"15s"},{"selected":true,"text":"30s","value":"30s"},{"selected":false,"text":"1m","value":"1m"},{"selected":false,"text":"3m","value":"3m"},{"selected":false,"text":"5m","value":"5m"}],"query":"1s, 15s, 30s, 1m, 3m, 5m","queryValue":"","skipUrlSync":false,"type":"custom"},{"current":{},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(kube_node_info{cluster=~\"$cluster\"}, node)","hide":0,"name":"node","options":[],"query":{"query":"label_values(kube_node_info{cluster=~\"$cluster\"}, node)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":1,"type":"query"},{"current":{},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(node_uname_info{nodename=~\"(?i:($node)(\\\\.[a-z0-9.-]+)?)\",cluster=~\"$cluster\"}, instance)","hide":2,"name":"instance","options":[],"query":{"query":"label_values(node_uname_info{nodename=~\"(?i:($node)(\\\\.[a-z0-9.-]+)?)\",cluster=~\"$cluster\"}, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{},"timezone":"utc","title":"Kubernetes / Views / Nodes","uid":"k8s_views_nodes","version":40,"weekStart":""}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-kubernetes-views-pods
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  kubernetes-views-pods.json: | 
    {"__elements":[],"__inputs":[{"description":"","label":"Prometheus","name":"DS_PROMETHEUS","pluginId":"prometheus","pluginName":"Prometheus","type":"datasource"}],"__requires":[{"id":"grafana","name":"Grafana","type":"grafana","version":"8.3.4"},{"id":"prometheus","name":"Prometheus","type":"datasource","version":"5.0.0"},{"id":"timeseries","name":"Time series","type":"panel","version":""},{"id":"stat","name":"Stat","type":"panel","version":""},{"id":"gauge","name":"Gauge","type":"panel","version":""},{"id":"table","name":"Table","type":"panel","version":""}],"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"#5c4ee5","name":"terraform","target":{"limit":100,"matchAny":false,"tags":["terraform"],"type":"tags"}},{"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":false,"iconColor":"red","name":"oncall","target":{"limit":100,"matchAny":false,"tags":["oncall"],"type":"tags"}}]},"description":"This is a modern 'Pods View' dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes","editable":false,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":43,"title":"Information","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"Panel only works when a single pod is selected.","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":0}]},"unit":"none"},"overrides":[]},"gridPos":{"h":2,"w":12,"x":0,"y":1},"id":2,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showPercentChange":false,"textMode":"name","wideLayout":true},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"kube_pod_info{namespace=\"$namespace\",pod=\"$pod\",cluster=~\"$cluster\"}","instant":true,"interval":"","legendFormat":"{{ created_by_kind }}: {{ created_by_name }}","refId":"A"}],"title":"Created by","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"Panel only works when a single pod is selected.","fieldConfig":{"defaults":{"links":[{"title":"","url":"/d/k8s_views_nodes/kubernetes-views-nodes?var-datasource=${datasource}&var-node=${__field.labels.node}&${cluster:queryparam}"}],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":0}]},"unit":"none"},"overrides":[]},"gridPos":{"h":2,"w":6,"x":12,"y":1},"id":33,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showPercentChange":false,"textMode":"name","wideLayout":true},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"kube_pod_info{namespace=\"$namespace\",pod=\"$pod\",cluster=~\"$cluster\"}","instant":true,"interval":"","legendFormat":"{{ node }}","refId":"A"}],"title":"Running on","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"Panel only works when a single pod is selected.","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":0}]},"unit":"none"},"overrides":[]},"gridPos":{"h":2,"w":6,"x":18,"y":1},"id":41,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showPercentChange":false,"textMode":"name","wideLayout":true},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"kube_pod_info{namespace=\"$namespace\",pod=\"$pod\",cluster=~\"$cluster\"}","instant":true,"interval":"","legendFormat":"{{ pod_ip }}","refId":"A"}],"title":"Pod IP","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"Panel only works when a single pod is selected.","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":0}]},"unit":"none"},"overrides":[]},"gridPos":{"h":2,"w":5,"x":0,"y":3},"id":52,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showPercentChange":false,"textMode":"name","wideLayout":true},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"kube_pod_info{namespace=\"$namespace\",pod=\"$pod\",priority_class!=\"\",cluster=~\"$cluster\"}","format":"time_series","instant":true,"interval":"","legendFormat":"{{ priority_class }}","range":false,"refId":"A"}],"title":"Priority Class","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"Panel only works when a single pod is selected.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"none"},"overrides":[{"matcher":{"id":"byName","options":"Burstable"},"properties":[{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"BestEffort"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":2,"w":7,"x":5,"y":3},"id":53,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":[],"fields":"","values":false},"showPercentChange":false,"textMode":"name","wideLayout":true},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"kube_pod_status_qos_class{namespace=\"$namespace\",pod=\"$pod\",cluster=~\"$cluster\"} > 0","instant":true,"interval":"","legendFormat":"{{ qos_class }}","refId":"A"}],"title":"QOS Class","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"Panel only works when a single pod is selected.","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"text","value":0},{"color":"red","value":1}]},"unit":"none"},"overrides":[]},"gridPos":{"h":2,"w":6,"x":12,"y":3},"id":56,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":[],"fields":"","values":false},"showPercentChange":false,"textMode":"name","wideLayout":true},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"kube_pod_container_status_last_terminated_reason{namespace=\"$namespace\",pod=\"$pod\",cluster=~\"$cluster\"}","instant":true,"interval":"","legendFormat":"{{ reason }}","refId":"A"}],"title":"Last Terminated Reason","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"Panel only works when a single pod is selected.","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"text","value":0},{"color":"red","value":1},{"color":"#EAB839","value":2}]},"unit":"none"},"overrides":[]},"gridPos":{"h":2,"w":6,"x":18,"y":3},"id":57,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":[],"fields":"","values":true},"showPercentChange":false,"textMode":"value","wideLayout":true},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"kube_pod_container_status_last_terminated_exitcode{namespace=\"$namespace\",pod=\"$pod\",cluster=~\"$cluster\"}","instant":true,"interval":"","legendFormat":"__auto","range":false,"refId":"A"}],"title":"Last Terminated Exit Code","type":"stat"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":5},"id":47,"title":"Resources","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"fixedColor":"blue","mode":"fixed"},"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":0},{"color":"#EAB839","value":60},{"color":"red","value":75}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":3,"x":0,"y":6},"id":39,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\",pod=~\"$pod\",resource=\"cpu\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","instant":true,"interval":"$resolution","legendFormat":"Requests","refId":"A"}],"title":"Total pod CPU Requests usage","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":0},{"color":"#EAB839","value":60},{"color":"red","value":75}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":3,"x":3,"y":6},"id":48,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\",pod=~\"$pod\",resource=\"cpu\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","instant":true,"interval":"$resolution","legendFormat":"Limits","refId":"A"}],"title":"Total pod CPU Limits usage","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"fixedColor":"blue","mode":"fixed"},"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"blue","value":0},{"color":"#EAB839","value":80},{"color":"red","value":99}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":3,"x":6,"y":6},"id":40,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",cluster=~\"$cluster\"}) by(cluster) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\",pod=~\"$pod\",resource=\"memory\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","instant":true,"interval":"$resolution","legendFormat":"Requests","refId":"A"}],"title":"Total pod RAM Requests usage","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":0},{"color":"#EAB839","value":60},{"color":"red","value":75}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":3,"x":9,"y":6},"id":49,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",cluster=~\"$cluster\"}) by(cluster) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\",pod=~\"$pod\",resource=\"memory\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(cluster)","instant":true,"interval":"$resolution","legendFormat":"Limits","refId":"B"}],"title":"Total pod RAM Limits usage","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"custom":{"align":"auto","cellOptions":{"type":"auto"},"filterable":false,"footer":{"reducers":[]},"inspect":false,"minWidth":100},"decimals":4,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"blue","value":0}]},"unit":"none"},"overrides":[{"matcher":{"id":"byName","options":"Memory Requests"},"properties":[{"id":"unit","value":"bytes"},{"id":"decimals","value":2}]},{"matcher":{"id":"byName","options":"Memory Limits"},"properties":[{"id":"unit","value":"bytes"},{"id":"decimals","value":2}]},{"matcher":{"id":"byName","options":"Memory Used"},"properties":[{"id":"unit","value":"bytes"},{"id":"decimals","value":2}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":6},"id":38,"options":{"cellHeight":"sm","showHeader":true,"sortBy":[]},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_requests{namespace=\"$namespace\",pod=~\"$pod\",resource=\"cpu\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(container,cluster)","format":"table","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_limits{namespace=\"$namespace\",pod=~\"$pod\",resource=\"cpu\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(container,cluster)","format":"table","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_requests{namespace=\"$namespace\",pod=~\"$pod\",resource=\"memory\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(container,cluster)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_limits{namespace=\"$namespace\",pod=~\"$pod\",resource=\"memory\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(container,cluster)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"D"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",container!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(container,cluster)","format":"table","hide":false,"instant":true,"legendFormat":"__auto","range":false,"refId":"E"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",container!=\"\",cluster=~\"$cluster\"}) by(container,cluster)","format":"table","hide":false,"instant":true,"range":false,"refId":"F"}],"title":"Resources by container","transformations":[{"id":"seriesToColumns","options":{"byField":"container"}},{"id":"organize","options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 4":true,"__name__":true,"__name__ 1":true,"__name__ 2":true,"__name__ 3":true,"__name__ 4":true,"container":false,"endpoint":true,"endpoint 2":true,"endpoint 3":true,"endpoint 4":true,"instance":true,"instance 2":true,"instance 3":true,"instance 4":true,"job":true,"job 2":true,"job 3":true,"job 4":true,"namespace":true,"namespace 2":true,"namespace 3":true,"namespace 4":true,"node":true,"node 2":true,"node 3":true,"node 4":true,"pod":true,"pod 2":true,"pod 3":true,"pod 4":true,"resource 1":true,"resource 2":true,"resource 3":true,"resource 4":true,"service":true,"service 2":true,"service 3":true,"service 4":true,"uid 1":true,"uid 2":true,"uid 3":true,"uid 4":true,"unit 1":true,"unit 2":true,"unit 3":true,"unit 4":true},"indexByName":{"Time 1":7,"Time 2":8,"Time 3":9,"Time 4":10,"Time 5":11,"Time 6":12,"Value #A":2,"Value #B":3,"Value #C":5,"Value #D":6,"Value #E":1,"Value #F":4,"container":0},"renameByName":{"Value #A":"CPU Requests","Value #B":"CPU Limits","Value #C":"Memory Requests","Value #D":"Memory Limits","Value #E":"CPU Used","Value #F":"Memory Used","container":"Container"}}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Percent","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"area"}},"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"red","value":0},{"color":"yellow","value":20},{"color":"green","value":30},{"color":"yellow","value":70},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":14},"id":50,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(container,cluster) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\",pod=~\"$pod\",resource=\"cpu\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(container,cluster)","interval":"$resolution","legendFormat":"{{ container }}  REQUESTS","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(container,cluster) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\",pod=~\"$pod\",resource=\"cpu\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(container,cluster)","hide":false,"legendFormat":"{{ container }}  LIMITS","range":true,"refId":"B"}],"title":"CPU Usage / Requests & Limits by container","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"fixedColor":"blue","mode":"thresholds"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Percent","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"area"}},"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"red","value":0},{"color":"yellow","value":20},{"color":"green","value":30},{"color":"#EAB839","value":70},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":14},"id":30,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",cluster=~\"$cluster\"}) by(container,cluster) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\",pod=~\"$pod\",resource=\"memory\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(container,cluster)","interval":"","legendFormat":"{{ container }} REQUESTS","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",cluster=~\"$cluster\"}) by(container,cluster) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\",pod=~\"$pod\",resource=\"memory\",job=~\"$job\",cluster=~\"$cluster\"} and on(namespace,pod,cluster) max(kube_pod_status_phase{phase=\"Running\",namespace=\"$namespace\",pod=~\"$pod\",job=~\"$job\",cluster=~\"$cluster\"} == 1) by(namespace,pod,cluster)) by(container,cluster)","hide":false,"legendFormat":"{{ container }} LIMITS","range":true,"refId":"B"}],"title":"Memory Usage / Requests & Limits by container","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"CPU Cores","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":4,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"none"},"overrides":[{"matcher":{"id":"byName","options":"limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":22},"id":29,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",container!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(container,id,cluster)","interval":"$resolution","legendFormat":"{{ container }}","range":true,"refId":"A"}],"title":"CPU Usage by container","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Bytes","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":22},"id":51,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",container!=\"\",cluster=~\"$cluster\"}) by(container,id,cluster)","interval":"","legendFormat":"{{ container }}","range":true,"refId":"A"}],"title":"Memory Usage by container","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"SECONDS","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":30},"id":59,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=\"$namespace\",pod=~\"$pod\",image!=\"\",container!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(container,cluster)","interval":"$resolution","legendFormat":"{{ container }}","range":true,"refId":"A"}],"title":"CPU Throttled seconds by container","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":38},"id":62,"title":"Kubernetes","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"fixedColor":"blue","mode":"thresholds"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Percent","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"red","value":0}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":39},"id":60,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(increase(container_oom_events_total{namespace=\"${namespace}\",pod=~\"${pod}\",container!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(container,cluster) > 0","interval":"","legendFormat":"{{ container }}","range":true,"refId":"A"}],"title":"OOM Events by container","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"fixedColor":"blue","mode":"thresholds"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Percent","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"red","value":0}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":39},"id":61,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(increase(kube_pod_container_status_restarts_total{namespace=\"${namespace}\",pod=~\"${pod}\",container!=\"\",job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(container,cluster) > 0","interval":"","legendFormat":"{{ container }}","range":true,"refId":"A"}],"title":"Container Restarts by container","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"filterable":true,"footer":{"reducers":[]},"inspect":false,"wrapText":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"#EAB839","value":1},{"color":"red","value":5}]}},"overrides":[{"matcher":{"id":"byName","options":"reason"},"properties":[{"id":"custom.cellOptions","value":{"type":"color-background"}},{"id":"mappings","value":[{"options":{"CrashLoopBackOff":{"color":"light-red","index":0,"text":"CrashLoopBackOff"},"ErrImagePull":{"color":"light-orange","index":1,"text":"ErrImagePull"},"ImagePullBackOff":{"color":"light-orange","index":2,"text":"ImagePullBackOff"}},"type":"value"}]}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":47},"id":63,"options":{"cellHeight":"sm","showHeader":true},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"kube_pod_container_status_waiting_reason{reason=~\"ErrImagePull|ImagePullBackOff|CrashLoopBackOff\",cluster=~\"$cluster\"} == 1","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],"title":"Pods with Container Issues","transformations":[{"id":"labelsToFields","options":{"keepLabels":["namespace","pod","reason","container"],"mode":"columns"}},{"id":"filterFieldsByName","options":{"include":{"names":["namespace","pod","reason","container"]}}},{"id":"organize","options":{"excludeByName":{},"includeByName":{},"indexByName":{},"renameByName":{"container":"Container","namespace":"Namespace","pod":"Pod Name","reason":"Reason"}}},{"id":"sortBy","options":{"fields":{},"sort":[{"field":"Pod Name"}]}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"filterable":true,"footer":{"reducers":[]},"inspect":false,"wrapText":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"#EAB839","value":1},{"color":"red","value":5}]}},"overrides":[{"matcher":{"id":"byName","options":"condition"},"properties":[{"id":"custom.cellOptions","value":{"type":"color-background"}},{"id":"mappings","value":[{"options":{"false":{"color":"light-red","index":0,"text":"False"}},"type":"value"}]},{"id":"custom.width","value":104}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":47},"id":64,"options":{"cellHeight":"sm","showHeader":true,"sortBy":[]},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"kube_pod_status_scheduled{condition=\"false\",cluster=~\"$cluster\"} == 1","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],"title":"Unscheduled Pods","transformations":[{"id":"labelsToFields","options":{"keepLabels":["namespace","pod","reason","container"],"mode":"columns"}},{"id":"filterFieldsByName","options":{"include":{"names":["container","namespace","pod","condition"]}}},{"id":"organize","options":{"excludeByName":{},"includeByName":{},"indexByName":{"condition":3,"container":0,"namespace":1,"pod":2},"renameByName":{"condition":"Condition","container":"Container","namespace":"Namespace","pod":"Pod Name"}}}],"type":"table"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":55},"id":45,"title":"Network","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"binBps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"id":31,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_network_receive_bytes_total{namespace=\"$namespace\",pod=~\"$pod\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Received","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"0 - sum(rate(container_network_transmit_bytes_total{namespace=\"$namespace\",pod=~\"$pod\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Transmitted","refId":"B"}],"title":"Network - Bandwidth","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"id":34,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_network_receive_packets_total{namespace=\"$namespace\",pod=~\"$pod\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Received","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"0 - sum(rate(container_network_transmit_packets_total{namespace=\"$namespace\",pod=~\"$pod\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Transmitted","refId":"B"}],"title":"Network - Packets Rate","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"id":36,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_network_receive_packets_dropped_total{namespace=\"$namespace\",pod=~\"$pod\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Received","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"0 - sum(rate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\",pod=~\"$pod\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Transmitted","refId":"B"}],"title":"Network - Packets Dropped","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"id":37,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.1","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_network_receive_errors_total{namespace=\"$namespace\",pod=~\"$pod\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Received","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"0 - sum(rate(container_network_transmit_errors_total{namespace=\"$namespace\",pod=~\"$pod\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"$resolution","legendFormat":"Transmitted","refId":"B"}],"title":"Network - Errors","type":"timeseries"}],"preload":false,"refresh":"30s","schemaVersion":42,"tags":["Kubernetes","Prometheus","vm-k8s-stack"],"templating":{"list":[{"current":{"selected":false,"text":"","value":""},"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"current":{"text":"","value":""},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":".*","hide":2,"name":"cluster","options":[],"query":".*","refresh":1,"regex":"","sort":1,"type":"constant"},{"current":{"selected":false,"text":"monitoring","value":"monitoring"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(kube_pod_info{cluster=~\"$cluster\"}, namespace)","name":"namespace","options":[],"query":{"query":"label_values(kube_pod_info{cluster=~\"$cluster\"}, namespace)","refId":"Prometheus-namespace-Variable-Query"},"refresh":1,"regex":"","sort":1,"type":"query"},{"allValue":".*","current":{"selected":false,"text":"","value":""},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(kube_pod_info{namespace=\"$namespace\",cluster=~\"$cluster\"}, pod)","includeAll":true,"multi":true,"name":"pod","options":[],"query":{"query":"label_values(kube_pod_info{namespace=\"$namespace\",cluster=~\"$cluster\"}, pod)","refId":"Prometheus-pod-Variable-Query"},"refresh":2,"regex":"","sort":1,"type":"query"},{"current":{"text":"30s","value":"30s"},"name":"resolution","options":[{"selected":false,"text":"1s","value":"1s"},{"selected":false,"text":"15s","value":"15s"},{"selected":true,"text":"30s","value":"30s"},{"selected":false,"text":"1m","value":"1m"},{"selected":false,"text":"3m","value":"3m"},{"selected":false,"text":"5m","value":"5m"}],"query":"1s, 15s, 30s, 1m, 3m, 5m","type":"custom"},{"current":{"selected":false,"text":"kube-state-metrics","value":"kube-state-metrics"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(kube_pod_info{namespace=\"$namespace\",cluster=~\"$cluster\"}, job)","multi":true,"name":"job","options":[],"query":{"qryType":1,"query":"label_values(kube_pod_info{namespace=\"$namespace\",cluster=~\"$cluster\"}, job)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{},"timezone":"utc","title":"Kubernetes / Views / Pods","uid":"k8s_views_pods","version":38,"weekStart":""}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-node-exporter-full
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  node-exporter-full.json: | 
    {"__requires":[{"id":"bargauge","name":"Bar gauge","type":"panel","version":""},{"id":"gauge","name":"Gauge","type":"panel","version":""},{"id":"grafana","name":"Grafana","type":"grafana","version":"11.6.1"},{"id":"prometheus","name":"Prometheus","type":"datasource","version":"1.0.0"},{"id":"stat","name":"Stat","type":"panel","version":""},{"id":"timeseries","name":"Time series","type":"panel","version":""}],"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":false,"fiscalYearStartMonth":0,"graphTooltip":1,"id":null,"links":[{"icon":"external link","tags":[],"targetBlank":true,"title":"GitHub","type":"link","url":"https://github.com/rfmoz/grafana-dashboards"},{"icon":"external link","tags":[],"targetBlank":true,"title":"Grafana","type":"link","url":"https://grafana.com/grafana/dashboards/1860"}],"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":261,"title":"Quick CPU / Mem / Disk","type":"row"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Resource pressure via PSI","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"links":[],"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green"},{"color":"dark-yellow","value":70},{"color":"dark-red","value":90}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":0,"y":1},"id":323,"options":{"displayMode":"basic","legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"maxVizHeight":300,"minVizHeight":10,"minVizWidth":0,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","text":{},"valueMode":"color"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"irate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","instant":true,"legendFormat":"CPU","range":false,"refId":"A","step":240},{"editorMode":"code","exemplar":false,"expr":"irate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","instant":true,"legendFormat":"Mem","range":false,"refId":"B","step":240},{"editorMode":"code","exemplar":false,"expr":"irate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","instant":true,"legendFormat":"I/O","range":false,"refId":"C","step":240},{"editorMode":"code","exemplar":false,"expr":"irate(node_pressure_irq_stalled_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","instant":true,"legendFormat":"Irq","range":false,"refId":"D","step":240}],"title":"Pressure","type":"bargauge"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Overall CPU busy percentage (averaged across all cores)","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":3,"y":1},"id":20,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster))","instant":true,"legendFormat":"","range":false,"refId":"A","step":240}],"title":"CPU Busy","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"System load over all CPU cores together","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":85},{"color":"rgba(245, 54, 54, 0.9)","value":95}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":6,"y":1},"id":155,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"(scalar(node_load1{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) * 100) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster)","format":"time_series","instant":true,"range":false,"refId":"A","step":240}],"title":"Sys Load","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Real RAM usage excluding cache and reclaimable memory","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":9,"y":1},"id":16,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"clamp_min((1 - (node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} / node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"})) * 100, 0)","format":"time_series","instant":true,"range":false,"refId":"B","step":240}],"title":"RAM Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Percentage of swap space currently used by the system","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":10},{"color":"rgba(245, 54, 54, 0.9)","value":25}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":12,"y":1},"id":21,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} >bool 0) * ((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) / node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"})) * 100","instant":true,"range":false,"refId":"A","step":240}],"title":"SWAP Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Used Root FS","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":15,"y":1},"id":154,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"((node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\",cluster=~\"$cluster\"} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\",cluster=~\"$cluster\"}) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\",cluster=~\"$cluster\"}) * 100","format":"time_series","instant":true,"range":false,"refId":"A","step":240}],"title":"Root FS Used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":18,"y":1},"id":14,"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster)","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],"title":"CPU Cores","type":"stat"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":20,"y":1},"id":75,"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","instant":true,"range":false,"refId":"A","step":240}],"title":"RAM Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":22,"y":1},"id":18,"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","instant":true,"range":false,"refId":"A","step":240}],"title":"SWAP Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":70},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":2,"w":2,"x":18,"y":3},"id":23,"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\",cluster=~\"$cluster\"}","format":"time_series","instant":true,"range":false,"refId":"A","step":240}],"title":"RootFS Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":2,"w":4,"x":20,"y":3},"id":15,"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"node_time_seconds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","instant":true,"range":false,"refId":"A","step":240}],"title":"Uptime","type":"stat"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":5},"id":263,"title":"Basic CPU / Mem / Net / Disk","type":"row"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"CPU time spent busy vs idle, split by activity type","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"Busy Iowait"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Idle"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy System"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy User"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Busy Other"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":6},"id":77,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true,"width":250},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",mode=\"system\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","instant":false,"legendFormat":"Busy System","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",mode=\"user\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"Busy User","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",mode=\"iowait\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"Busy Iowait","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",mode=~\".*irq\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"Busy IRQs","range":true,"refId":"D","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",mode!=\"idle\",mode!=\"user\",mode!=\"system\",mode!=\"iowait\",mode!=\"irq\",mode!=\"softirq\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"Busy Other","range":true,"refId":"E","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",mode=\"idle\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"Idle","range":true,"refId":"F","step":240}],"title":"CPU Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"RAM and swap usage overview, including caches","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Swap used"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]},{"matcher":{"id":"byName","options":"Cache + Buffer"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]}]},"gridPos":{"h":7,"w":12,"x":12,"y":6},"id":78,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true,"width":350},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Total","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) - ((node_memory_Cached_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"})","format":"time_series","legendFormat":"Used","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"(node_memory_Cached_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Cache + Buffer","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Free","range":true,"refId":"D","step":240},{"editorMode":"code","expr":"node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Swap used","range":true,"refId":"E","step":240}],"title":"Memory Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Per-interface network traffic (receive and transmit) in bits per second","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Tx.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":13},"id":74,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) * 8","format":"time_series","legendFormat":"Rx {{device}}","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) * 8","format":"time_series","legendFormat":"Tx {{device}}","range":true,"refId":"B","step":240}],"title":"Network Traffic Basic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Percentage of filesystem space used for each mounted device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":12,"y":13},"id":152,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"((node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~\"rootfs\",cluster=~\"$cluster\"} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~\"rootfs\",cluster=~\"$cluster\"}) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~\"rootfs\",cluster=~\"$cluster\"}) * 100","format":"time_series","legendFormat":"{{mountpoint}}","range":true,"refId":"A","step":240}],"title":"Disk Space Used Basic","type":"timeseries"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":20},"id":265,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"CPU time usage split by state, normalized across all CPU cores","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":70,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byName","options":"Idle - Waiting for something to happen"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Iowait - Waiting for I/O to complete"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Irq - Servicing interrupts"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Nice - Niced processes executing in user mode"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Softirq - Servicing softirqs"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Steal - Time spent in other operating systems when running in a virtualized environment"},"properties":[{"id":"color","value":{"fixedColor":"#FCE2DE","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"System - Processes executing in kernel mode"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"User - Normal processes executing in user mode"},"properties":[{"id":"color","value":{"fixedColor":"#5195CE","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Guest CPU usage"},"properties":[{"id":"custom.fillOpacity","value":0},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}},{"id":"custom.stacking","value":{"group":"A","mode":"none"}}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":21},"id":3,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":250},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{mode=\"system\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","interval":"","legendFormat":"System - Processes executing in kernel mode","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{mode=\"user\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"User - Normal processes executing in user mode","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{mode=\"nice\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"Nice - Niced processes executing in user mode","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{mode=\"iowait\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"Iowait - Waiting for I/O to complete","range":true,"refId":"D","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{mode=\"irq\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"Irq - Servicing interrupts","range":true,"refId":"E","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{mode=\"softirq\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"Softirq - Servicing softirqs","range":true,"refId":"F","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{mode=\"steal\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"Steal - Time spent in other operating systems when running in a virtualized environment","range":true,"refId":"G","step":240},{"editorMode":"code","expr":"sum(irate(node_cpu_seconds_total{mode=\"idle\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster))","format":"time_series","legendFormat":"Idle - Waiting for something to happen","range":true,"refId":"H","step":240},{"editorMode":"code","expr":"(sum(irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(instance,cluster) / on(instance,cluster) group_left() sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(instance,cluster)) > 0","format":"time_series","legendFormat":"Guest CPU usage","range":true,"refId":"I","step":240}],"title":"CPU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Breakdown of physical memory and swap usage. Hardware-detected memory errors are also displayed","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Apps"},"properties":[{"id":"color","value":{"fixedColor":"#629E51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#614D93","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#0A437C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#584477","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"RAM_Free"},"properties":[{"id":"color","value":{"fixedColor":"#E0F9D7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab"},"properties":[{"id":"color","value":{"fixedColor":"#806EB7","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#E0752D","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap - Swap memory usage"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Free"},"properties":[{"id":"color","value":{"fixedColor":"#2F575E","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Unused - Free memory unassigned"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Hardware Corrupted - *./"},"properties":[{"id":"custom.stacking","value":{"group":false,"mode":"normal"}}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":21},"id":24,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"(((((node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Apps - Memory used by user-space applications","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"PageTables - Memory used to map between virtual and physical memory addresses","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"node_memory_Slab_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)","range":true,"refId":"D","step":240},{"editorMode":"code","expr":"node_memory_Cached_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Cache - Parked file data (file content) cache","range":true,"refId":"E","step":240},{"editorMode":"code","expr":"node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Buffers - Block device (e.g. harddisk) cache","range":true,"refId":"F","step":240},{"editorMode":"code","expr":"node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Unused - Free memory unassigned","range":true,"refId":"G","step":240},{"editorMode":"code","expr":"node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Swap - Swap space used","range":true,"refId":"H","step":240},{"editorMode":"code","expr":"node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working","range":true,"refId":"I","step":240}],"title":"Memory","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Incoming and outgoing network traffic per interface","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":433},"id":84,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) * 8","format":"time_series","legendFormat":"{{device}} - Rx in","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) * 8","format":"time_series","legendFormat":"{{device}} - Tx out","range":true,"refId":"B","step":240}],"title":"Network Traffic","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Network interface utilization as a percentage of its maximum capacity","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":433},"id":338,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"(node_network_speed_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} >bool 0) * (rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) / ignoring(speed) node_network_speed_bytes{instance=\"$node\",job=\"$job\",speed!=\"-1\",cluster=~\"$cluster\"})","format":"time_series","legendFormat":"{{device}} - Rx in","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"(node_network_speed_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} >bool 0) * (rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) / ignoring(speed) node_network_speed_bytes{instance=\"$node\",job=\"$job\",speed!=\"-1\",cluster=~\"$cluster\"})","format":"time_series","legendFormat":"{{device}} - Tx out","range":true,"refId":"B","step":240}],"title":"Network Saturation","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Disk I/O operations per second for each device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":12,"w":12,"x":0,"y":445},"id":229,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\",cluster=~\"$cluster\"}[$__rate_interval])","legendFormat":"{{device}} - Read","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\",cluster=~\"$cluster\"}[$__rate_interval])","legendFormat":"{{device}} - Write","range":true,"refId":"B","step":240}],"title":"Disk IOps","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Disk I/O throughput per device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"read (-) / write (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"Bps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read*./"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":445},"id":42,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Read","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Write","range":true,"refId":"B","step":240}],"title":"Disk Throughput","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Amount of available disk space per mounted filesystem, excluding rootfs. Based on block availability to non-root users","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":12,"w":12,"x":0,"y":457},"id":43,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~\"rootfs\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"{{mountpoint}}","metric":"","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~\"rootfs\",cluster=~\"$cluster\"}","format":"time_series","hide":true,"legendFormat":"{{mountpoint}} - Free","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~\"rootfs\",cluster=~\"$cluster\"}","format":"time_series","hide":true,"legendFormat":"{{mountpoint}} - Size","range":true,"refId":"C","step":240}],"title":"Filesystem Space Available","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Disk usage (used = total - available) per mountpoint","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":12,"w":12,"x":12,"y":457},"id":156,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~\"rootfs\",cluster=~\"$cluster\"} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~\"rootfs\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"{{mountpoint}}","range":true,"refId":"A","step":240}],"title":"Filesystem Used","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Percentage of time the disk was actively processing I/O operations","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":40,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":12,"w":12,"x":0,"y":469},"id":127,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"{{device}}","range":true,"refId":"A","step":240}],"title":"Disk I/O Utilization","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"How often tasks experience CPU, memory, or I/O delays. 'Some' indicates partial slowdown; 'Full' indicates all tasks are stalled. Based on Linux PSI metrics:\nhttps://docs.kernel.org/accounting/psi.html","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"some (-) / full (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Some.*/"},"properties":[{"id":"custom.fillOpacity","value":0}]},{"matcher":{"id":"byRegexp","options":"/.*Some.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":12,"w":12,"x":12,"y":469},"id":322,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"CPU - Some","range":true,"refId":"CPU some","step":240},{"editorMode":"code","expr":"rate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Memory - Some","range":true,"refId":"Memory some","step":240},{"editorMode":"code","expr":"rate(node_pressure_memory_stalled_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Memory - Full","range":true,"refId":"Memory full","step":240},{"editorMode":"code","expr":"rate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"I/O - Some","range":true,"refId":"I/O some","step":240},{"editorMode":"code","expr":"rate(node_pressure_io_stalled_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"I/O - Full","range":true,"refId":"I/O full","step":240},{"editorMode":"code","expr":"rate(node_pressure_irq_stalled_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"IRQ - Full","range":true,"refId":"A","step":240}],"title":"Pressure Stall Information","type":"timeseries"}],"title":"CPU / Memory / Net / Disk","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":21},"id":266,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Displays committed memory usage versus the system's commit limit. Exceeding the limit is allowed under Linux overcommit policies but may increase OOM risks under high load","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*CommitLimit - *./"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":732},"id":135,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Committed_AS – Memory promised to processes (not necessarily used)","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"CommitLimit - Max allowable committed memory","range":true,"refId":"B","step":240}],"title":"Memory Committed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Memory currently dirty (modified but not yet written to disk), being actively written back, or held by writeback buffers. High dirty or writeback memory may indicate disk I/O pressure or delayed flushing","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":732},"id":130,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Writeback – Memory currently being flushed to disk","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"WritebackTmp – FUSE temporary writeback buffers","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Dirty – Memory marked dirty (pending write to disk)","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"NFS Unstable – Pages sent to NFS server, awaiting storage commit","range":true,"refId":"D","step":240}],"title":"Memory Writeback and Dirty","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Kernel slab memory usage, separated into reclaimable and non-reclaimable categories. Reclaimable memory can be freed under memory pressure (e.g., caches), while unreclaimable memory is locked by the kernel for core functions","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":932},"id":131,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"SUnreclaim – Non-reclaimable slab memory (kernel objects)","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"SReclaimable – Potentially reclaimable slab memory (e.g., inode cache)","range":true,"refId":"B","step":240}],"title":"Memory Slab","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Memory used for mapped files (such as libraries) and shared memory (shmem and tmpfs), including variants backed by huge pages","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":932},"id":138,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Mapped – Memory mapped from files (e.g., libraries, mmap)","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Shmem – Shared memory used by processes and tmpfs","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"ShmemHugePages – Shared memory (shmem/tmpfs) allocated with HugePages","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"PMD Mapped – Shmem/tmpfs backed by Transparent HugePages (PMD)","range":true,"refId":"D","step":240}],"title":"Memory Shared and Mapped","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Proportion of memory pages in the kernel's active and inactive LRU lists relative to total RAM. Active pages have been recently used, while inactive pages are less recently accessed but still resident in memory","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Active.*/"},"properties":[{"id":"color","value":{"fixedColor":"green","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"/.*Inactive.*/"},"properties":[{"id":"color","value":{"fixedColor":"dark-blue","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":942},"id":136,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} / node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Inactive – Less recently used memory, more likely to be reclaimed","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_Active_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} / node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Active – Recently used memory, retained unless under pressure","range":true,"refId":"B","step":240}],"title":"Memory LRU Active / Inactive (%)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Breakdown of memory pages in the kernel's active and inactive LRU lists, separated by anonymous (heap, tmpfs) and file-backed (caches, mmap) pages.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":942},"id":191,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Inactive_file - File-backed memory on inactive LRU list","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Inactive_anon – Anonymous memory on inactive LRU (incl. tmpfs & swap cache)","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Active_file - File-backed memory on active LRU list","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Active_anon – Anonymous memory on active LRU (incl. tmpfs & swap cache)","range":true,"refId":"D","step":240}],"title":"Memory LRU Active / Inactive Detail","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Tracks kernel memory used for CPU-local structures, per-thread stacks, and bounce buffers used for I/O on DMA-limited devices. These areas are typically small but critical for low-level operations","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":952},"id":160,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"KernelStack – Kernel stack memory (per-thread, non-reclaimable)","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"PerCPU – Dynamically allocated per-CPU memory (used by kernel modules)","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Bounce Memory – I/O buffer for DMA-limited devices","range":true,"refId":"C","step":240}],"title":"Memory Kernel / CPU / IO","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Usage of the kernel's vmalloc area, which provides virtual memory allocations for kernel modules and drivers. Includes total, used, and largest free block sizes","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Total.*/"},"properties":[{"id":"custom.fillOpacity","value":0},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}},{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":952},"id":70,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Vmalloc Free Chunk – Largest available block in vmalloc area","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Vmalloc Total – Total size of the vmalloc memory area","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Vmalloc Used – Portion of vmalloc area currently in use","range":true,"refId":"C","step":240}],"title":"Memory Vmalloc","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Memory used by anonymous pages (not backed by files), including standard and huge page allocations. Includes heap, stack, and memory-mapped anonymous regions","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":962},"id":129,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"AnonHugePages – Anonymous memory using HugePages","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"AnonPages – Anonymous memory (non-file-backed)","range":true,"refId":"B","step":240}],"title":"Memory Anonymous","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Memory that is locked in RAM and cannot be swapped out. Includes both kernel-unevictable memory and user-level memory locked with mlock()","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working"},"properties":[{"id":"color","value":{"fixedColor":"#CFFAFF","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":962},"id":137,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Unevictable – Kernel-pinned memory (not swappable)","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Mlocked – Application-locked memory via mlock()","range":true,"refId":"B","step":240}],"title":"Memory Unevictable and MLocked","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"How much memory is directly mapped in the kernel using different page sizes (4K, 2M, 1G). Helps monitor large page utilization in the direct map region","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#99440A","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Buffers"},"properties":[{"id":"color","value":{"fixedColor":"#58140C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cache"},"properties":[{"id":"color","value":{"fixedColor":"#6D1F62","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Cached"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Committed"},"properties":[{"id":"color","value":{"fixedColor":"#508642","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Dirty"},"properties":[{"id":"color","value":{"fixedColor":"#6ED0E0","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Free"},"properties":[{"id":"color","value":{"fixedColor":"#B7DBAB","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Mapped"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"PageTables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Page_Tables"},"properties":[{"id":"color","value":{"fixedColor":"#0A50A1","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Slab_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#EAB839","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap"},"properties":[{"id":"color","value":{"fixedColor":"#BF1B00","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Swap_Cache"},"properties":[{"id":"color","value":{"fixedColor":"#C15C17","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"color","value":{"fixedColor":"#511749","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total RAM + Swap"},"properties":[{"id":"color","value":{"fixedColor":"#052B51","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"VmallocUsed"},"properties":[{"id":"color","value":{"fixedColor":"#EA6460","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":972},"id":128,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"DirectMap 1G – Memory mapped with 1GB pages","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"DirectMap 2M – Memory mapped with 2MB pages","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"DirectMap 4K – Memory mapped with 4KB pages","range":true,"refId":"C","step":240}],"title":"Memory DirectMap","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Displays HugePages memory usage in bytes, including allocated, free, reserved, and surplus memory. All values are calculated based on the number of huge pages multiplied by their configured size","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":972},"id":140,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"(node_memory_HugePages_Total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} - node_memory_HugePages_Free{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"HugePages Used – Currently allocated","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"HugePages Reserved – Promised but unused","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"HugePages Surplus – Dynamic pool extension","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"node_memory_HugePages_Total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"HugePages Total – Reserved memory","range":true,"refId":"D","step":240}],"title":"Memory HugePages","type":"timeseries"}],"title":"Memory Meminfo","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":22},"id":267,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of memory pages being read from or written to disk (page-in and page-out operations). High page-out may indicate memory pressure or swapping activity","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"ops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":733},"id":176,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Pagesin - Page in ops","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Pagesout - Page out ops","range":true,"refId":"B","step":240}],"title":"Memory Pages In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate at which memory pages are being swapped in from or out to disk. High swap-out activity may indicate memory pressure","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"ops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":733},"id":22,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Pswpin - Pages swapped in","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Pswpout - Pages swapped out","range":true,"refId":"B","step":240}],"title":"Memory Pages Swap In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of memory page faults, split into total, major (disk-backed), and derived minor (non-disk) faults. High major fault rates may indicate memory pressure or insufficient RAM","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"ops"},"overrides":[{"matcher":{"id":"byName","options":"Pgfault - Page major and minor fault ops"},"properties":[{"id":"custom.fillOpacity","value":0},{"id":"custom.stacking","value":{"group":false,"mode":"none"}},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}},{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":913},"id":175,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":350},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Pgfault - Page major and minor fault ops","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Pgmajfault - Major page fault ops","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Pgminfault - Minor page fault ops","range":true,"refId":"C","step":240}],"title":"Memory Page Faults","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of Out-of-Memory (OOM) kill events. A non-zero value indicates the kernel has terminated one or more processes due to memory exhaustion","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"ops"},"overrides":[{"matcher":{"id":"byName","options":"OOM Kills"},"properties":[{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":913},"id":307,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"OOM Kills","range":true,"refId":"A","step":240}],"title":"OOM Killer","type":"timeseries"}],"title":"Memory Vmstat","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":23},"id":293,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Tracks the system clock's estimated and maximum error, as well as its offset from the reference clock (e.g., via NTP). Useful for detecting synchronization drift","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":734},"id":260,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Estimated error","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_timex_offset_seconds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Offset local vs reference","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Maximum error","range":true,"refId":"C","step":240}],"title":"Time Synchronized Drift","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"NTP phase-locked loop (PLL) time constant used by the kernel to control time adjustments. Lower values mean faster correction but less stability","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":734},"id":291,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_timex_loop_time_constant{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"PLL Time Constant","range":true,"refId":"A","step":240}],"title":"Time PLL Adjust","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Shows whether the system clock is synchronized to a reliable time source, and the current frequency correction ratio applied by the kernel to maintain synchronization","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":884},"id":168,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_timex_sync_status{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Sync status (1 = ok)","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Frequency Adjustment","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_timex_tick_seconds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","hide":true,"interval":"","legendFormat":"Tick Interval","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","hide":true,"interval":"","legendFormat":"TAI Offset","range":true,"refId":"D","step":240}],"title":"Time Synchronized Status","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Displays the PPS signal's frequency offset and stability (jitter) in hertz. Useful for monitoring high-precision time sources like GPS or atomic clocks","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"rothz"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":884},"id":333,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_timex_pps_frequency_hertz{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"PPS Frequency Offset","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_timex_pps_stability_hertz{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"PPS Frequency Stability","range":true,"refId":"B","step":240}],"title":"PPS Frequency / Stability","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Tracks PPS signal timing jitter and shift compared to system clock","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":894},"id":334,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_timex_pps_jitter_seconds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"PPS Jitter","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_timex_pps_shift_seconds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"PPS Shift","range":true,"refId":"B","step":240}],"title":"PPS Time Accuracy","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of PPS synchronization diagnostics including calibration events, jitter violations, errors, and frequency stability exceedances","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":894},"id":335,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_timex_pps_calibration_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"PPS Calibrations/sec","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_timex_pps_error_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"PPS Errors/sec","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"irate(node_timex_pps_stability_exceeded_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"PPS Stability Exceeded/sec","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"irate(node_timex_pps_jitter_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"PPS Jitter Events/sec","range":true,"refId":"D","step":240}],"title":"PPS Sync Events","type":"timeseries"}],"title":"System Timesync","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":24},"id":312,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Processes currently in runnable or blocked states. Helps identify CPU contention or I/O wait bottlenecks.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":735},"id":62,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_procs_blocked{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Blocked (I/O Wait)","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_procs_running{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Runnable (Ready for CPU)","range":true,"refId":"B","step":240}],"title":"Processes Status","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Current number of processes in each state (e.g., running, sleeping, zombie). Requires --collector.processes to be enabled in node_exporter","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"D"},"properties":[{"id":"displayName","value":"Uninterruptible Sleeping"}]},{"matcher":{"id":"byName","options":"I"},"properties":[{"id":"displayName","value":"Idle Kernel Thread"}]},{"matcher":{"id":"byName","options":"R"},"properties":[{"id":"displayName","value":"Running"}]},{"matcher":{"id":"byName","options":"S"},"properties":[{"id":"displayName","value":"Interruptible Sleeping"}]},{"matcher":{"id":"byName","options":"T"},"properties":[{"id":"displayName","value":"Stopped"}]},{"matcher":{"id":"byName","options":"X"},"properties":[{"id":"displayName","value":"Dead"}]},{"matcher":{"id":"byName","options":"Z"},"properties":[{"id":"displayName","value":"Zombie"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":735},"id":315,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_processes_state{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"{{ state }}","range":true,"refId":"A","step":240}],"title":"Processes Detailed States","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of new processes being created on the system (forks/sec).","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":765},"id":148,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_forks_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Process Forks per second","range":true,"refId":"A","step":240}],"title":"Processes Forks","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Shows CPU saturation per core, calculated as the proportion of time spent waiting to run relative to total time demanded (running + waiting).","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*waiting.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":765},"id":305,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","hide":true,"interval":"","legendFormat":"CPU {{ cpu }} - Running","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","hide":true,"interval":"","legendFormat":"CPU {{cpu}} - Waiting Queue","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"((irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) + irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) >bool 0) * (irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) / (irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) + irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])))","format":"time_series","interval":"","legendFormat":"CPU {{cpu}}","range":true,"refId":"C","step":240}],"title":"CPU Saturation per Core","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of active PIDs on the system and the configured maximum allowed. Useful for detecting PID exhaustion risk. Requires --collector.processes in node_exporter","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"PIDs limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":775},"id":313,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_processes_pids{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Number of PIDs","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_processes_max_processes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"PIDs limit","range":true,"refId":"B","step":240}],"title":"PIDs Number and Limit","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of active threads on the system and the configured thread limit. Useful for monitoring thread pressure. Requires --collector.processes in node_exporter","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Threads limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":775},"id":314,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_processes_threads{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Allocated threads","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_processes_max_threads{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Threads limit","range":true,"refId":"B","step":240}],"title":"Threads Number and Limit","type":"timeseries"}],"title":"System Processes","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":25},"id":269,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Per-second rate of context switches and hardware interrupts. High values may indicate intense CPU or I/O activity","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":816},"id":8,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_context_switches_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Context switches","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_intr_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"Interrupts","range":true,"refId":"B","step":240}],"title":"Context Switches / Interrupts","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"System load average over 1, 5, and 15 minutes. Reflects the number of active or waiting processes. Values above CPU core count may indicate overload","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"CPU Core Count"},"properties":[{"id":"custom.fillOpacity","value":0},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}},{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":816},"id":7,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_load1{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Load 1m","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_load5{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Load 5m","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_load15{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Load 15m","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cpu,cluster)) by(cluster)","format":"time_series","legendFormat":"CPU Core Count","range":true,"refId":"D","step":240}],"title":"System Load","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Real-time CPU frequency scaling per core, including average minimum and maximum allowed scaling frequencies","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"hertz"},"overrides":[{"matcher":{"id":"byName","options":"Max"},"properties":[{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}},{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}},{"id":"custom.hideFrom","value":{"legend":true,"tooltip":false,"viz":false}}]},{"matcher":{"id":"byName","options":"Min"},"properties":[{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}},{"id":"color","value":{"fixedColor":"blue","mode":"fixed"}},{"id":"custom.hideFrom","value":{"legend":true,"tooltip":false,"viz":false}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":826},"id":321,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_cpu_scaling_frequency_hertz{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"CPU {{ cpu }}","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"avg(node_cpu_scaling_frequency_max_hertz{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cluster)","format":"time_series","interval":"","legendFormat":"Max","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"avg(node_cpu_scaling_frequency_min_hertz{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) by(cluster)","format":"time_series","interval":"","legendFormat":"Min","range":true,"refId":"C","step":240}],"title":"CPU Frequency Scaling","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of scheduling timeslices executed per CPU. Reflects how frequently the scheduler switches tasks on each core","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":826},"id":306,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"CPU {{ cpu }}","range":true,"refId":"A","step":240}],"title":"CPU Schedule Timeslices","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Breaks down hardware interrupts by type and device. Useful for diagnosing IRQ load on network, disk, or CPU interfaces. Requires --collector.interrupts to be enabled in node_exporter","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":836},"id":259,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_interrupts_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"{{ type }} - {{ info }}","range":true,"refId":"A","step":240}],"title":"IRQ Detail","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of bits of entropy currently available to the system's random number generators (e.g., /dev/random). Low values may indicate that random number generation could block or degrade performance of cryptographic operations","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"decbits"},"overrides":[{"matcher":{"id":"byName","options":"Entropy pool max"},"properties":[{"id":"custom.fillOpacity","value":0},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}},{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":836},"id":151,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_entropy_available_bits{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Entropy available","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_entropy_pool_size_bits{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Entropy pool max","range":true,"refId":"B","step":240}],"title":"Entropy","type":"timeseries"}],"title":"System Misc","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":26},"id":304,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Monitors hardware sensor temperatures and critical thresholds as exposed by Linux hwmon. Includes CPU, GPU, and motherboard sensors where available","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"celsius"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Critical*./"},"properties":[{"id":"color","value":{"fixedColor":"#E24D42","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":737},"id":158,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} * on(chip,cluster) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"{{ chip_name }} {{ sensor }}","range":true,"refId":"A","step":240},{"expr":"node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} * on(chip,cluster) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","hide":true,"interval":"","legendFormat":"{{ chip_name }} {{ sensor }} Critical Alarm","refId":"B","step":240},{"editorMode":"code","expr":"node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} * on(chip,cluster) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"{{ chip_name }} {{ sensor }} Critical","range":true,"refId":"C","step":240},{"expr":"node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} * on(chip,cluster) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","hide":true,"interval":"","legendFormat":"{{ chip_name }} {{ sensor }} Critical Historical","refId":"D","step":240},{"expr":"node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} * on(chip,cluster) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","hide":true,"interval":"","legendFormat":"{{ chip_name }} {{ sensor }} Max","refId":"E","step":240}],"title":"Hardware Temperature Monitor","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Shows how hard each cooling device (fan/throttle) is working relative to its maximum capacity","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"percent"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#EF843C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":737},"id":300,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"(node_cooling_device_max_state{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} >bool 0) * ((100 * node_cooling_device_cur_state{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}) / node_cooling_device_max_state{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"})","format":"time_series","interval":"","legendFormat":"{{name}} - {{type}}","range":true,"refId":"A","step":240}],"title":"Cooling Device Utilization","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Shows the online status of power supplies (e.g., AC, battery). A value of 1-Yes indicates the power supply is active/online","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bool_yes_no"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":747},"id":302,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_power_supply_online{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"{{ power_supply }} online","range":true,"refId":"A","step":240}],"title":"Power Supply","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Displays the current fan speeds (RPM) from hardware sensors via the hwmon interface","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"rotrpm"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":747},"id":325,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_hwmon_fan_rpm{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} * on(chip,cluster) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"{{ chip_name }} {{ sensor }}","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_hwmon_fan_min_rpm{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} * on(chip,cluster) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","hide":true,"interval":"","legendFormat":"{{ chip_name }} {{ sensor }} rpm min","range":true,"refId":"B","step":240}],"title":"Hardware Fan Speed","type":"timeseries"}],"title":"Hardware Misc","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":27},"id":296,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Current number of systemd units in each operational state, such as active, failed, inactive, or transitioning","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Failed"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"color","value":{"fixedColor":"#73BF69","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Activating"},"properties":[{"id":"color","value":{"fixedColor":"#C8F2C2","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Deactivating"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Inactive"},"properties":[{"id":"color","value":{"fixedColor":"dark-blue","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":4228},"id":298,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Activating","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Active","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Deactivating","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Failed","range":true,"refId":"D","step":240},{"editorMode":"code","expr":"node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Inactive","range":true,"refId":"E","step":240}],"title":"Systemd Units State","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Current number of active connections per systemd socket, as reported by the Node Exporter systemd collector","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":4228},"id":331,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_systemd_socket_current_connections{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"{{ name }}","range":true,"refId":"A","step":240}],"title":"Systemd Sockets Current","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of accepted connections per second for each systemd socket","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"eps"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":4238},"id":297,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"{{ name }}","range":true,"refId":"A","step":240}],"title":"Systemd Sockets Accepted","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of systemd socket connection refusals per second, typically due to service unavailability or backlog overflow","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"eps"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":4238},"id":332,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_systemd_socket_refused_connections_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"{{ name }}","range":true,"refId":"A","step":240}],"title":"Systemd Sockets Refused","type":"timeseries"}],"title":"Systemd","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":28},"id":270,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of I/O operations completed per second for the device (after merges), including both reads and writes","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"read (–) / write (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":29},"id":9,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","legendFormat":"{{device}} - Read","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","legendFormat":"{{device}} - Write","range":true,"refId":"B","step":240}],"title":"Disk Read/Write IOps","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of bytes read from or written to the device per second","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"read (–) / write (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"Bps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":29},"id":33,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Read","range":true,"refId":"A","step":240},{"editorMode":"code","exemplar":false,"expr":"irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","instant":false,"legendFormat":"{{device}} - Write","range":true,"refId":"B","step":240}],"title":"Disk Read/Write Data","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"read (–) / write (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"s"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":389},"id":37,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"(irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) >bool 0) * (irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]))","interval":"","legendFormat":"{{device}} - Read","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"(irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) >bool 0) * (irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval]))","interval":"","legendFormat":"{{device}} - Write","range":true,"refId":"B","step":240}],"title":"Disk Average Wait Time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Average queue length of the requests that were issued to the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"none"},"overrides":[{"matcher":{"id":"byRegexp","options":"/sda_*/"},"properties":[{"id":"color","value":{"fixedColor":"#7EB26D","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":389},"id":35,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{device}}","range":true,"refId":"A","step":240}],"title":"Average Queue Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of read and write requests merged per second that were queued to the device","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"read (–) / write (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"iops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":399},"id":133,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","legendFormat":"{{device}} - Read","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","legendFormat":"{{device}} - Write","range":true,"refId":"B","step":240}],"title":"Disk R/W Merged","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Percentage of time the disk spent actively processing I/O operations, including general I/O, discards (TRIM), and write cache flushes","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":399},"id":36,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{device}} - General IO","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{device}} - Discard/TRIM","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"irate(node_disk_flush_requests_time_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{device}} - Flush (write cache)","range":true,"refId":"C","step":240}],"title":"Time Spent Doing I/Os","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Per-second rate of discard (TRIM) and flush (write cache) operations. Useful for monitoring low-level disk activity on SSDs and advanced storage","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"ops"},"overrides":[{"matcher":{"id":"byRegexp","options":"/sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":409},"id":301,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{device}} - Discards completed","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{device}} - Discards merged","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"irate(node_disk_flush_requests_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{device}} - Flush","range":true,"refId":"C","step":240}],"title":"Disk Ops Discards / Flush","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Shows how many disk sectors are discarded (TRIMed) per second. Useful for monitoring SSD behavior and storage efficiency","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":409},"id":326,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_disk_discarded_sectors_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{device}}","range":true,"refId":"A","step":240}],"title":"Disk Sectors Discarded Successfully","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of in-progress I/O requests at the time of sampling (active requests in the disk queue)","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"none"},"overrides":[{"matcher":{"id":"byRegexp","options":"/sda.*/"},"properties":[{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":419},"id":34,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_disk_io_now{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","interval":"","legendFormat":"{{device}}","range":true,"refId":"A","step":240}],"title":"Instantaneous Queue Size","type":"timeseries"}],"title":"Storage Disk","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":29},"id":271,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of file descriptors currently allocated system-wide versus the system limit. Important for detecting descriptor exhaustion risks","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max.*/"},"properties":[{"id":"custom.fillOpacity","value":0},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}},{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":30},"id":28,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_filefd_maximum{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Max open files","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_filefd_allocated{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"Open files","range":true,"refId":"B","step":240}],"title":"File Descriptor","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of free file nodes (inodes) available per mounted filesystem. A low count may prevent file creation even if disk space is available","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":30},"id":41,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~\"rootfs\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"{{mountpoint}}","range":true,"refId":"A","step":240}],"title":"File Nodes Free","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Indicates filesystems mounted in read-only mode or reporting device-level I/O errors.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bool_yes_no"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":370},"id":44,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~\"rootfs\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"{{mountpoint}} - ReadOnly","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~\"rootfs\",fstype!~\"tmpfs\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"{{mountpoint}} - Device error","range":true,"refId":"B","step":240}],"title":"Filesystem in ReadOnly / Error","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of file nodes (inodes) available per mounted filesystem. Reflects maximum file capacity regardless of disk size","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":370},"id":219,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_filesystem_files{instance=\"$node\",job=\"$job\",device!~\"rootfs\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"{{mountpoint}}","range":true,"refId":"A","step":240}],"title":"File Nodes Size","type":"timeseries"}],"title":"Storage Filesystem","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":30},"id":272,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of network packets received and transmitted per second, by interface.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":31},"id":60,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"{{device}} - Rx in","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"rate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"{{device}} - Tx out","range":true,"refId":"B","step":240}],"title":"Network Traffic by Packets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of packet-level errors for each network interface. Receive errors may indicate physical or driver issues; transmit errors may reflect collisions or hardware faults","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":31},"id":142,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Rx in","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"rate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Tx out","range":true,"refId":"B","step":240}],"title":"Network Traffic Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of dropped packets per network interface. Receive drops can indicate buffer overflow or driver issues; transmit drops may result from outbound congestion or queuing limits","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":251},"id":143,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Rx in","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"rate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Tx out","range":true,"refId":"B","step":240}],"title":"Network Traffic Drop","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of compressed network packets received and transmitted per interface. These are common in low-bandwidth or special interfaces like PPP or SLIP","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":251},"id":141,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Rx in","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"rate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Tx out","range":true,"refId":"B","step":240}],"title":"Network Traffic Compressed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of incoming multicast packets received per network interface. Multicast is used by protocols such as mDNS, SSDP, and some streaming or cluster services","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":261},"id":146,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Rx in","range":true,"refId":"A","step":240}],"title":"Network Traffic Multicast","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of received packets that could not be processed due to missing protocol or handler in the kernel. May indicate unsupported traffic or misconfiguration","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":261},"id":327,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_receive_nohandler_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Rx in","range":true,"refId":"A","step":240}],"title":"Network Traffic NoHandler","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of frame errors on received packets, typically caused by physical layer issues such as bad cables, duplex mismatches, or hardware problems","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":271},"id":145,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Rx in","range":true,"refId":"A","step":240}],"title":"Network Traffic Frame","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Tracks FIFO buffer overrun errors on network interfaces. These occur when incoming or outgoing packets are dropped due to queue or buffer overflows, often indicating congestion or hardware limits","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":271},"id":144,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Rx in","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"rate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Tx out","range":true,"refId":"B","step":240}],"title":"Network Traffic Fifo","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of packet collisions detected during transmission. Mostly relevant on half-duplex or legacy Ethernet networks","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":281},"id":232,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Tx out","range":true,"refId":"A","step":240}],"title":"Network Traffic Collision","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of carrier errors during transmission. These typically indicate physical layer issues like faulty cabling or duplex mismatches","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":281},"id":231,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"rate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"{{device}} - Tx out","range":true,"refId":"A","step":240}],"title":"Network Traffic Carrier Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of ARP entries per interface. Useful for detecting excessive ARP traffic or table growth due to scanning or misconfiguration","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":291},"id":230,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_arp_entries{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"{{ device }} ARP Table","range":true,"refId":"A","step":240}],"title":"ARP Entries","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Current and maximum connection tracking entries used by Netfilter (nf_conntrack). High usage approaching the limit may cause packet drops or connection issues","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"NF conntrack limit"},"properties":[{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":291},"id":61,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_nf_conntrack_entries{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"NF conntrack entries","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"NF conntrack limit","range":true,"refId":"B","step":240}],"title":"NF Conntrack","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Operational and physical link status of each network interface. Values are Yes for 'up' or link present, and No for 'down' or no carrier.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bool_yes_no"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":301},"id":309,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","hide":true,"legendFormat":"{{interface}} - Operational state UP","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_network_carrier{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","instant":false,"legendFormat":"{{device}} - Physical link","refId":"B"}],"title":"Network Operational Status","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Maximum speed of each network interface as reported by the operating system. This is a static hardware capability, not current throughput","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"fieldMinMax":false,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bps"},"overrides":[]},"gridPos":{"h":10,"w":6,"x":12,"y":301},"id":280,"options":{"displayMode":"basic","legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"maxVizHeight":30,"minVizHeight":16,"minVizWidth":8,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"manual","valueMode":"color"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_network_speed_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} * 8","format":"time_series","legendFormat":"{{ device }}","range":true,"refId":"A","step":240}],"title":"Speed","type":"bargauge"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"MTU (Maximum Transmission Unit) in bytes for each network interface. Affects packet size and transmission efficiency","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"none"},"overrides":[]},"gridPos":{"h":10,"w":6,"x":18,"y":301},"id":288,"options":{"displayMode":"basic","legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"maxVizHeight":30,"minVizHeight":16,"minVizWidth":8,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"manual","valueMode":"color"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_network_mtu_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"{{ device }}","range":true,"refId":"A","step":240}],"title":"MTU","type":"bargauge"}],"title":"Network Traffic","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":31},"id":273,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Tracks TCP socket usage and memory per node","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":32},"id":63,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Allocated Sockets","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"In-Use Sockets","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Orphaned Sockets","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"TIME_WAIT Sockets","range":true,"refId":"D","step":240}],"title":"Sockstat TCP","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of UDP and UDPLite sockets currently in use","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":32},"id":124,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"UDPLite - In-Use Sockets","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"UDP - In-Use Sockets","range":true,"refId":"B","step":240}],"title":"Sockstat UDP","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Total number of sockets currently in use across all protocols (TCP, UDP, UNIX, etc.), as reported by /proc/net/sockstat","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":42},"id":126,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_sockstat_sockets_used{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Total sockets","range":true,"refId":"A","step":240}],"title":"Sockstat Used","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of FRAG and RAW sockets currently in use. RAW sockets are used for custom protocols or tools like ping; FRAG sockets are used internally for IP packet defragmentation","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":42},"id":125,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"FRAG - In-Use Sockets","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"RAW - In-Use Sockets","range":true,"refId":"C","step":240}],"title":"Sockstat FRAG / RAW","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Kernel memory used by TCP, UDP, and IP fragmentation buffers","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":52},"id":220,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"TCP","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"UDP","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","interval":"","legendFormat":"Fragmentation","range":true,"refId":"C"}],"title":"Sockstat Memory Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Average memory used per socket (TCP/UDP). Helps tune net.ipv4.tcp_rmem / tcp_wmem","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":52},"id":339,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"(node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} >bool 0) * (node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} / node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"})","format":"time_series","interval":"","legendFormat":"TCP","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"(node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} >bool 0) * (node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"} / node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"})","format":"time_series","interval":"","legendFormat":"UDP","range":true,"refId":"B","step":240}],"title":"Sockstat Average Socket Memory","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"TCP/UDP socket memory usage in kernel (in pages)","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":62},"id":336,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"TCP","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"UDP","range":true,"refId":"B","step":240}],"title":"TCP/UDP Kernel Buffer Memory Pages","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Packets processed and dropped by the softnet network stack per CPU. Drops may indicate CPU saturation or network driver limitations","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"drop (-) / process (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Dropped.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":62},"id":290,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"CPU {{cpu}} - Processed","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"CPU {{cpu}} - Dropped","range":true,"refId":"B","step":240}],"title":"Softnet Packets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"How often the kernel was unable to process all packets in the softnet queue before time ran out. Frequent squeezes may indicate CPU contention or driver inefficiency","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"eps"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":72},"id":310,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"CPU {{cpu}} - Times Squeezed","range":true,"refId":"A","step":240}],"title":"Softnet Out of Quota","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Tracks the number of packets processed or dropped by Receive Packet Steering (RPS), a mechanism to distribute packet processing across CPUs","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Dropped.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"},{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":72},"id":330,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_softnet_received_rps_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"CPU {{cpu}} - Processed","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_softnet_flow_limit_count_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"CPU {{cpu}} - Dropped","range":true,"refId":"B","step":240}],"title":"Softnet RPS","type":"timeseries"}],"title":"Network Sockstat","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":32},"id":274,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of octets sent and received at the IP layer, as reported by /proc/net/netstat","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"Bps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":163},"id":221,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"width":300},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"IP Rx in","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","legendFormat":"IP Tx out","range":true,"refId":"B","step":240}],"title":"Netstat IP In / Out Octets","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of TCP segments sent and received per second, including data and control segments","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]},{"matcher":{"id":"byRegexp","options":"/.*Snd.*/"},"properties":[]}]},"gridPos":{"h":10,"w":12,"x":12,"y":163},"id":299,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","instant":false,"interval":"","legendFormat":"TCP Rx in","refId":"A","step":240},{"editorMode":"code","expr":"irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"TCP Tx out","range":true,"refId":"B","step":240}],"title":"TCP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of UDP datagrams sent and received per second, based on /proc/net/netstat","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":193},"id":55,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"UDP Rx in","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"UDP Tx out","range":true,"refId":"B","step":240}],"title":"UDP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of ICMP messages sent and received per second, including error and control messages","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":193},"id":115,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"ICMP Rx in","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"ICMP Tx out","range":true,"refId":"B","step":240}],"title":"ICMP In / Out","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Tracks various TCP error and congestion-related events, including retransmissions, timeouts, dropped connections, and buffer issues","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":203},"id":104,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"Listen Overflows","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"Listen Drops","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"SYN Retransmits","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"Segment Retransmits","range":true,"refId":"D"},{"editorMode":"code","expr":"irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"Receive Errors","range":true,"refId":"E"},{"editorMode":"code","expr":"irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"RST Sent","range":true,"refId":"F"},{"editorMode":"code","expr":"irate(node_netstat_TcpExt_TCPRcvQDrop{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"Receive Queue Drops","range":true,"refId":"G"},{"editorMode":"code","expr":"irate(node_netstat_TcpExt_TCPOFOQueue{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"Out-of-order Queued","range":true,"refId":"H"},{"editorMode":"code","expr":"irate(node_netstat_TcpExt_TCPTimeouts{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"TCP Timeouts","range":true,"refId":"I"}],"title":"TCP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of UDP and UDPLite datagram delivery errors, including missing listeners, buffer overflows, and protocol-specific issues","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":203},"id":109,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"UDP Rx in Errors","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"UDP No Listener","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"UDPLite Rx in Errors","range":true,"refId":"C"},{"editorMode":"code","expr":"irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"UDP Rx in Buffer Errors","range":true,"refId":"D","step":240},{"editorMode":"code","expr":"irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"UDP Tx out Buffer Errors","range":true,"refId":"E","step":240}],"title":"UDP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of incoming ICMP messages that contained protocol-specific errors, such as bad checksums or invalid lengths","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"out (-) / in (+)","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"pps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":213},"id":50,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"ICMP Rx In","range":true,"refId":"A","step":240}],"title":"ICMP Errors","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of TCP SYN cookies sent, validated, and failed. These are used to protect against SYN flood attacks and manage TCP handshake resources under load","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"eps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Failed.*/"},"properties":[{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]}]},"gridPos":{"h":10,"w":12,"x":12,"y":213},"id":91,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"SYN Cookies Failed","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"SYN Cookies Validated","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"SYN Cookies Sent","range":true,"refId":"C","step":240}],"title":"TCP SynCookie","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of currently established TCP connections and the system's max supported limit. On Linux, MaxConn may return -1 to indicate a dynamic/unlimited configuration","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}}]}]},"gridPos":{"h":10,"w":12,"x":0,"y":223},"id":85,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Current Connections","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Max Connections","range":true,"refId":"B","step":240}],"title":"TCP Connections","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of UDP packets currently queued in the receive (RX) and transmit (TX) buffers. A growing queue may indicate a bottleneck","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":223},"id":337,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_udp_queues{instance=\"$node\",job=\"$job\",ip=\"v4\",queue=\"rx\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"UDP Rx in Queue","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_udp_queues{instance=\"$node\",job=\"$job\",ip=\"v4\",queue=\"tx\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"UDP Tx out Queue","range":true,"refId":"B","step":240}],"title":"UDP Queue","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of TCP connection initiations per second. 'Active' opens are initiated by this host. 'Passive' opens are accepted from incoming connections","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"eps"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":233},"id":82,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"Active Opens","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"Passive Opens","range":true,"refId":"B","step":240}],"title":"TCP Direct Transition","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of TCP sockets in key connection states. Requires the --collector.tcpstat flag on node_exporter","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":233},"id":320,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Established","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"FIN_WAIT2","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"Listen","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"TIME_WAIT","range":true,"refId":"D","step":240},{"editorMode":"code","expr":"node_tcp_connection_states{state=\"close_wait\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"CLOSE_WAIT","range":true,"refId":"E","step":240}],"title":"TCP Stat Persistent","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Transient TCP connection states. These are typically short-lived during connection establishment and teardown","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":243},"id":339,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_tcp_connection_states{state=\"syn_sent\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"SYN_SENT","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_tcp_connection_states{state=\"syn_recv\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"SYN_RECV","range":true,"refId":"B","step":240},{"editorMode":"code","expr":"node_tcp_connection_states{state=\"fin_wait1\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"FIN_WAIT1","range":true,"refId":"C","step":240},{"editorMode":"code","expr":"node_tcp_connection_states{state=\"close\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"CLOSE","range":true,"refId":"D","step":240},{"editorMode":"code","expr":"node_tcp_connection_states{state=\"last_ack\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"LAST_ACK","range":true,"refId":"E","step":240},{"editorMode":"code","expr":"node_tcp_connection_states{state=\"closing\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"CLOSING","range":true,"refId":"F","step":240}],"title":"TCP Stat Transient","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"TCP socket queue sizes. High rx_queued_bytes indicates application not reading fast enough. High tx_queued_bytes indicates network congestion or slow receiver","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":243},"id":340,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_tcp_connection_states{state=\"rx_queued_bytes\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"RX Queued (waiting to be read)","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"node_tcp_connection_states{state=\"tx_queued_bytes\",instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","legendFormat":"TX Queued (waiting to be sent)","range":true,"refId":"B","step":240}],"title":"TCP Socket Queue","type":"timeseries"}],"title":"Network Netstat","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":33},"id":279,"panels":[{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Duration of each individual collector executed during a Node Exporter scrape. Useful for identifying slow or failing collectors","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":0,"y":164},"id":40,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"{{collector}}","range":true,"refId":"A","step":240}],"title":"Node Exporter Scrape Time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Rate of CPU time used by the process exposing this metric (user + system mode)","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":10,"w":12,"x":12,"y":164},"id":308,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","legendFormat":"Process CPU Usage","range":true,"refId":"A","step":240}],"title":"Exporter Process CPU Usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Tracks the memory usage of the process exposing this metric (e.g., node_exporter), including current virtual memory and maximum virtual memory limit","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"Virtual Memory Limit"},"properties":[{"id":"custom.fillOpacity","value":0},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}},{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]},{"__systemRef":"hideSeriesFrom","matcher":{"id":"byNames","options":{"mode":"exclude","names":["Virtual Memory"],"prefix":"All except:","readOnly":true}},"properties":[{"id":"custom.hideFrom","value":{"legend":false,"tooltip":false,"viz":true}}]}]},"gridPos":{"h":10,"w":10,"x":0,"y":174},"id":149,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"process_virtual_memory_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","interval":"","legendFormat":"Virtual Memory","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","interval":"","legendFormat":"Virtual Memory Limit","range":true,"refId":"B","step":240}],"title":"Exporter Processes Memory","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Number of file descriptors used by the exporter process versus its configured limit","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":20,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*Max*./"},"properties":[{"id":"color","value":{"fixedColor":"#890F02","mode":"fixed"}},{"id":"custom.fillOpacity","value":0},{"id":"custom.lineStyle","value":{"dash":[10,10],"fill":"dash"}}]},{"__systemRef":"hideSeriesFrom","matcher":{"id":"byNames","options":{"mode":"exclude","names":["Open file descriptors"],"prefix":"All except:","readOnly":true}},"properties":[{"id":"custom.hideFrom","value":{"legend":false,"tooltip":false,"viz":true}}]}]},"gridPos":{"h":10,"w":10,"x":10,"y":174},"id":64,"options":{"legend":{"calcs":["min","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"process_max_fds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","interval":"","legendFormat":"Maximum open file descriptors","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"process_open_fds{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","interval":"","legendFormat":"Open file descriptors","range":true,"refId":"B","step":240}],"title":"Exporter File Descriptor Usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"description":"Shows whether each Node Exporter collector scraped successfully (1 = success, 0 = failure), and whether the textfile collector returned an error.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"dark-red","value":0},{"color":"green","value":1}]},"unit":"bool"},"overrides":[]},"gridPos":{"h":10,"w":4,"x":20,"y":174},"id":157,"options":{"displayMode":"basic","legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"maxVizHeight":300,"minVizHeight":16,"minVizWidth":8,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"node_scrape_collector_success{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"{{collector}}","range":true,"refId":"A","step":240},{"editorMode":"code","expr":"1 - node_textfile_scrape_error{instance=\"$node\",job=\"$job\",cluster=~\"$cluster\"}","format":"time_series","interval":"","legendFormat":"textfile","range":true,"refId":"B","step":240}],"title":"Node Exporter Scrape","type":"bargauge"}],"title":"Node Exporter","type":"row"}],"refresh":"1m","schemaVersion":41,"tags":["linux","vm-k8s-stack"],"templating":{"list":[{"current":{},"label":"Datasource","name":"ds_prometheus","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"current":{},"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"label":"Job","name":"job","options":[],"query":{"query":"label_values(node_uname_info, job)","refId":"Prometheus-job-Variable-Query"},"refresh":1,"regex":"","sort":1,"type":"query"},{"current":{},"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"definition":"label_values(node_uname_info{job=\"$job\",cluster=~\"$cluster\"}, nodename)","label":"Nodename","name":"nodename","options":[],"query":{"query":"label_values(node_uname_info{job=\"$job\",cluster=~\"$cluster\"}, nodename)","refId":"Prometheus-nodename-Variable-Query"},"refresh":1,"regex":"","sort":1,"type":"query"},{"current":{},"datasource":{"type":"prometheus","uid":"${ds_prometheus}"},"definition":"label_values(node_uname_info{job=\"$job\",nodename=\"$nodename\",cluster=~\"$cluster\"}, instance)","label":"Instance","name":"node","options":[],"query":{"query":"label_values(node_uname_info{job=\"$job\",nodename=\"$nodename\",cluster=~\"$cluster\"}, instance)","refId":"Prometheus-node-Variable-Query"},"refresh":1,"regex":"","sort":1,"type":"query"},{"allValue":".*","datasource":{"type":"prometheus"},"hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","query":".*","type":"constant"}]},"time":{"from":"now-24h","to":"now"},"timepicker":{},"timezone":"utc","title":"Node Exporter Full","uid":"rYdddlPWk","version":99,"weekStart":""}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-victoriametrics-operator
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  victoriametrics-operator.json: | 
    {"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},{"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"expr":"sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__interval])) by(job,cluster)","hide":false,"iconColor":"dark-yellow","name":"restarts","textFormat":"{{job}} restarted"}]},"description":"Overview for operator VictoriaMetrics v0.25.0 or higher","editable":false,"fiscalYearStartMonth":0,"graphTooltip":0,"id":0,"links":[{"icon":"doc","tags":[],"targetBlank":true,"title":"Docs","type":"link","url":"https://docs.victoriametrics.com/operator/"},{"icon":"info","tags":[],"targetBlank":true,"title":"Community support","type":"link","url":"https://victoriametrics.com/support/community-support/"},{"asDropdown":false,"icon":"bolt","includeVars":false,"keepTime":false,"tags":[],"targetBlank":true,"title":"Enterprise support","tooltip":"","type":"link","url":"https://victoriametrics.com/support/enterprise-support/"},{"icon":"info","tags":[],"targetBlank":true,"title":"Releases","type":"link","url":"https://github.com/VictoriaMetrics/operator/releases"}],"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":8,"title":"Overview","type":"row"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]}},"overrides":[]},"gridPos":{"h":3,"w":4,"x":0,"y":1},"id":24,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"/^short_version$/","values":false},"showPercentChange":false,"textMode":"value","wideLayout":true},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}","format":"table","instant":true,"interval":"","legendFormat":"{{short_version}}","range":false,"refId":"A"}],"title":"Version","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Number of objects at kubernetes cluster per each controller","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":20,"x":4,"y":1},"id":14,"options":{"colorMode":"none","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(operator_controller_objects_count{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(controller,cluster)","legendFormat":"{{controller}}","range":true,"refId":"A"}],"title":"CRD Objects count by controller","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":4,"w":4,"x":0,"y":4},"id":22,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"vm_app_uptime_seconds{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}","format":"table","instant":true,"interval":"","legendFormat":"{{instance}}","range":false,"refId":"A"}],"title":"Uptime","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":" Shows per namespace watchers for Prometheus Operator objects (ServiceMonitors, PodMonitors, etc) ","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":4,"x":0,"y":8},"id":39,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(operator_prometheus_converter_active_watchers) by(cluster)","instant":false,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Prometheus Objects watchers","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":" Number of operator instances with obtained leader status. \n Value above 1 indicates that instances with the same job may behave incorrectly.\n It's recommend to check Operator logs. ","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":4,"x":4,"y":8},"id":40,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(leader_election_master_status{job=~\"$job\",cluster=~\"$cluster\"}) by(cluster)","instant":false,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Elected Leaders","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":" Shows number of active reconcile workers","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"min":0,"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":4,"x":8,"y":8},"id":41,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(controller_runtime_active_workers{job=~\"$job\",cluster=~\"$cluster\"}) by(cluster)","instant":false,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Active workers","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":" Shows amount of Prometheus Operator objects processed by Operator.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":12,"x":12,"y":8},"id":38,"options":{"legend":{"calcs":["max","mean"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(operator_prometheus_converter_watch_events_total{job=~\"$job\",cluster=~\"$cluster\"}[$__interval])) by(event_type,object_type_name,cluster)","instant":false,"legendFormat":"{{object_type_name}} {{event_type}}","range":true,"refId":"A"}],"title":"Prometheus Converter Watch events","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":12,"x":0,"y":13},"id":43,"options":{"cellHeight":"sm","showHeader":true},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(flag{is_set=\"true\",job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,name,value,cluster)","format":"table","instant":true,"legendFormat":"{{object_type_name}} {{event_type}}","range":false,"refId":"A"}],"title":"Non-default flags","transformations":[{"id":"groupBy","options":{"fields":{"instance":{"aggregations":["uniqueValues"],"operation":"aggregate"},"job":{"aggregations":[],"operation":"groupby"},"name":{"aggregations":[],"operation":"groupby"},"value":{"aggregations":[],"operation":"groupby"}}}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":12,"x":12,"y":13},"id":45,"options":{"cellHeight":"sm","showHeader":true},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(config_parameter{job=~\"$job\",instance=~\"$instance\",value!=\"\",cluster=~\"$cluster\"}) by(job,instance,name,value,cluster)","format":"table","instant":true,"legendFormat":"{{object_type_name}} {{event_type}}","range":false,"refId":"A"}],"title":"Global settings","transformations":[{"id":"groupBy","options":{"fields":{"instance":{"aggregations":["uniqueValues"],"operation":"aggregate"},"job":{"aggregations":[]},"name":{"aggregations":[],"operation":"groupby"},"value":{"aggregations":[],"operation":"groupby"}}}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":13,"w":12,"x":0,"y":18},"id":12,"options":{"legend":{"calcs":["mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(controller_runtime_reconcile_total{job=~\"$job\",instance=~\"$instance\",result=~\"requeue_after|requeue|success\",cluster=~\"$cluster\"}[$__rate_interval])) by(controller,cluster)","legendFormat":"{{controller}}","range":true,"refId":"A"}],"title":"Reconciliation rate by controller","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":13,"w":12,"x":12,"y":18},"id":16,"options":{"legend":{"calcs":["mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(operator_log_messages_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(level,cluster)","legendFormat":"{{label_name}}","range":true,"refId":"A"}],"title":"Log message rate","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":31},"id":6,"title":"Troubleshooting","type":"row"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Non zero metrics indicates about error with CR object definition (typos or incorrect values) or errors with kubernetes API connection.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"id":10,"options":{"legend":{"calcs":["lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(controller_runtime_reconcile_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(controller,cluster) > 0","instant":false,"legendFormat":"{{controller}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(controller_runtime_reconcile_total{job=~\"$job\",instance=~\"$instance\",result=\"error\",cluster=~\"$cluster\"}[$__rate_interval])) by(controller,cluster) > 0","hide":false,"legendFormat":"result errors: {{controller}}","range":true,"refId":"B"}],"title":"Reconcile errors by controller","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Operator limits number of reconcile configuration events to 5 events per 2 seconds by default.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"id":18,"options":{"legend":{"calcs":["mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(operator_reconcile_throttled_events_total[$__rate_interval])) by(controller,cluster)","legendFormat":"{{controller}}","range":true,"refId":"A"}],"title":"Throttled reconciliation config events","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Number of objects waiting in the queue for reconciliation. Non-zero values indicate that operator cannot process CR objects changes with the given resources.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":11,"w":12,"x":0,"y":40},"id":20,"options":{"legend":{"calcs":["max","mean"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(workqueue_depth{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(name,cluster)","legendFormat":"{{label_name}}","range":true,"refId":"A"}],"title":"Working queue depth","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":" For controllers with StatefulSet it's ok to see latency greater then 3 seconds. It could be vmalertmanager,vmcluster or vmagent in statefulMode.\n\n For other controllers, latency greater then 2 second may indicate issues with kubernetes cluster or operator's performance.\n ","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":11,"w":12,"x":12,"y":40},"id":26,"options":{"legend":{"calcs":["mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Mean","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(le,controller,cluster))","legendFormat":"q.99 {{controller}}","range":true,"refId":"A"}],"title":"Reconciliation latency by controller","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Number of HTTP requests to the Kubernetes API server break down by code and method","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":51},"id":36,"options":{"legend":{"calcs":["max","mean"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(rest_client_requests_total{job=~\"$job\",cluster=~\"$cluster\"}[$__interval])) by(method,code,cluster)","instant":false,"legendFormat":"{{method}} {{code}}","range":true,"refId":"A"}],"title":"REST client requests","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows how many ongoing reconcile events are taking place, where:\n* `max` - equal to the value of flag`-controller.maxConcurrentReconciles`;\n* `current` - current number of reconcile workers processing CRD objects.\n\nWhen `current` hits `max` constantly, it means operator cannot process events in time. It should be either increased value for flag `-controller.maxConcurrentReconciles` or allocated  additional CPU resources to the operator.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"Max"},"properties":[{"id":"custom.axisColorMode","value":"text"},{"id":"color","value":{"fixedColor":"dark-red","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":51},"id":42,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(max_over_time(controller_runtime_active_workers{job=~\"$job\",cluster=~\"$cluster\"}[$__interval])) by(cluster)","instant":false,"legendFormat":"Current","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"min(controller_runtime_max_concurrent_reconciles{job=~\"$job\",cluster=~\"$cluster\"}) by(cluster)","hide":false,"instant":false,"legendFormat":"Max","range":true,"refId":"B"}],"title":"Concurrent reconcile ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":59},"id":37,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,le,cluster))) by(job,cluster)","instant":false,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Go scheduling latency","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":" Requests latency to the Kubernetes API server.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":59},"id":35,"options":{"legend":{"calcs":["max","mean"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(le,method,api,cluster))","instant":false,"legendFormat":"{{method}} {{api}}","range":true,"refId":"A"}],"title":"REST client latency","type":"timeseries"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":67},"id":4,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":63},"id":28,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(go_memstats_sys_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","legendFormat":"requested from system","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(go_memstats_heap_inuse_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"heap inuse","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(go_memstats_stack_inuse_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"stack inuse","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","hide":false,"legendFormat":"resident","range":true,"refId":"D"}],"title":"Memory usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":63},"id":30,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","legendFormat":"CPU cores used","range":true,"refId":"A"}],"title":"CPU ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":71},"id":32,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(go_goroutines{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","legendFormat":"goroutines","range":true,"refId":"A"}],"title":"Goroutines ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":71},"id":34,"options":{"legend":{"calcs":["mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(go_gc_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) / sum(rate(go_gc_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","legendFormat":"avg gc duration","range":true,"refId":"A"}],"title":"GC duration ($instance)","type":"timeseries"}],"title":"Resources","type":"row"}],"preload":false,"refresh":"","schemaVersion":42,"tags":["victoriametrics","vm-k8s-stack"],"templating":{"list":[{"current":{},"name":"ds","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(operator_log_messages_total, job)","name":"job","options":[],"query":{"query":"label_values(operator_log_messages_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","type":"query"},{"current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(operator_log_messages_total{job=~\"$job\",cluster=~\"$cluster\"}, instance)","includeAll":true,"name":"instance","options":[],"query":{"query":"label_values(operator_log_messages_total{job=~\"$job\",cluster=~\"$cluster\"}, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","type":"query"},{"current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{job=\"$job\",instance=\"$instance\",cluster=~\"$cluster\"}, version)","hide":2,"name":"version","options":[],"query":{"query":"label_values(vm_app_version{job=\"$job\",instance=\"$instance\",cluster=~\"$cluster\"}, version)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","sort":2,"type":"query"},{"baseFilters":[],"datasource":{"type":"prometheus","uid":"$ds"},"filters":[],"name":"adhoc","type":"adhoc"},{"allValue":".*","datasource":{"type":"prometheus"},"hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","query":".*","type":"constant"}]},"time":{"from":"now-15m","to":"now"},"timepicker":{},"timezone":"utc","title":"VictoriaMetrics - operator","uid":"1H179hunk","version":8}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-victoriametrics-single-node
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  victoriametrics-single-node.json: | 
    {"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},{"datasource":{"type":"prometheus","uid":"$ds"},"enable":false,"expr":"sum(ALERTS{job=~\"$job\",instance=~\"$instance\",alertgroup=\"vmsingle\",alertstate=\"firing\",show_at=\"dashboard\",cluster=~\"$cluster\"}) by(alertname,cluster)","hide":false,"iconColor":"red","name":"alerts","titleFormat":"{{alertname}}"},{"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"expr":"sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(version,cluster) unless sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} offset $__interval) by(version,cluster)","hide":true,"iconColor":"dark-blue","name":"version","textFormat":"{{version}}","titleFormat":"Version change"},{"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"expr":"sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__interval])) by(job,cluster)","hide":false,"iconColor":"dark-yellow","name":"restarts","textFormat":"{{job}} restarted"}]},"description":"Overview for single-node VictoriaMetrics v1.117.0 or higher","editable":false,"fiscalYearStartMonth":0,"graphTooltip":1,"id":13,"links":[{"icon":"doc","tags":[],"targetBlank":true,"title":"Docs","type":"link","url":"https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/"},{"asDropdown":false,"icon":"question","includeVars":false,"keepTime":false,"tags":[],"targetBlank":true,"title":"Troubleshooting","tooltip":"","type":"link","url":"https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#troubleshooting"},{"icon":"info","tags":[],"targetBlank":true,"title":"Community support","type":"link","url":"https://victoriametrics.com/support/community-support/"},{"asDropdown":false,"icon":"bolt","includeVars":false,"keepTime":false,"tags":[],"targetBlank":true,"title":"Enterprise support","tooltip":"","type":"link","url":"https://victoriametrics.com/support/enterprise-support/"},{"icon":"info","tags":[],"targetBlank":true,"title":"Releases","type":"link","url":"https://github.com/VictoriaMetrics/VictoriaMetrics/releases"}],"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":6,"title":"Stats","type":"row"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"How many datapoints are in storage","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":0,"y":1},"id":26,"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",cluster=~\"$cluster\"}) by(cluster)","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Total datapoints","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the datapoints ingestion rate.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":6,"y":1},"id":107,"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vm_rows_inserted_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Ingestion rate","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of HTTP read requests.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"req/s"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":12,"y":1},"id":108,"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vm_http_requests_total{job=~\"$job\",instance=~\"$instance\",path!~\".*(/write|/metrics)\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Read requests","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Total size of available memory for VM process","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":18,"y":1},"id":78,"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"uid":"$ds"},"exemplar":false,"expr":"sum(vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Available memory","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":0,"y":4},"id":38,"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vm_cache_entries{job=~\"$job\",instance=~\"$instance\",type=\"storage/hour_metric_ids\",cluster=~\"$cluster\"}) by(cluster)","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Active series","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Total amount of used disk space","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":6,"y":4},"id":81,"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Disk space usage","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Average disk usage per datapoint.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":12,"y":4},"id":82,"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)) by(cluster)","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Bytes per point","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Total number of available CPUs for VM process","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":18,"y":4},"id":77,"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"exemplar":false,"expr":"sum(vm_available_cpu_cores{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Available CPU","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false,"minWidth":50},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"Time"},"properties":[{"id":"custom.hideFrom.viz","value":true}]},{"matcher":{"id":"byName","options":"Value"},"properties":[{"id":"displayName","value":"Count"}]}]},"gridPos":{"h":5,"w":9,"x":0,"y":7},"id":155,"options":{"cellHeight":"sm","showHeader":true,"sortBy":[{"desc":true,"displayName":"Count"}]},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,short_version,cluster)","format":"table","instant":true,"range":false,"refId":"A"}],"type":"table"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMin":0,"barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"stepAfter","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":5,"w":15,"x":9,"y":7},"id":156,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(min_over_time(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","instant":false,"legendFormat":"{{job}}","refId":"A"}],"title":"Uptime ($job)","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":12},"id":24,"title":"Overview","type":"row"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"How many datapoints are inserted into storage per second","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":13},"id":106,"options":{"legend":{"calcs":["mean","lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_rows_inserted_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(type,instance,cluster) > 0","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}} - {{type}}","range":true,"refId":"A"}],"title":"Datapoints ingestion rate","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":13},"id":12,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_http_requests_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(path,cluster) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"A"}],"title":"Requests rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown. \n\nSee following link for details:","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":21},"id":51,"links":[{"targetBlank":true,"title":"troubleshooting","url":"https://docs.victoriametrics.com/victoriametrics/troubleshooting/"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_cache_entries{job=~\"$job\",instance=~\"$instance\",type=\"storage/hour_metric_ids\",cluster=~\"$cluster\"}) by(cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],"title":"Active time series","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"The less time it takes is better.\n* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":21},"id":22,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.4.0-20418128491","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(vm_request_duration_seconds{job=~\"$job\",instance=~\"$instance\",quantile=\"0.99\",cluster=~\"$cluster\"}) by(path,cluster) > 0","format":"time_series","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"A"}],"title":"Query duration 0.99 quantile ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":29},"id":35,"options":{"legend":{"calcs":["mean","lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vm_http_request_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,path,cluster) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{path}}","range":true,"refId":"A"}],"title":"Requests error rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"bars","fillOpacity":100,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Drilldown","url":"/d/wNf0q_kZk?viewPanel=154&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":29},"id":110,"options":{"legend":{"calcs":["lastNotNull","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.3.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\",level!=\"info\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,level,cluster) > 0","interval":"5m","legendFormat":"{{job}} - {{level}}","range":true,"refId":"A"}],"title":"Logging rate","type":"timeseries"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":37},"id":46,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"description":"How much resident memory (RAM) the process is actually using, compared to its allowed container or system limit.\n\n- Good: Below 70% most of the time, maybe spiking a bit under load.\n- Bad: Above 90% for more than 5 minutes = risk of out-of-memory (OOM) kill.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Drilldown - RSS memory usage","url":"/d/wNf0q_kZk?viewPanel=148&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"},{"targetBlank":true,"title":"Drilldown - Memory usage breakdown","url":"/d/wNf0q_kZk?viewPanel=141&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":31},"id":112,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"RSS memory % usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Drilldown","url":"/d/wNf0q_kZk?viewPanel=151&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":31},"id":114,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"CPU % usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Drilldown","url":"/d/wNf0q_kZk?viewPanel=149&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":39},"id":123,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(max_over_time(process_resident_memory_anon_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"RSS anonymous memory % usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). Helps troubleshoot high CPU usage or throttling.\nLower is better.\n\n- waiting: The percentage of time at least one task in the process was ready to run (runnable) but couldn't get scheduled on the CPU.\n- stalled: The percentage of time all tasks in the process (except idle ones) were unable to get CPU time — a full CPU stall.\n\nIf there's a CPU burst, it's normal to see waiting or stalled > 1%. It only becomes a concern if it consistently climbs above 5–10% and aligns with latency spikes or GC slowdowns.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":39},"id":134,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_cpu_waiting_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{job}}-{{instance}} - waiting","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_cpu_stalled_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{job}}-{{instance}} - waiting","range":true,"refId":"B"}],"title":"CPU pressure","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"111","url":"111"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":47},"id":44,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(go_memstats_sys_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster) + sum(vm_cache_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{job}} - requested from system","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(go_memstats_heap_inuse_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster) + sum(vm_cache_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{job}} - heap inuse","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(go_memstats_stack_inuse_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{job}} - stack inuse","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}} - resident","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(process_resident_memory_anon_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}} - resident anonymous","range":true,"refId":"E"}],"title":"Memory usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"CPU cores used by instance","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"Limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":47},"id":57,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"min(process_cpu_cores_available{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Limit","range":true,"refId":"B"}],"title":"CPU","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\nLower is better.\n\n- waiting: Time fraction where at least one thread was blocked on memory.\n- stalled: Time fraction where every thread was blocked on memory (severe pressure).\n\nIf queries slow down and both series spike, the host is likely limited by RAM or I/O throughput.\n\nSee also major page faults rate panel in Troubleshooting section if this metric continued to be high.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":55},"id":135,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_memory_waiting_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{job}} - waiting","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_memory_stalled_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{job}} - stalled","range":true,"refId":"B"}],"title":"Memory pressure","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of bytes read/write from the storage layer.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*read/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":55},"id":76,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_storage_read_bytes_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}} - read","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_storage_written_bytes_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}} - write","range":true,"refId":"B"}],"title":"Disk writes/reads ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the percentage of open file descriptors compared to the limit set in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[{"matcher":{"id":"byRegexp","options":"/max.*/"},"properties":[{"id":"color","value":{"fixedColor":"#C4162A","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":63},"id":142,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max(max_over_time(process_open_fds{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / process_max_fds{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Open FDs usage % ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of read/write syscalls such as read, pread, write, pwrite.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/.*read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":63},"id":124,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_read_syscalls_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}} - read calls","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_write_syscalls_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}} - write calls","range":true,"refId":"B"}],"title":"Disk write/read calls ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":71},"id":47,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(go_goroutines{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","intervalFactor":2,"legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Goroutines ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows IO pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\nThe lower the better.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":71},"id":136,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_io_waiting_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{job}} - waiting","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_io_stalled_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{job}} - stalled","range":true,"refId":"B"}],"title":"IO pressure","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":79},"id":143,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(max_over_time(process_num_threads{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Threads ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":79},"id":37,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_tcplistener_conns{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"TCP connections ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ ","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":87},"id":125,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(rate(go_gc_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"CPU spent on GC ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":87},"id":49,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_tcplistener_accepts_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"TCP connections rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":95},"id":133,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(rate(go_memstats_alloc_bytes_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Memory allocations rate","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of            insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":0.1}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":95},"id":128,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,le,cluster))) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Go scheduling latency","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows network usage by VM:\n* Writes show traffic sent to clients\n* Reads show traffic received from clients","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"decbits"},"overrides":[{"matcher":{"id":"byRegexp","options":"/read.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":103},"id":127,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"(sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(name,cluster) * 8) > 0","format":"time_series","intervalFactor":1,"legendFormat":"read via {{name}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"(sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(name,cluster) * 8) > 0","format":"time_series","intervalFactor":1,"legendFormat":"write via {{name}}","range":true,"refId":"B"}],"title":"Network usage","type":"timeseries"}],"title":"Resource usage","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":38},"id":71,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate and total number of new series created over last 24h.\n\nHigh [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected [cardinality](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#cardinality) \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"id":66,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_new_timeseries_created_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"","legendFormat":"churn rate","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(increase(vm_new_timeseries_created_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[24h])) by(cluster)","interval":"","legendFormat":"new series over 24h","range":true,"refId":"B"}],"title":"Churn rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"The percentage of [slow inserts](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \n\nThe lower the better. \n\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \n* Ingestion of completely new, not seen before time series;\n* Not enough memory to maintain big enough caches for the current workload.\n\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series). \n\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line+area"}},"decimals":2,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"transparent","value":0},{"color":"red","value":0.1}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"id":68,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(rate(vm_slow_row_inserts_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval]) / rate(vm_rows_added_to_storage_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"slow inserts","range":true,"refId":"A"}],"title":"Slow inserts %","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Merge assist happens when storage can't keep up with merging parts. This is usually a sign of overload for storage.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"bars","fillOpacity":100,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"id":116,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vm_assisted_merges_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(instance,type,cluster) > 0","format":"time_series","interval":"5m","intervalFactor":1,"legendFormat":"{{instance}} - {{type}}","range":true,"refId":"A"}],"title":"Assisted merges","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Slow queries rate according to `search.logSlowQueryDuration` flag, which is `5s` by default.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"id":60,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_slow_queries_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Slow queries rate","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":0,"y":48},"id":90,"options":{"legend":{"calcs":["lastNotNull","mean","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(vm_cache_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} / vm_cache_size_max_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(type,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Cache usage % by type","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows cache miss ratio. Lower is better.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":9,"w":12,"x":12,"y":48},"id":118,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(rate(vm_cache_misses_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / rate(vm_cache_requests_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(type,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Cache miss ratio ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows how many samples were ignored or dropped on insertion due to various reasons:\n* timestamp out of retention period or timestamp in future;\n* invalid metric name;\n* exceeding limit for labels length or number;\n* dropped by [relabeling configuration](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#relabeling).","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":57},"id":131,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(increase(vm_rows_ignored_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1h])) by(job,reason,cluster)","interval":"","legendFormat":"{{instance}} - {{reason}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(increase(vm_relabel_metrics_dropped_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1h])) by(job,cluster)","instant":false,"legendFormat":"{{job}} - relabeling","range":true,"refId":"B"}],"title":"Samples dropped for last 1h","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows rate of deduplicated samples during [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) or [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling), according to the configured `-dedup.minScrapeInterval` or `-downsampling.period`.\n\nShould be empty if `-dedup.minScrapeInterval` and `-downsampling.period` isn't set.\n\nDeduplication happens in two places:\n* `type=\"merge\"` - during [background merges](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage). Deduplication during merges is permanent.\n* `type=\"select\"` - during [read queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data). Deduplication happens in-flight and has no permanent effect.\n\nThe more samples need to be deduplicated, the higher will be resource usage.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":57},"id":132,"links":[{"targetBlank":true,"title":"Readonly mode","url":"https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vm_deduplicated_samples_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,type,cluster) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Deduplication rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Flags explicitly set to non-default values","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"filterable":true,"footer":{"reducers":[]},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"Time"},"properties":[{"id":"custom.hideFrom.viz","value":true}]},{"matcher":{"id":"byName","options":"Value"},"properties":[{"id":"custom.hideFrom.viz","value":true}]},{"matcher":{"id":"byName","options":"job"},"properties":[{"id":"custom.hideFrom.viz","value":true}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":65},"id":126,"options":{"cellHeight":"sm","filterable":true,"showHeader":true,"showSearch":true},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(flag{is_set=\"true\",job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,name,value,cluster)","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],"title":"Non-default flags","type":"table"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMin":0,"barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"stepAfter","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":65},"id":129,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) > 0) by(job,cluster)","format":"time_series","instant":false,"legendFormat":"{{job}}","refId":"A"}],"title":"Restarts ($job)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds}"},"description":"Major page faults occur when required data is not in physical memory and must be loaded from disk (storage or swap). It usually happens with:\n- insufficient physical memory\n- active use of swap space\n- file mapping access\n\nIt may cause significant performance slowdown if sustained high.\n\nPossible workaround:\n- Extend RAM to scale up\n- Reduce swap usage by adjusting vm.swappiness","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Drilldown","url":"/d/wNf0q_kZk?viewPanel=140&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":73},"id":137,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"sum(rate(process_major_pagefaults_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) > 0","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Major page faults rate","type":"timeseries"}],"title":"Troubleshooting","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":39},"id":14,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"description":"How many datapoints are inserted into storage per second","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":33},"id":10,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_rows_inserted_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,type,cluster) > 0","format":"time_series","intervalFactor":1,"legendFormat":"{{job}} - {{type}}","range":true,"refId":"A"}],"title":"Datapoints ingestion rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the approx time needed to reach 100% of allowed disk capacity based on the following params:\n* free disk space (after -storage.minFreeDiskSpaceBytes);\n* row ingestion rate;\n* compression.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Drilldown","url":"/d/wNf0q_kZk?viewPanel=150&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":33},"id":73,"options":{"legend":{"calcs":["mean","lastNotNull","min"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"((vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) / ignoring(path) (((rate(vm_rows_added_to_storage_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1d]) - sum(rate(vm_deduplicated_samples_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1d])) without(type)) * (sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",cluster=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",cluster=~\"$cluster\"}) without(type))) + (rate(vm_new_timeseries_created_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1d]) * (sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=\"indexdb/file\",cluster=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type=\"indexdb/file\",cluster=~\"$cluster\"}) without(type))))) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"min ETA","range":true,"refId":"A"}],"title":"Storage full ETA ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"max"},"properties":[{"id":"color","value":{"fixedColor":"#C4162A","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":41},"id":144,"links":[{"targetBlank":true,"title":"Related discussion","url":"https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max(max_over_time(vm_concurrent_insert_current{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"","legendFormat":"current","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"min(vm_concurrent_insert_capacity{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"max","range":true,"refId":"B"}],"title":"Concurrent flushes on disk ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"How many data points are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `5*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every 5 seconds. The index datapoints value in general is much lower","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"pending index entries"},"properties":[{"id":"unit","value":"none"},{"id":"decimals","value":3}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":41},"id":34,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"vm_pending_rows{job=~\"$job\",instance=~\"$instance\",type=\"storage\",cluster=~\"$cluster\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{job}} - pending datapoints","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"vm_pending_rows{job=~\"$job\",instance=~\"$instance\",type=\"indexdb\",cluster=~\"$cluster\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{job}} - pending index entries","range":true,"refId":"B"}],"title":"Pending datapoints ","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"The number of on-going merges in storage nodes.  It is expected to have high numbers for `storage/small` metric.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":49},"id":62,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_active_merges{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,type,cluster)","legendFormat":"{{job}} - {{type}}","range":true,"refId":"A"}],"title":"Active merges ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"The number of rows merged per second by storage nodes.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":49},"id":64,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_rows_merged_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,type,cluster)","legendFormat":"{{job}} - {{type}}","range":true,"refId":"A"}],"title":"Merge speed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"links":[{"targetBlank":true,"title":"Drilldown","url":"/d/wNf0q_kZk?viewPanel=153&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":57},"id":145,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster) / (sum(vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster) + sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster))) by(cluster)","format":"time_series","intervalFactor":1,"legendFormat":"max","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"min(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster) / (sum(vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster) + sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster))) by(cluster)","format":"time_series","intervalFactor":1,"legendFormat":"min","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"quantile(0.5, sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster) / (sum(vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster) + sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster))) by(cluster)","format":"time_series","intervalFactor":1,"legendFormat":"median","range":true,"refId":"C"}],"title":"Disk space usage %  ($instance)","type":"timeseries"},{"datasource":{"uid":"$ds"},"description":"The max number of data parts of LSM tree across all storage nodes in the last [partition](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage).\nIncrease in number of parts (the hard limit is 512) is an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage (hot data)\n* `storage/big` -  small parts gradually merged into bigger parts (cold data)","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":57},"id":36,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_parts{job=~\"$job\",instance=~\"$instance\",type=~\"indexdb.*\",cluster=~\"$cluster\"}) by(job,type,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{job}} - {{type}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(vm_last_partition_parts{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,type,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{job}} - {{type}}","range":true,"refId":"B"}],"title":"LSM parts max by type ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the percentage of used disk space by type: datapoints or indexdb. Normally, indexdb takes much less space comparing to datapoints. But with high [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) the size of the indexdb could grow significantly.\n\nThe sum of the % can be > 100% since panel shows max % per-job and per-instance. It means different instance can have different ratio between datapoints and indexdb size.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"line"}},"links":[{"targetBlank":true,"title":"Drilldown","url":"/d/wNf0q_kZk?viewPanel=152&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":65},"id":146,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=~\"indexdb.*\",cluster=~\"$cluster\"}) by(job,instance,cluster) / sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster)) by(cluster)","format":"time_series","intervalFactor":1,"legendFormat":"indexdb","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",cluster=~\"$cluster\"}) by(job,instance,cluster) / sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster)) by(cluster)","format":"time_series","intervalFactor":1,"legendFormat":"datapoints","range":true,"refId":"B"}],"title":"Disk space usage % by type  ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows amount of on-disk space occupied by data points and the remaining disk space at `-storageDataPath`. Calculation accounts for -storage.minFreeDiskSpaceBytes","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":65},"id":53,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}} - Used (datapoints)","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}} - Free","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=~\"indexdb.*\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}} - Used (index)","range":true,"refId":"C"}],"title":"Disk space usage - datapoints","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows how many datapoints are in the storage and what is average disk usage per datapoint.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"bytes-per-datapoint"},"properties":[{"id":"unit","value":"bytes"},{"id":"decimals","value":2}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":73},"id":30,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}} - total datapoints","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}} - bytes-per-datapoint","range":true,"refId":"B"}],"title":"Datapoints","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"How many data points are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `5*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every 5 seconds. The index datapoints value in general is much lower.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"pending index entries"},"properties":[{"id":"unit","value":"none"},{"id":"decimals","value":3}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":73},"id":147,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(max_over_time(vm_pending_rows{job=~\"$job\",instance=~\"$instance\",type=\"storage\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","format":"time_series","intervalFactor":1,"legendFormat":"pending datapoints","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(max_over_time(vm_pending_rows{job=~\"$job\",instance=~\"$instance\",type=\"indexdb\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","format":"time_series","intervalFactor":1,"legendFormat":"pending index entries","range":true,"refId":"B"}],"title":"Pending datapoints ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps and how many series were ignored on insertion or during scrape due to too long label names, values and excessive labels count.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":81},"id":58,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vm_rows_ignored_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1h])) by(instance,reason,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}} - {{reason}}","range":true,"refId":"A"}],"title":"Rows ignored for last 1h","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per queried series.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":81},"id":103,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,vmrange,cluster))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Datapoints read per series","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. However, these two actions are separated and controlled via different concurrency limiters. The `max` on this panel can't be changed and always equal to number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU.\n\n","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"max"},"properties":[{"id":"color","value":{"fixedColor":"#C4162A","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":89},"id":59,"options":{"legend":{"calcs":["mean","lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max_over_time(vm_concurrent_insert_capacity{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}} - max","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_concurrent_insert_current{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(instance,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}} - current","range":true,"refId":"B"}],"title":"Concurrent flushes on disk {$instance}","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) scanner per query.\n\nThis number can exceed number of RowsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":89},"id":105,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,vmrange,cluster))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Datapoints scanned per query","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"99th percentile of number of series read per query.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":97},"id":99,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,vmrange,cluster))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Series read per query","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per query.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":97},"id":122,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,vmrange,cluster))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Datapoints read per query","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows number of existing [snapshots](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots).\n\nRecently made snapshots do not occupy disk space. But with time, as snapshots become older, they start to occupy more and more disk space.\n\nIt is recommended deleting old snapshots when they are no longer needed to free up disk space.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":105},"id":130,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"vm_snapshots{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}","interval":"","legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Number of snapshots","type":"timeseries"}],"title":"Storage","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":40},"id":138,"panels":[{"fieldConfig":{"defaults":{},"overrides":[]},"gridPos":{"h":2,"w":17,"x":0,"y":427},"id":139,"options":{"code":{"language":"plaintext","showLineNumbers":false,"showMiniMap":false},"content":"Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance, or a single instance in cases like memory breakdown.","mode":"markdown"},"pluginVersion":"12.2.0","transparent":true,"type":"text"},{"datasource":{"type":"prometheus","uid":"${ds}"},"description":"Major page faults occur when required data is not in physical memory and must be loaded from disk (storage or swap). It usually happens with:\n- insufficient physical memory\n- active use of swap space\n- file mapping access\n\nIt may cause significant performance slowdown if sustained high.\n\nPossible workaround:\n- Extend RAM to scale up\n- Reduce swap usage by adjusting vm.swappiness","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":429},"id":140,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"sum(rate(process_major_pagefaults_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster) > 0","legendFormat":"{{instance}} ({{job}})","range":true,"refId":"A"}],"title":"Major page faults rate","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds}"},"description":"This panel breaks down memory usage by type. It’s intended for inspecting a single instance.\n\nHow to use:\n- Start from the high-level RSS panel.\n- Identify an instance with unexpected or abnormal memory growth.\n- Filter to that instance to inspect the detailed breakdown.\n\nInterpretation\n- A steadily rising Go Heap usually indicates a memory leak. Collect pprof memory profile.\n- A growing Go Stack commonly points to a goroutine leak.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":429},"id":141,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"max(((go_memstats_gc_sys_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} + go_memstats_mspan_sys_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) + go_memstats_other_sys_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) + go_memstats_mcache_sys_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","legendFormat":"Go: Other","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"max(go_memstats_stack_inuse_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","instant":false,"legendFormat":"Go: Stack","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"max(go_memstats_heap_inuse_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} + (go_memstats_heap_idle_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} - go_memstats_heap_released_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"})) by(cluster)","instant":false,"legendFormat":"Go: Heap","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"max(go_memstats_heap_released_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","instant":false,"legendFormat":"Go: Heap Reserved","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"max(sum(vm_cache_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster)) by(cluster)","instant":false,"legendFormat":"Mmap: VM Cache","range":true,"refId":"E"},{"datasource":{"type":"prometheus","uid":"${ds}"},"editorMode":"code","expr":"max(process_resident_memory_file_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","instant":false,"legendFormat":"File cache","range":true,"refId":"F"}],"title":"Memory usage breakdown","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":437},"id":148,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{instance}} ({{job}})","range":true,"refId":"A"}],"title":"RSS memory usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":437},"id":149,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max_over_time(process_resident_memory_anon_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{instance}} ({{job}})","range":true,"refId":"A"}],"title":"RSS anonymous memory usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the approx time needed to reach 100% of allowed disk capacity based on the following params:\n* free disk space (after -storage.minFreeDiskSpaceBytes);\n* row ingestion rate;\n* compression.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":445},"id":150,"options":{"legend":{"calcs":["mean","lastNotNull","min"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"((vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) / ignoring(path) (((rate(vm_rows_added_to_storage_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1d]) - sum(rate(vm_deduplicated_samples_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1d])) without(type)) * (sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",cluster=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",cluster=~\"$cluster\"}) without(type))) + (rate(vm_new_timeseries_created_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1d]) * (sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=\"indexdb/file\",cluster=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type=\"indexdb/file\",cluster=~\"$cluster\"}) without(type))))) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}}","range":true,"refId":"A"}],"title":"Storage full ETA ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":445},"id":151,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}} ({{job}})","range":true,"refId":"A"}],"title":"CPU usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"percent"},"thresholdsStyle":{"mode":"line"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":453},"id":152,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=~\"indexdb.*\",cluster=~\"$cluster\"}) by(job,instance,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{job}}:{{instance}} (indexdb)","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",cluster=~\"$cluster\"}) by(job,instance,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"{{job}}:{{instance}} (datapoints)","range":true,"refId":"B"}],"title":"Disk space usage by type ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":453},"id":153,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster)","format":"time_series","intervalFactor":1,"legendFormat":"","range":true,"refId":"A"}],"title":"Disk space usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"bars","fillOpacity":100,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Source Code","url":"https://l2s.victoriametrics.com/?app_version=${__field.labels.app_version}&location=${__field.labels.location}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":461},"id":154,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\",level!=\"info\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,level,app_version,location,cluster) > 0","format":"time_series","interval":"5m","intervalFactor":1,"legendFormat":"{{instance}} ({{job}}) - {{level}}: {{location}}","range":true,"refId":"A"}],"title":"Logging rate","type":"timeseries"}],"title":"Drilldown","type":"row"}],"preload":false,"refresh":"","schemaVersion":42,"tags":["victoriametrics","vm-k8s-stack"],"templating":{"list":[{"current":{"text":"prometheus","value":"P1809F7CD0C75ACF3"},"name":"ds","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"current":{"text":"vmsingle-homelab","value":"vmsingle-homelab"},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{version=~\"victoria-metrics-.*\",cluster=~\"$cluster\"}, job)","name":"job","options":[],"query":{"query":"label_values(vm_app_version{version=~\"victoria-metrics-.*\",cluster=~\"$cluster\"}, job)","refId":"VictoriaMetrics-job-Variable-Query"},"refresh":1,"regex":"","regexApplyTo":"value","type":"query"},{"current":{"text":"v1.132.0","value":"v1.132.0"},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}, version)","hide":2,"name":"version","options":[],"query":{"query":"label_values(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}, version)","refId":"VictoriaMetrics-version-Variable-Query"},"refresh":1,"regex":"/.*-(?:tags|heads)-(.*)-(?:0|dirty)-.*/","regexApplyTo":"value","sort":2,"type":"query"},{"allValue":".*","current":{"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{job=~\"$job\",cluster=~\"$cluster\"}, instance)","includeAll":true,"multi":true,"name":"instance","options":[],"query":{"query":"label_values(vm_app_version{job=~\"$job\",cluster=~\"$cluster\"}, instance)","refId":"VictoriaMetrics-instance-Variable-Query"},"refresh":1,"regex":"","regexApplyTo":"value","type":"query"},{"baseFilters":[],"datasource":{"type":"prometheus","uid":"$ds"},"filters":[],"name":"adhoc","type":"adhoc"},{"allValue":".*","datasource":{"type":"prometheus"},"hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","query":".*","type":"constant"}]},"time":{"from":"now-3h","to":"now"},"timepicker":{"refresh_intervals":["10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone":"utc","title":"VictoriaMetrics - single-node","uid":"wNf0q_kZk","version":1,"weekStart":""}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-victoriametrics-vmagent
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  victoriametrics-vmagent.json: | 
    {"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},{"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"expr":"sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(short_version,cluster) unless sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} offset $__interval) by(short_version,cluster)","hide":true,"iconColor":"dark-blue","name":"version","textFormat":"{{short_version}}","titleFormat":"Version change"},{"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"expr":"sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__interval])) by(job,instance,cluster)","hide":false,"iconColor":"dark-yellow","name":"restarts","textFormat":"{{job}}:{{instance}} restarted"}]},"description":"Overview for VictoriaMetrics vmagent v1.117.0 or higher","editable":false,"fiscalYearStartMonth":0,"graphTooltip":1,"id":3,"links":[{"icon":"doc","tags":[],"targetBlank":true,"title":"Docs","type":"link","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/"},{"asDropdown":false,"icon":"question","includeVars":false,"keepTime":false,"tags":[],"targetBlank":true,"title":"Troubleshooting","tooltip":"","type":"link","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting"},{"icon":"info","tags":[],"targetBlank":true,"title":"Community support","type":"link","url":"https://victoriametrics.com/support/community-support/"},{"asDropdown":false,"icon":"bolt","includeVars":false,"keepTime":false,"tags":[],"targetBlank":true,"title":"Enterprise support","tooltip":"","type":"link","url":"https://victoriametrics.com/support/enterprise-support/"},{"icon":"info","tags":[],"targetBlank":true,"title":"Releases","type":"link","url":"https://github.com/VictoriaMetrics/VictoriaMetrics/releases"}],"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":105,"title":"Stats","type":"row"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) scraped from configured targets.","fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]}},"overrides":[]},"gridPos":{"h":3,"w":6,"x":0,"y":1},"id":103,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Samples scraped/s","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of targets scraped per second.","fieldConfig":{"defaults":{"decimals":1,"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]}},"overrides":[]},"gridPos":{"h":3,"w":6,"x":6,"y":1},"id":134,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_promscrape_scrapes_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Targets scraped/s","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Persistent queue size shows size of pending samples in bytes which hasn't been flushed to remote storage yet. \nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.","fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":10485760}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":12,"y":1},"id":56,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"expr":"sum(vmagent_remotewrite_pending_data_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"","refId":"A"}],"title":"Persistent queue size","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Total number of available CPUs for selected vmagents. ","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":18,"y":1},"id":152,"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(vm_available_cpu_cores{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Available CPU","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of ingested [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)","fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]}},"overrides":[]},"gridPos":{"h":3,"w":6,"x":0,"y":4},"id":102,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vmagent_rows_inserted_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Samples ingested/s","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows total number of all configured scrape targets in state `up` or `down`.\n\nSee `http://vmagent-host:8429/targets` to get list of all targets. \n","fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]}},"overrides":[{"matcher":{"id":"byName","options":"down"},"properties":[{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":1}]}}]}]},"gridPos":{"h":3,"w":6,"x":6,"y":4},"id":72,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_promscrape_targets{job=~\"$job\",instance=~\"$instance\",status=\"up\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"up","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(vm_promscrape_targets{job=~\"$job\",instance=~\"$instance\",status=\"down\",cluster=~\"$cluster\"}) by(cluster)","hide":false,"interval":"","legendFormat":"down","range":true,"refId":"B"}],"title":"Scrape targets","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows number of generated  error messages in  logs over last 30m. Non-zero value may be a sign of connectivity or misconfiguration errors.","fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":1}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":12,"y":4},"id":16,"links":[{"targetBlank":true,"title":"Troubleshooting","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting"}],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"expr":"sum(increase(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\",level!=\"info\",cluster=~\"$cluster\"}[30m])) by(cluster)","interval":"","legendFormat":"","refId":"A"}],"title":"Log messages (30m)","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Total size of available memory for selected vmagents.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":3,"w":6,"x":18,"y":4},"id":153,"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(cluster)","format":"time_series","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Available memory","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false,"minWidth":50},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"Time"},"properties":[{"id":"custom.hideFrom.viz","value":true}]},{"matcher":{"id":"byName","options":"Value"},"properties":[{"id":"displayName","value":"Count"}]}]},"gridPos":{"h":5,"w":8,"x":0,"y":7},"id":101,"options":{"cellHeight":"sm","showHeader":true},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,short_version,cluster)","format":"table","instant":true,"range":false,"refId":"A"}],"type":"table"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"stepAfter","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":5,"w":16,"x":8,"y":7},"id":13,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(up{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","instant":false,"interval":"","legendFormat":"__auto","refId":"A"}],"title":"Uptime","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":12},"id":24,"title":"Overview","type":"row"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows in/out [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target.\n\nClick on the line and choose Drilldown to show CPU usage per instance\n","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"title":"Drilldown","url":"/d/G7Z9GzMGz?viewPanel=123&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/out .*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":13},"id":5,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate({__name__=~\"vmagent_rows_inserted_total|vm_promscrape_scraped_samples_sum\",job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","hide":false,"interval":"","legendFormat":"in {{job}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","interval":"","legendFormat":"out {{job}}","range":true,"refId":"B"}],"title":"Samples rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the persistent queue size of pending samples in bytes >2MB which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\nClick on the line and choose Drilldown to show the persistent queue size per instance.\n","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Drilldown - Persistent queue size","url":"/d/G7Z9GzMGz?viewPanel=125&var-url=${__field.labels.url}&var-ds=$ds&var-instance=$instance&var-job=$job&${__url_time_range}"},{"targetBlank":true,"title":"Drilldown - Persistent queue full ETA","url":"/d/G7Z9GzMGz?viewPanel=161&var-url=${__field.labels.url}&var-ds=$ds&var-instance=$instance&var-job=$job&${__url_time_range}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":13},"id":17,"links":[{"title":"Troubleshooting","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(vmagent_remotewrite_pending_data_bytes{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}) by(job,url,cluster) > 2e6","interval":"","legendFormat":"{{job}} => {{url}}","range":true,"refId":"A"}],"title":"Persistent queue size  ($instance) to ($url)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"bars","fillOpacity":100,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Drilldown","url":"/d/G7Z9GzMGz?viewPanel=162&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":21},"id":107,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\",level!=\"info\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,level,cluster) > 0","format":"time_series","hide":false,"interval":"5m","intervalFactor":1,"legendFormat":"{{job}} - {{level}}","range":true,"refId":"A"}],"title":"Logging rate","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows rate of dropped [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) from persistent queue. vmagent drops samples from queue if in-memory and on-disk queues are full and it is unable to flush them to remote storage.\nThe max size of on-disk queue is configured by `-remoteWrite.maxDiskUsagePerURL` flag.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":21},"id":49,"links":[{"targetBlank":true,"title":"Troubleshooting","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_persistentqueue_bytes_dropped_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(path,cluster) > 0","interval":"","legendFormat":"{{ path }}","range":true,"refId":"A"}],"title":"Persistent queue dropped rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of requests served by vmagent HTTP server.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":29},"id":15,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vmagent_http_requests_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,path,cluster) > 0","interval":"","legendFormat":"","range":true,"refId":"A"}],"title":"Requests rate  ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Errors rate shows rate for multiple metrics that track possible errors in vmagent, such as network or parsing errors.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":29},"id":69,"links":[{"targetBlank":true,"title":"Troubleshooting","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vmagent_http_request_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,protocol,cluster) > 0","interval":"","legendFormat":"requests:{{protocol}} ({{job}})","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_protoparser_read_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,type,cluster) > 0","interval":"","legendFormat":"parse: {{type}} ({{job}})","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_ingestserver_request_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,type,cluster) > 0","interval":"","legendFormat":"ingest: {{type}} ({{job}})","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_protoparser_unmarshal_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,type,cluster) > 0","interval":"","legendFormat":"unmarshal: {{type}} ({{job}})","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_promscrape_dial_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) > 0","interval":"","legendFormat":"scrape dial ({{job}})","range":true,"refId":"E"}],"title":"Errors rate  ($instance)","type":"timeseries"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":37},"id":45,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"description":"How much resident memory (RAM) the process is actually using, compared to its allowed container or system limit.\n\n- Good: Below 70% most of the time, maybe spiking a bit under load.\n- Bad: Above 90% for more than 5 minutes = risk of out-of-memory (OOM) kill.\n\nClick on the line and choose Drilldown to show memory usage per instance","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Drilldown","url":"/d/G7Z9GzMGz?viewPanel=117&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":141},"id":111,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max(max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"RSS memory % usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Drilldown","url":"/d/G7Z9GzMGz?viewPanel=119&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":141},"id":157,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(instance,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"CPU % usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%\n\nClick on the line and choose Drilldown to show memory usage per instance","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":196},"id":155,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(max_over_time(process_resident_memory_anon_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(instance,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"RSS anonymous memory % usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). Helps troubleshoot high CPU usage or throttling.\nLower is better.\n\n- waiting: The percentage of time at least one task in the process was ready to run (runnable) but couldn't get scheduled on the CPU.\n- stalled: The percentage of time all tasks in the process (except idle ones) were unable to get CPU time — a full CPU stall.\n\nIf there's a CPU burst, it's normal to see waiting or stalled > 1%. It only becomes a concern if it consistently climbs above 5–10% and aligns with latency spikes or GC slowdowns.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":196},"id":158,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_cpu_waiting_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{instance}} - waiting","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_cpu_stalled_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - stalled","range":true,"refId":"B"}],"title":"CPU pressure","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\nLower is better.\n\n- waiting: Time fraction where at least one thread was blocked on memory.\n- stalled: Time fraction where every thread was blocked on memory (severe pressure).\n\nIf queries slow down and both series spike, the host is likely limited by RAM or I/O throughput.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":204},"id":156,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_memory_waiting_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{instance}} - waiting","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_memory_stalled_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - stalled","range":true,"refId":"B"}],"title":"Memory pressure","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data.\n\nClick on the line and choose Drilldown to show CPU usage per instance","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Drilldown","url":"/d/G7Z9GzMGz?viewPanel=121&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}"}],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"read"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":204},"id":81,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_storage_read_bytes_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"read {{job}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_storage_written_bytes_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"write {{job}}","range":true,"refId":"B"}],"title":"Disk writes/reads ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":212},"id":39,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(max_over_time(go_goroutines{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Goroutines ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows IO pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\nThe lower the better.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":212},"id":159,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_io_waiting_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{instance}} - waiting","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_io_stalled_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - stalled","range":true,"refId":"B"}],"title":"IO pressure","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":220},"id":41,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(max_over_time(process_num_threads{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Threads ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Network usage shows the bytes rate for data accepted by vmagent and pushed via remotewrite protocol.\nDiscrepancies are possible because of different protocols used for ingesting, scraping and writing data.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"bps"},"overrides":[{"matcher":{"id":"byRegexp","options":"/out.*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":220},"id":7,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"(sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) * 8) + (sum(rate(vm_promscrape_conn_bytes_read_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) * 8)","interval":"","legendFormat":"in {{job}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) * 8","interval":"","legendFormat":"out {{job}}","range":true,"refId":"B"}],"title":"Network  usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ ","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":228},"id":135,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(rate(go_gc_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"CPU spent on GC ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of            insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":0.1}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":228},"id":149,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,le,cluster))) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Go scheduling latency","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":236},"id":154,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(go_memstats_alloc_bytes_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Memory allocations rate","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Panel shows the percentage of open file descriptors in the OS per instance.\nReaching the limit of open files (100%) can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":5,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":236},"id":83,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(max_over_time(process_open_fds{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / process_max_fds{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Open FDs usage % ($instance)","type":"timeseries"}],"title":"Resource usage","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":38},"id":94,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows top 10 jobs by the number of new series registered by vmagent over the 5min range. These jobs generate the most of the churn rate.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":142},"id":92,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"topk(10, sum(sum_over_time(scrape_series_added[5m])) by(job,cluster)) by(cluster) > 0","interval":"","legendFormat":"{{ job }}","range":true,"refId":"A"}],"title":"Top 10 jobs by newly added series","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows top 10 targets by the number of new series registered by vmagent over the 5min range. These instances generate the most of the churn rate.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":142},"id":95,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"topk(10, sum(sum_over_time(scrape_series_added[5m])) by(job,instance,cluster)) by(cluster) > 0","interval":"","legendFormat":"{{job}}-{{instance}}","range":true,"refId":"A"}],"title":"Top 10 targets by newly added series","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows write saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMax":2,"barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"transparent","value":0},{"color":"red","value":0.9}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":150},"id":98,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max(rate(vm_persistentqueue_write_duration_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Persistent queue write saturation ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows read saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMax":2,"barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"transparent","value":0},{"color":"red","value":0.9}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":150},"id":99,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max(rate(vm_persistentqueue_read_duration_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Persistent queue read saturation ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of dropped data blocks in cases when remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses.\n\nSee https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":158},"id":79,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vmagent_remotewrite_packets_dropped_total{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,url,cluster) > 0","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Data blocks dropped ($instance) to ($url)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of dropped [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) due to relabeling. \nMetric tracks drops for `-remoteWrite.relabelConfig` configuration only.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":158},"id":18,"links":[{"targetBlank":true,"title":"Relabeling","url":"https://docs.victoriametrics.com/victoriametrics/relabeling/"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vmagent_remotewrite_relabel_metrics_dropped_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,url,cluster) > 0","interval":"","legendFormat":"__auto","range":true,"refId":"B"}],"title":"Rows dropped by relabeling ($instance) to ($url)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of parsed datapoints from write or scrape requests.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":166},"id":127,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vm_protoparser_rows_read_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,type,cluster) > 0","interval":"","legendFormat":"{{ type }} ({{job}})","range":true,"refId":"A"}],"title":"Datapoints rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Tracks the rate of dropped invalid rows because of errors while unmarshaling write requests. The exact errors messages will be printed in logs.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":166},"id":50,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vm_rows_invalid_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,type,cluster) > 0","interval":"","legendFormat":"{{type}} ({{job}})","range":true,"refId":"A"}],"title":"Invalid datapoints rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"filterable":true,"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"Value"},"properties":[{"id":"custom.hidden","value":true}]},{"matcher":{"id":"byName","options":"Time"},"properties":[{"id":"custom.hidden","value":true}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":174},"id":129,"options":{"cellHeight":"sm","filterable":true,"footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true,"showSearch":true,"sortBy":[{"desc":true,"displayName":"job"}]},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(flag{is_set=\"true\",job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,name,value,cluster)","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],"title":"Non-default flags","transformations":[{"id":"groupBy","options":{"fields":{"instance":{"aggregations":[]},"job":{"aggregations":[],"operation":"groupby"},"name":{"aggregations":[],"operation":"groupby"},"value":{"aggregations":[],"operation":"groupby"}}}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMin":0,"barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"stepAfter","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":12,"y":174},"id":150,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) > 0) by(job,cluster)","format":"time_series","instant":false,"legendFormat":"{{job}}","refId":"A"}],"title":"Restarts ($job)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows how many samples were ignored on insertion due to various reasons. See the reason of rejection application logs.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":181},"id":151,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(increase(vm_rows_ignored_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1h])) by(reason,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Rows ignored for last 1h ($instance)","type":"timeseries"}],"title":"Troubleshooting","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":39},"id":28,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":0,"y":4209},"id":48,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(vm_promscrape_targets{job=~\"$job\",instance=~\"$instance\",status=\"up\",cluster=~\"$cluster\"}) by(job,type,cluster) > 0","format":"time_series","interval":"","legendFormat":"{{type}} ({{job}})","range":true,"refId":"A"}],"title":"Scrape targets UP(By Type)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":12,"y":4209},"id":76,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(vm_promscrape_targets{job=~\"$job\",instance=~\"$instance\",status=\"down\",cluster=~\"$cluster\"}) by(job,type,cluster) > 0","format":"time_series","interval":"","legendFormat":"{{type}} ({{job}})","range":true,"refId":"A"}],"title":"Scrape targets DOWN(By Type)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":0,"y":4216},"id":132,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(vm_promscrape_scrape_pool_targets{job=~\"$job\",instance=~\"$instance\",status=\"up\",cluster=~\"$cluster\"}) by(job,scrape_job,cluster) > 0","format":"time_series","interval":"","legendFormat":"{{job}}: {{scrape_job}}","range":true,"refId":"A"}],"title":"Scrape targets UP(By Job)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":12,"y":4216},"id":133,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(vm_promscrape_scrape_pool_targets{job=~\"$job\",instance=~\"$instance\",status=\"down\",cluster=~\"$cluster\"}) by(job,scrape_job,cluster) > 0","format":"time_series","interval":"","legendFormat":"{{job}}: {{scrape_job}}","range":true,"refId":"A"}],"title":"Scrape targets DOWN(By Job)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of scrapes per second.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":4223},"id":20,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vm_promscrape_scrapes_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Scrape rate  ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of datapoints scraped per second.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":4223},"id":126,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Scraped datapoints rate  ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":4231},"id":46,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,vmrange,cluster))) by(job,cluster)","format":"time_series","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Scrape response size 0.99 quantile ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":4231},"id":148,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,vmrange,cluster))) by(job,cluster)","format":"time_series","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Scrape duration 0.99 quantile ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":4239},"id":31,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_promscrape_scrapes_failed_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) > 0","interval":"","legendFormat":"scrapes failed ({{job}})","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_promscrape_scrapes_timed_out_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) > 0","interval":"","legendFormat":"timeouts ({{job}})","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_promscrape_scrapes_gunzip_failed_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) > 0","interval":"","legendFormat":"gunzip fails ({{job}})","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_promscrape_dial_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) > 0","interval":"","legendFormat":"dial fails ({{job}})","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_promscrape_max_scrape_size_exceeded_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) > 0","hide":false,"interval":"","legendFormat":"max scrape size exceeded ({{job}})","range":true,"refId":"E"}],"title":"Scrape fails  ($instance)","type":"timeseries"}],"title":"Scraping","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":40},"id":71,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of write requests served by ingestserver (UDP, TCP connections) and HTTP server.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":3931},"id":73,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vm_ingestserver_requests_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,type,net,cluster) > 0","interval":"","legendFormat":"{{net}}: {{ type }} ({{job}})","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vmagent_http_requests_total{job=~\"$job\",instance=~\"$instance\",protocol!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,protocol,cluster) > 0","interval":"","legendFormat":"{{ protocol }}: http ({{job}})","range":true,"refId":"B"}],"title":"Requests rate  ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of rows ingested in vmagent via push protocols.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":3931},"id":131,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vmagent_rows_inserted_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,type,cluster) > 0","interval":"","legendFormat":"{{ type }} ({{job}})","range":true,"refId":"A"}],"title":"Rows rate  ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows how many concurrent inserts are taking place.\n\nIf the number of concurrent inserts hitting the `limit` or is close to the `limit` constantly - it might be a sign of a resource shortage.\n\n If vmagent's CPU usage and remote write connection saturation are at normal level, it might be that `-maxConcurrentInserts` cmd-line flag need to be increased.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":4207},"id":130,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max_over_time(vm_concurrent_insert_current{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{instance}} ({{job}})","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"min(vm_concurrent_insert_capacity{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","interval":"","legendFormat":"limit ({{job}})","range":true,"refId":"B"}],"title":"Concurrent inserts ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of write errors in ingestserver (UDP, TCP connections) and HTTP server.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":4207},"id":77,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"exemplar":true,"expr":"sum(rate(vm_ingestserver_request_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(type,net,cluster) > 0","interval":"","legendFormat":"{{ type }} ({{net}})","refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"exemplar":true,"expr":"sum(rate(vmagent_http_request_errors_total{job=~\"$job\",instance=~\"$instance\",protocol!=\"\",cluster=~\"$cluster\"}[$__rate_interval])) by(protocol,cluster) > 0","interval":"","legendFormat":"{{ protocol }} (http)","refId":"B"}],"title":"Error rate  ($instance)","type":"timeseries"}],"title":"Ingestion","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":41},"id":136,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of matched samples by the aggregation rule. \n\nThe more samples is matched, the more work this aggregation rule does. The matching rule is specified via `match` param.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). ","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMin":0,"barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":4254},"id":146,"options":{"legend":{"calcs":["min","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_streamaggr_matched_samples_total{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}[$__rate_interval])) without(instance,pod) > 0","instant":false,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Matched samples ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"The rate of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples ","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMin":0,"barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":4254},"id":143,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_streamaggr_ignored_samples_total{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}[$__rate_interval]) > 0) without(instance,pod)","instant":false,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Ignored samples ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of produced samples by the aggregation rule. \n\nNumber of produced samples depend on params like `by`, `without`, `interval`, etc.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). ","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMin":0,"barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":4262},"id":147,"options":{"legend":{"calcs":["min","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vm_streamaggr_output_samples_total{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}[$__rate_interval])) without(instance,pod) > 0","instant":false,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Produced samples ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMin":0,"barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":-5,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":4262},"id":139,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}[$__rate_interval]) > 0","instant":false,"legendFormat":"aggregation: {{url}} ({{job}}): {{path}}:{{position}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}[$__rate_interval]) > 0","hide":false,"instant":false,"legendFormat":"deduplication: {{url}} ({{job}}): {{path}}:{{position}}","range":true,"refId":"B"}],"title":"Flush timeouts ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the max lag between samples timestamps within one batch passed to the aggregation input.\n\nLower is better.\n\nToo high lag or lag exceeding the interval might be a sign that data was delayed before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.\n\nSee https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMin":0,"barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":4270},"id":142,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}[$__rate_interval])) without(instance,pod))","instant":false,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Samples lag 0.99 quantile ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMin":0,"barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":4270},"id":137,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"histogram_quantile(0.99, rate(vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}[$__rate_interval]))","instant":false,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Dedup flush duration 0.99 quantile ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the size of Label Compressor in number of entries.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMin":0,"barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byRegexp","options":"/bytes.*/"},"properties":[{"id":"custom.axisPlacement","value":"right"},{"id":"unit","value":"bytes"}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":4310},"id":141,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(vm_streamaggr_labels_compressor_items_count{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster)","hide":false,"instant":false,"legendFormat":"items: {{instance}} ({{job}})","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(vm_streamaggr_labels_compressor_size_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,cluster)","hide":false,"instant":false,"legendFormat":"bytes: {{instance}} ({{job}})","range":true,"refId":"B"}],"title":"Labels compressor ($instance)","type":"timeseries"}],"title":"Streaming aggregation","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":42},"id":58,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of requests to configured remote write endpoints by url and status code.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2259},"id":60,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vmagent_remotewrite_requests_total{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,url,status_code,cluster) > 0","interval":"","legendFormat":"","range":true,"refId":"A"}],"title":"Requests rate ($instance) to ($url)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the global rate for number of written bytes via remote write connections.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2259},"id":66,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) > 0","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Bytes write rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows requests retry rate by url. Number of retries is unlimited but protected with delays up to 1m between attempts.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2267},"id":61,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vmagent_remotewrite_retries_count_total{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}[$__rate_interval])) by(url,cluster) > 0","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Retry rate ($instance) to ($url)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows current number of established connections to remote write endpoints.\n\n","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2267},"id":65,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(max_over_time(vmagent_remotewrite_conns{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Connections ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the current limit usage of unique series over an hourly period. Vmagent will start to drop series once the limit is reached.\n\nPlease note, panel will be blank if `remoteWrite.maxHourlySeries` is not set.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"mappings":[],"max":100,"thresholds":{"mode":"absolute","steps":[{"color":"transparent","value":0},{"color":"red","value":0.9}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2275},"id":88,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max(vmagent_hourly_series_limit_current_series{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} / vmagent_hourly_series_limit_max_series{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster) * 100","interval":"","legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Hourly series limit","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows saturation of every connection to remote storage. If the threshold of 90% is reached, then the connection is saturated (busy or slow) by more than 90%, so vmagent won't be able to keep up and can start buffering data. \n\nThis usually means that `-remoteWrite.queues` command-line flag must be increased in order to increase the number of connections per each remote storage.\n","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"transparent","value":0},{"color":"red","value":0.9}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2275},"id":84,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max(rate(vmagent_remotewrite_send_duration_seconds_total{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}[$__rate_interval]) / vmagent_remotewrite_queues{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}) by(job,url,cluster)","interval":"","legendFormat":"","range":true,"refId":"A"}],"title":"Remote write connection saturation ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the current limit usage of unique series over a daily period. Vmagent will start to drop series once the limit is reached.\n\nPlease note, panel will be blank if `remoteWrite.maxDailySeries` is not set.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"mappings":[],"max":100,"thresholds":{"mode":"absolute","steps":[{"color":"transparent","value":0},{"color":"red","value":0.9}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2283},"id":90,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max(vmagent_daily_series_limit_current_series{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} / vmagent_daily_series_limit_max_series{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","interval":"","legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Daily series limit","type":"timeseries"}],"title":"Remote write","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":43},"id":113,"panels":[{"fieldConfig":{"defaults":{},"overrides":[]},"gridPos":{"h":2,"w":24,"x":0,"y":918},"id":115,"options":{"code":{"language":"plaintext","showLineNumbers":false,"showMiniMap":false},"content":"Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.","mode":"markdown"},"pluginVersion":"12.2.0","transparent":true,"type":"text"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":920},"id":119,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{instance}} ({{job}})","range":true,"refId":"A"}],"title":"CPU usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":920},"id":117,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])","interval":"","legendFormat":"{{instance}} ({{job}})","range":true,"refId":"A"}],"title":"RSS memory usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the persistent queue size of pending samples in bytes which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":928},"id":125,"links":[{"title":"Troubleshooting","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(vmagent_remotewrite_pending_data_bytes{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"}) by(instance,url,cluster)","interval":"","legendFormat":"{{instance}} => {{url}}","range":true,"refId":"A"}],"title":"Persistent queue size  ($instance) to ($url)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows in/out samples rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[{"matcher":{"id":"byRegexp","options":"/out .*/"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":928},"id":123,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate({__name__=~\"vm_promscrape_scraped_samples_sum|vmagent_rows_inserted_total\",job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","hide":false,"interval":"","legendFormat":"in {{instance}} {{job}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","interval":"","legendFormat":"out {{instance}} {{job}}","range":true,"refId":"B"}],"title":"Samples rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[{"matcher":{"id":"byName","options":"read"},"properties":[{"id":"custom.transform","value":"negative-Y"}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":936},"id":121,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_storage_read_bytes_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"read {{instance}} {{job}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_io_storage_written_bytes_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"write {{instance}} {{job}}","range":true,"refId":"B"}],"title":"Disk writes/reads ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows an estimated time until a persistent queue reaches its disk usage limit for a given instance and -remoteWrite.url.\n\nThe ETA is shown only when the pending queue size is actively growing, which typically indicates that remote write cannot keep up with ingestion (e.g. the remote endpoint is slow or unavailable).\n\nOnce the disk usage limit is reached, newly ingested metrics will be dropped.\n\nThe estimation is approximate and may fluctuate due to short-term changes in the queue growth rate.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":936},"id":161,"links":[{"title":"Troubleshooting","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"(sum(vm_persistentqueue_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,path,cluster) / (sum(deriv(vm_persistentqueue_bytes_pending{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[1m])) by(job,instance,path,cluster) > 0)) * on(job,instance,path,cluster) group_left(url) ((vmagent_remotewrite_pending_data_bytes{job=~\"$job\",instance=~\"$instance\",url=~\"$url\",cluster=~\"$cluster\"} * 0) + 1)","interval":"","legendFormat":"{{instance}} => {{url}}","range":true,"refId":"A"}],"title":"Persistent queue Full ETA ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"bars","fillOpacity":100,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","showValues":false,"spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[{"targetBlank":true,"title":"Source Code","url":"https://l2s.victoriametrics.com/?app_version=${__field.labels.app_version}&location=${__field.labels.location}"}],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":0},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1309},"id":162,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.2.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\",level!=\"info\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,level,app_version,location,cluster) > 0","format":"time_series","interval":"5m","intervalFactor":1,"legendFormat":"{{instance}} - {{level}}: {{location}}","range":true,"refId":"A"}],"title":"Logging rate","type":"timeseries"}],"title":"Drilldown","type":"row"}],"preload":false,"refresh":"","schemaVersion":42,"tags":["victoriametrics","vm-k8s-stack"],"templating":{"list":[{"current":{},"name":"ds","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{version=~\"^vmagent.*\",cluster=~\"$cluster\"}, job)","includeAll":true,"multi":true,"name":"job","options":[],"query":{"query":"label_values(vm_app_version{version=~\"^vmagent.*\",cluster=~\"$cluster\"}, job)","refId":"VictoriaMetrics-job-Variable-Query"},"refresh":1,"regex":"","type":"query"},{"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{job=~\"$job\",cluster=~\"$cluster\"}, instance)","includeAll":true,"multi":true,"name":"instance","options":[],"query":{"query":"label_values(vm_app_version{job=~\"$job\",cluster=~\"$cluster\"}, instance)","refId":"VictoriaMetrics-instance-Variable-Query"},"refresh":1,"regex":"","type":"query"},{"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vmagent_remotewrite_requests_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}, url)","description":"The remote write URLs","includeAll":true,"multi":true,"name":"url","options":[],"query":{"query":"label_values(vmagent_remotewrite_requests_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}, url)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","type":"query"},{"baseFilters":[],"datasource":{"type":"prometheus","uid":"$ds"},"filters":[],"name":"adhoc","type":"adhoc"},{"allValue":".*","datasource":{"type":"prometheus"},"hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","query":".*","type":"constant"}]},"time":{"from":"now-3h","to":"now"},"timepicker":{"refresh_intervals":["10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone":"utc","title":"VictoriaMetrics - vmagent","uid":"G7Z9GzMGz","version":1}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: vmks-victoriametrics-vmalert
  labels:
    grafana_dashboard: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  victoriametrics-vmalert.json: | 
    {"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"},{"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"expr":"sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(short_version,cluster) unless sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} offset $__interval) by(short_version,cluster)","hide":true,"iconColor":"dark-blue","name":"version","textFormat":"{{short_version}}","titleFormat":"Version change"},{"datasource":{"type":"prometheus","uid":"$ds"},"enable":true,"expr":"sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__interval])) by(job,instance,cluster)","hide":false,"iconColor":"dark-yellow","name":"restarts","textFormat":"{{job}}:{{instance}} restarted"}]},"description":"Overview for VictoriaMetrics vmalert v1.117.0 or higher","editable":false,"fiscalYearStartMonth":0,"graphTooltip":1,"id":3,"links":[{"icon":"doc","tags":[],"targetBlank":true,"title":"Docs","type":"link","url":"https://docs.victoriametrics.com/victoriametrics/vmalert/"},{"asDropdown":false,"icon":"question","includeVars":false,"keepTime":false,"tags":[],"targetBlank":true,"title":"Troubleshooting","tooltip":"","type":"link","url":"https://docs.victoriametrics.com/victoriametrics/vmalert/#troubleshooting"},{"icon":"info","tags":[],"targetBlank":true,"title":"Community support","type":"link","url":"https://victoriametrics.com/support/community-support/"},{"asDropdown":false,"icon":"bolt","includeVars":false,"keepTime":false,"tags":[],"targetBlank":true,"title":"Enterprise support","tooltip":"","type":"link","url":"https://victoriametrics.com/support/enterprise-support/"},{"icon":"info","tags":[],"targetBlank":true,"title":"Releases","type":"link","url":"https://github.com/VictoriaMetrics/VictoriaMetrics/releases"}],"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":47,"title":"Stats","type":"row"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows if the last configuration update was successful. \"Not Ok\" means there was an unsuccessful attempt to update the configuration due to some error. Check the log for details.","fieldConfig":{"defaults":{"mappings":[{"options":{"match":"null","result":{"color":"green","index":0,"text":"Ok"}},"type":"special"},{"options":{"from":1,"result":{"color":"red","index":1,"text":"Not Ok"},"to":999999},"type":"range"}],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"gridPos":{"h":3,"w":4,"x":0,"y":1},"id":6,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.0.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"count(vmalert_config_last_reload_successful{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} < 1) by(cluster)","interval":"","legendFormat":"","range":true,"refId":"A"}],"title":"Config update","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the total number of loaded alerting rules across selected instances and groups.","fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"gridPos":{"h":3,"w":5,"x":4,"y":1},"id":9,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.0.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"exemplar":false,"expr":"count(vmalert_alerting_rules_last_evaluation_samples{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"","refId":"A"}],"title":"Alerting rules","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the total number of loaded recording rules across selected instances and groups.","fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"gridPos":{"h":3,"w":5,"x":9,"y":1},"id":7,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.0.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"exemplar":false,"expr":"count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"}) by(cluster)","interval":"","legendFormat":"","refId":"A"}],"title":"Recording rules","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the total number of errors generated by recording/alerting rules for selected instances and groups.","fieldConfig":{"defaults":{"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":1}]}},"overrides":[]},"gridPos":{"h":3,"w":5,"x":14,"y":1},"id":8,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.0.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster) + sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"}[$__rate_interval])) by(cluster)","interval":"","legendFormat":"","range":true,"refId":"A"}],"title":"Errors","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows number of Recording Rules which produce no data.\n\n Usually it means that such rules are misconfigured, since they give no output during the evaluation.\nPlease check if rule's expression is correct and it is working as expected.","fieldConfig":{"defaults":{"mappings":[{"options":{"match":"null","result":{"index":1,"text":"0"}},"type":"special"}],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":1}]}},"overrides":[]},"gridPos":{"h":3,"w":5,"x":19,"y":1},"id":48,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},"pluginVersion":"12.0.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"} < 1) by(cluster)","interval":"","legendFormat":"","range":true,"refId":"A"}],"title":"No data errors","type":"stat"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"inspect":false,"minWidth":50},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"Time"},"properties":[{"id":"custom.hidden","value":true}]},{"matcher":{"id":"byName","options":"Value"},"properties":[{"id":"displayName","value":"Count"}]}]},"gridPos":{"h":4,"w":9,"x":0,"y":4},"id":45,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"12.0.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,short_version,cluster)","format":"table","instant":true,"range":false,"refId":"A"}],"type":"table"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"stepAfter","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":4,"w":15,"x":9,"y":4},"id":4,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"asc"}},"pluginVersion":"12.0.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(min_over_time(up{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","instant":false,"interval":"","legendFormat":"{{job}}","refId":"A"}],"title":"Uptime","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":8},"id":11,"title":"Overview ($instance)","type":"row"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of fired alerts by job.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":15,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.0.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vmalert_alerts_fired_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","interval":"","legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Alerts fired total ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Top $topk groups by evaluation duration. Shows groups that take the most of time during the evaluation across all instances.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":9},"id":23,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.0.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"topk($topk, max(sum(rate(vmalert_iteration_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"}[$__rate_interval]) / rate(vmalert_iteration_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,group,file,cluster)) by(job,group,file,cluster)) by(cluster)","interval":"","legendFormat":"({{job}}) {{group}}({{file}})","range":true,"refId":"A"}],"title":"Top $topk groups avg evaluation duration ($group)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows how many requests (executions) per second vmalert sends to the configured datasource.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":17},"id":24,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"12.0.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vmalert_execution_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","interval":"","legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Rules execution rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the error rate while executing configured rules. Non-zero value means there are some issues with existing rules. Check the logs to get more details.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":17},"id":25,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"12.0.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vmalert_execution_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) > 0","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Rules execution errors ($instance)","type":"timeseries"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":25},"id":43,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"description":"How much resident memory (RAM) the process is actually using, compared to its allowed container or system limit.\n\n- Good: Below 70% most of the time, maybe spiking a bit under load.\n- Bad: Above 90% for more than 5 minutes = risk of out-of-memory (OOM) kill.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":162},"id":37,"links":[{"targetBlank":true,"title":"Profiling","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Memory usage % ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the CPU usage percentage per vmalert instance. \nIf you think that usage is abnormal or unexpected pls file an issue and attach CPU profile if possible.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":162},"id":35,"links":[{"targetBlank":true,"title":"Profiling","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"CPU usage %($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":170},"id":65,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(max_over_time(process_resident_memory_anon_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(instance,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"RSS anonymous memory % usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the max number of CPU cores used  by a `job` and the corresponding limit.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":170},"id":56,"links":[{"targetBlank":true,"title":"Profiling","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{job}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"min(process_cpu_cores_available{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"limit ({{job}})","range":true,"refId":"B"}],"title":"CPU usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Amount of used RSS memory\n\nIf you think that usage is abnormal or unexpected, please file an issue and attach memory profile if possible.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":178},"id":57,"links":[{"targetBlank":true,"title":"Profiling","url":"https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling"}],"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","interval":"","legendFormat":"{{job}}","range":true,"refId":"A"}],"title":"Memory usage ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). Helps troubleshoot high CPU usage or throttling.\nLower is better.\n\n- waiting: The percentage of time at least one task in the process was ready to run (runnable) but couldn't get scheduled on the CPU.\n- stalled: The percentage of time all tasks in the process (except idle ones) were unable to get CPU time — a full CPU stall.\n\nIf there's a CPU burst, it's normal to see waiting or stalled > 1%. It only becomes a concern if it consistently climbs above 5–10% and aligns with latency spikes or GC slowdowns.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":178},"id":66,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_cpu_waiting_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{instance}} - waiting","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_cpu_stalled_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - stalled","range":true,"refId":"B"}],"title":"CPU pressure","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\nLower is better.\n\n- waiting: Time fraction where at least one thread was blocked on memory.\n- stalled: Time fraction where every thread was blocked on memory (severe pressure).\n\nIf queries slow down and both series spike, the host is likely limited by RAM or I/O throughput.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":186},"id":67,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_memory_waiting_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{instance}} - waiting","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(process_pressure_memory_stalled_seconds_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","hide":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}} - stalled","range":true,"refId":"B"}],"title":"Memory pressure","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":186},"id":41,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(go_goroutines{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Goroutines ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Panel shows the percentage of open file descriptors in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":3,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":194},"id":39,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"max(max_over_time(process_open_fds{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / process_max_fds{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Open FDs usage % ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of            insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":0.1}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":194},"id":61,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,le,cluster))) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Go scheduling latency","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ ","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":202},"id":59,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"max(rate(go_gc_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) / rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"CPU spent on GC ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line"}},"decimals":0,"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":210},"id":64,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"desc"}},"pluginVersion":"11.5.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(go_memstats_alloc_bytes_total{job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,instance,cluster)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"__auto","range":true,"refId":"A"}],"title":"Memory allocations rate","type":"timeseries"}],"title":"Resource usage","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":26},"id":62,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"filterable":true,"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"Value"},"properties":[{"id":"custom.hidden","value":true}]},{"matcher":{"id":"byName","options":"Time"},"properties":[{"id":"custom.hidden","value":true}]}]},"gridPos":{"h":7,"w":12,"x":0,"y":345},"id":50,"options":{"cellHeight":"sm","filterable":true,"footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true,"showSearch":true,"sortBy":[{"desc":true,"displayName":"job"}]},"pluginVersion":"10.4.2","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(flag{is_set=\"true\",job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}) by(job,instance,name,value,cluster)","format":"table","instant":true,"legendFormat":"__auto","range":false,"refId":"A"}],"title":"Non-default flags","transformations":[{"id":"groupBy","options":{"fields":{"instance":{"aggregations":["uniqueValues"],"operation":"aggregate"},"job":{"aggregations":[],"operation":"groupby"},"name":{"aggregations":[],"operation":"groupby"},"value":{"aggregations":[],"operation":"groupby"}}}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Missed evaluation means that group evaluation time takes longer than the configured evaluation interval. \nThis may result in missed alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency for such groups. See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups\n\nIf rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries.\"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"bars","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":12,"x":12,"y":345},"id":58,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vmalert_iteration_missed_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,group,file,cluster) > 0","interval":"1m","legendFormat":"({{job}}) {{group}}({{file}})","range":true,"refId":"A"}],"title":"Missed evaluations ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMin":0,"barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"stepAfter","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":352},"id":63,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.1.0","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval]) > 0) by(job,cluster)","format":"time_series","instant":false,"legendFormat":"{{job}}","refId":"A"}],"title":"Restarts ($job)","type":"timeseries"}],"title":"Troubleshooting","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":27},"id":17,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows top $topk current active (firing) alerting rules.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":370},"id":14,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"topk($topk, sum(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"}) by(job,group,file,alertname,cluster) > 0) by(cluster)","interval":"","legendFormat":"({{job}}) {{group}}.{{alertname}}({{file}})","range":true,"refId":"A"}],"title":"Top $topk active alerts ($group)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the events when rule execution resulted into an error. Check the logs for more details.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":370},"id":13,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,group,file,alertname,cluster) > 0","interval":"","legendFormat":"({{job}}) {{group}}.{{alertname}}({{file}})","range":true,"refId":"A"}],"title":"Errors ($group)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the current pending alerting rules per group.\nBy pending means the rule which remains active less than configured `for` parameter.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":378},"id":20,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Mean","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(vmalert_alerts_pending{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"}) by(job,group,file,alertname,cluster) > 0","interval":"","legendFormat":"({{job}}) {{group}}.{{alertname}}({{file}})","range":true,"refId":"A"}],"title":"Pending ($group)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the error rate for the attempts to send alerts to Alertmanager. If not zero it means there issues on attempt to send notification to Alertmanager and some alerts may be not delivered properly. Check the logs for more details.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":378},"id":32,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"exemplar":false,"expr":"sum(rate(vmalert_alerts_send_errors_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(instance,addr,cluster) > 0","interval":"","legendFormat":"{{instance}} => {{addr}}","refId":"A"}],"title":"Errors rate to Alertmanager","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows how many alerts are sent to Alertmanager per second. Only active alerts are sent.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":386},"id":26,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(rate(vmalert_alerts_sent_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,addr,cluster) > 0","interval":"","legendFormat":"{{job}} => {{addr}}","range":true,"refId":"A"}],"title":"Requests rate to Alertmanager by job ($group)","type":"timeseries"}],"title":"Alerting rules ($instance)","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":28},"id":28,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the top $topk recording rules which generate the most of [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples). Each generated sample is basically a time series which then ingested into configured remote storage. Rules with high numbers may cause the most pressure on the remote database and become a source of too high cardinality.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":385},"id":31,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"topk($topk, max(sum(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"}) by(job,instance,group,file,recording,cluster) > 0) by(job,group,file,recording,cluster)) by(cluster)","interval":"","legendFormat":"({{job}}) {{group}}.{{recording}}({{file}})","range":true,"refId":"A"}],"title":"Top $topk rules by produced samples ($group)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the rules which do not produce any [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) during the evaluation. Usually it means that such rules are misconfigured, since they give no output during the evaluation.\nPlease check if rule's expression is correct and it is working as expected.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":385},"id":33,"options":{"legend":{"calcs":["lastNotNull","max","mean"],"displayMode":"table","placement":"bottom","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.0.3","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"} < 1) by(job,group,file,recording,cluster)","interval":"","legendFormat":"({{job}}) {{group}}.{{recording}}({{file}})","range":true,"refId":"A"}],"title":"Rules with 0 produced samples ($group)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":393},"id":30,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.2.6","targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":false,"expr":"sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,group,file,recording,cluster) > 0","interval":"","legendFormat":"({{job}}) {{group}}.{{recording}}({{file}})","range":true,"refId":"A"}],"title":"Errors ($group)","type":"timeseries"}],"title":"Recording rules ($instance)","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":29},"id":55,"panels":[{"datasource":{"type":"prometheus","uid":"$ds"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":351},"id":52,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vmalert_remotewrite_sent_rows_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Datapoints send rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the number of datapoints dropped by vmalert while sending to the configured remote write URL. vmalert performs up to 5 retries before dropping the data. Check vmalert's error logs for the specific error message.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":351},"id":53,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","expr":"sum(rate(vmalert_remotewrite_dropped_rows_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) > 0","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Datapoints drop rate ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows current number of established connections to remote write endpoints.\n\n","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":378},"id":54,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(max_over_time(vmalert_remotewrite_conns{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster)","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Connections ($instance)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"$ds"},"description":"Shows the global rate for number of written bytes via remote write connections.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"links":[],"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":378},"id":60,"options":{"legend":{"calcs":["mean","lastNotNull","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"targets":[{"datasource":{"type":"prometheus","uid":"$ds"},"editorMode":"code","exemplar":true,"expr":"sum(rate(vmalert_remotewrite_conn_bytes_written_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}[$__rate_interval])) by(job,cluster) > 0","interval":"","legendFormat":"__auto","range":true,"refId":"A"}],"title":"Bytes write rate ($instance)","type":"timeseries"}],"title":"Remote write","type":"row"}],"preload":false,"refresh":"","schemaVersion":41,"tags":["victoriametrics","vm-k8s-stack"],"templating":{"list":[{"current":{"text":"VictoriaMetrics - cluster","value":"PAF93674D0B4E9963"},"name":"ds","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{version=~\"^vmalert.*\",cluster=~\"$cluster\"}, job)","includeAll":true,"multi":true,"name":"job","options":[],"query":{"query":"label_values(vm_app_version{version=~\"^vmalert.*\",cluster=~\"$cluster\"}, job)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","type":"query"},{"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vm_app_version{job=~\"$job\",cluster=~\"$cluster\"}, instance)","includeAll":true,"multi":true,"name":"instance","options":[],"query":{"query":"label_values(vm_app_version{job=~\"$job\",cluster=~\"$cluster\"}, instance)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","type":"query"},{"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vmalert_iteration_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}, file)","includeAll":true,"multi":true,"name":"file","options":[],"query":{"query":"label_values(vmalert_iteration_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}, file)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"},{"allValue":".*","current":{},"datasource":{"type":"prometheus","uid":"$ds"},"definition":"label_values(vmalert_iteration_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}, group)","includeAll":true,"multi":true,"name":"group","options":[],"query":{"query":"label_values(vmalert_iteration_total{job=~\"$job\",instance=~\"$instance\",cluster=~\"$cluster\"}, group)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","type":"query"},{"current":{"text":"5","value":"5"},"name":"topk","options":[{"selected":true,"text":"5","value":"5"},{"selected":false,"text":"10","value":"10"},{"selected":false,"text":"20","value":"20"},{"selected":false,"text":"30","value":"30"},{"selected":false,"text":"40","value":"40"},{"selected":false,"text":"50","value":"50"}],"query":"5, 10, 20, 30, 40, 50","type":"custom"},{"baseFilters":[],"datasource":{"type":"prometheus","uid":"${ds}"},"filters":[],"name":"filter","type":"adhoc"},{"allValue":".*","datasource":{"type":"prometheus"},"hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","query":".*","type":"constant"}]},"time":{"from":"now-3h","to":"now"},"timepicker":{},"timezone":"utc","title":"VictoriaMetrics - vmalert","uid":"LzldHAVnz","version":1}
---
# Source: victoria-metrics-k8s-stack/templates/grafana/datasource.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: vmks-grafana-ds
  namespace: default
  labels:
    grafana_datasource: "1"
    app: vmks-grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  datasource.yaml: |-
    apiVersion: 1
    datasources: 
      - access: proxy
        isDefault: true
        name: VictoriaMetrics
        type: prometheus
        url: http://vmsingle-vmks.default.svc.cluster.local.:8428
      - access: proxy
        isDefault: false
        name: VictoriaMetrics (DS)
        type: victoriametrics-metrics-datasource
        url: http://vmsingle-vmks.default.svc.cluster.local.:8428
      - access: proxy
        jsonData:
          implementation: prometheus
        name: Alertmanager
        type: alertmanager
        url: http://vmalertmanager-vmks.default.svc.cluster.local.:9093
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalertmanager/monzo-template.yaml
kind: ConfigMap
apiVersion: v1
metadata:
  name: vmalertmanager-vmks-monzo-tpl
  namespace: default
  labels: 
    app.kubernetes.io/component: vmks-alertmanager
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
data:
  monzo.tmpl: |-
    # This builds the silence URL. We exclude the alertname in the range
    # to avoid the issue of having trailing comma separator (%2C) at the end
    # of the generated URL
    {{ define "__alert_silence_link" -}}
        {{ .ExternalURL }}/#/silences/new?filter=%7B
        {{- range .CommonLabels.SortedPairs -}}
            {{- if ne .Name "alertname" -}}
                {{- .Name }}%3D"{{- .Value | urlquery -}}"%2C%20
            {{- end -}}
        {{- end -}}
        alertname%3D"{{ .CommonLabels.alertname }}"%7D
    {{- end }}

    {{ define "__alert_severity_prefix" -}}
        {{ if ne .Status "firing" -}}
        :lgtm:
        {{- else if eq .Labels.severity "critical" -}}
        :fire:
        {{- else if eq .Labels.severity "warning" -}}
        :warning:
        {{- else -}}
        :question:
        {{- end }}
    {{- end }}

    {{ define "__alert_severity_prefix_title" -}}
        {{ if ne .Status "firing" -}}
        :lgtm:
        {{- else if eq .CommonLabels.severity "critical" -}}
        :fire:
        {{- else if eq .CommonLabels.severity "warning" -}}
        :warning:
        {{- else if eq .CommonLabels.severity "info" -}}
        :information_source:
        {{- else -}}
        :question:
        {{- end }}
    {{- end }}


    {{/* First line of Slack alerts */}}
    {{ define "slack.monzo.title" -}}
        [{{ .Status | toUpper -}}
        {{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{- end -}}
        ] {{ template "__alert_severity_prefix_title" . }} {{ .CommonLabels.alertname }}
    {{- end }}


    {{/* Color of Slack attachment (appears as line next to alert )*/}}
    {{ define "slack.monzo.color" -}}
        {{ if eq .Status "firing" -}}
            {{ if eq .CommonLabels.severity "warning" -}}
                warning
            {{- else if eq .CommonLabels.severity "critical" -}}
                danger
            {{- else -}}
                #439FE0
            {{- end -}}
        {{ else -}}
        good
        {{- end }}
    {{- end }}

    {{/* Emoji to display as user icon (custom emoji supported!) */}}
    {{ define "slack.monzo.icon_emoji" }}:waitwhat:{{ end }}

    {{/* The test to display in the alert */}}
    {{ define "slack.monzo.text" -}}
        {{ range .Alerts }}
            {{- if .Annotations.message }}
     • {{ .Annotations.message }}
            {{- end }}
            {{- if .Annotations.description }}
     • {{ .Annotations.description }}
            {{- end }}
        {{- end }}
    {{- end }}

    {{ define "slack.monzo.link_button_text" -}}
        {{- if .CommonAnnotations.link_text -}}
            {{- .CommonAnnotations.link_text -}}
        {{- else -}}
            Link
        {{- end }} :link:
    {{- end }}
---
# Source: victoria-metrics-k8s-stack/charts/grafana/templates/clusterrole.yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    helm.sh/chart: grafana-11.3.2
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "12.4.1"
  name: release-name-grafana-clusterrole
rules:
  - apiGroups: [""] # "" indicates the core API group
    resources: ["configmaps", "secrets"]
    verbs: ["get", "watch", "list"]
---
# Source: victoria-metrics-k8s-stack/charts/kube-state-metrics/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:    
    helm.sh/chart: kube-state-metrics-7.2.1
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: kube-state-metrics
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.18.0"
  name: release-name-kube-state-metrics
rules:

- apiGroups: ["certificates.k8s.io"]
  resources:
  - certificatesigningrequests
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - configmaps
  verbs: ["list", "watch"]

- apiGroups: ["batch"]
  resources:
  - cronjobs
  verbs: ["list", "watch"]

- apiGroups: ["apps"]
  resources:
  - daemonsets
  verbs: ["list", "watch"]

- apiGroups: ["apps"]
  resources:
  - deployments
  verbs: ["list", "watch"]

- apiGroups: ["discovery.k8s.io"]
  resources:
  - endpointslices
  verbs: ["list", "watch"]

- apiGroups: ["autoscaling"]
  resources:
  - horizontalpodautoscalers
  verbs: ["list", "watch"]

- apiGroups: ["networking.k8s.io"]
  resources:
  - ingresses
  verbs: ["list", "watch"]

- apiGroups: ["batch"]
  resources:
  - jobs
  verbs: ["list", "watch"]

- apiGroups: ["coordination.k8s.io"]
  resources:
  - leases
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - limitranges
  verbs: ["list", "watch"]

- apiGroups: ["admissionregistration.k8s.io"]
  resources:
    - mutatingwebhookconfigurations
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - namespaces
  verbs: ["list", "watch"]

- apiGroups: ["networking.k8s.io"]
  resources:
  - networkpolicies
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - nodes
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - persistentvolumeclaims
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - persistentvolumes
  verbs: ["list", "watch"]

- apiGroups: ["policy"]
  resources:
    - poddisruptionbudgets
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - pods
  verbs: ["list", "watch"]

- apiGroups: ["apps"]
  resources:
  - replicasets
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - replicationcontrollers
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - resourcequotas
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - secrets
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - services
  verbs: ["list", "watch"]

- apiGroups: ["apps"]
  resources:
  - statefulsets
  verbs: ["list", "watch"]

- apiGroups: ["storage.k8s.io"]
  resources:
    - storageclasses
  verbs: ["list", "watch"]

- apiGroups: ["admissionregistration.k8s.io"]
  resources:
    - validatingwebhookconfigurations
  verbs: ["list", "watch"]

- apiGroups: ["storage.k8s.io"]
  resources:
    - volumeattachments
  verbs: ["list", "watch"]
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: release-name-victoria-metrics-operator
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
rules:
- nonResourceURLs: 
  - /metrics
  - /metrics/resources
  verbs:
  - get
  - watch
  - list
- apiGroups:
  - ""
  resources:
  - configmaps
  - configmaps/finalizers
  - endpoints
  - events
  - persistentvolumeclaims
  - persistentvolumeclaims/finalizers
  - pods
  - pods/eviction
  - secrets
  - secrets/finalizers
  - services
  - services/finalizers
  - serviceaccounts
  - serviceaccounts/finalizers
  verbs:
  - "*"
- apiGroups:
  - ""
  resources:
  - configmaps/status
  - nodes
  - nodes/metrics
  - namespaces
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - apps
  resources:
  - daemonsets
  - daemonsets/finalizers
  - deployments
  - deployments/finalizers
  - replicasets
  - statefulsets
  - statefulsets/finalizers
  - statefulsets/status
  verbs:
  - "*"
- apiGroups:
  - monitoring.coreos.com
  resources:
  - "*"
  verbs:
  - "*"
- apiGroups:
  - rbac.authorization.k8s.io
  resources:
  - clusterrolebindings
  - clusterrolebindings/finalizers
  - clusterroles
  - clusterroles/finalizers
  - roles
  - rolebindings
  verbs:
  - "*"
- apiGroups:
  - storage.k8s.io
  resources:
  - storageclasses
  verbs:
  - list
  - get
  - watch
- apiGroups:
  - policy
  resources:
  - poddisruptionbudgets
  - poddisruptionbudgets/finalizers
  verbs:
  - "*"
- apiGroups:
  - route.openshift.io
  - image.openshift.io
  resources:
  - routers/metrics
  - registry/metrics
  verbs:
  - get
- apiGroups:
  - autoscaling
  verbs:
  - "*"
  resources:
  - horizontalpodautoscalers
- apiGroups:
  - networking.k8s.io
  resources:
  - ingresses
  - ingresses/finalizers
  verbs:
  - "*"
- apiGroups:
    - gateway.networking.k8s.io
  resources:
    - httproutes
    - httproutes/finalizers
  verbs:
    - '*'
- apiGroups:
  - apiextensions.k8s.io
  resources:
  - customresourcedefinitions
  verbs:
  - get
  - list
- apiGroups:
  - discovery.k8s.io
  resources:
  - endpointslices
  verbs:
  - list
  - watch
  - get
- apiGroups:
  - operator.victoriametrics.com
  resources:
  - vlagents
  - vlagents/finalizers
  - vlagents/status
  - vlclusters
  - vlclusters/finalizers
  - vlclusters/status
  - vlogs
  - vlogs/finalizers
  - vlogs/status
  - vlsingles
  - vlsingles/finalizers
  - vlsingles/status
  - vmagents
  - vmagents/finalizers
  - vmagents/status
  - vmalertmanagerconfigs
  - vmalertmanagerconfigs/finalizers
  - vmalertmanagerconfigs/status
  - vmalertmanagers
  - vmalertmanagers/finalizers
  - vmalertmanagers/status
  - vmalerts
  - vmalerts/finalizers
  - vmalerts/status
  - vmanomalies
  - vmanomalies/finalizers
  - vmanomalies/status
  - vmauths
  - vmauths/finalizers
  - vmauths/status
  - vmclusters
  - vmclusters/finalizers
  - vmclusters/status
  - vmdistributed
  - vmdistributed/finalizers
  - vmdistributed/status
  - vmnodescrapes
  - vmnodescrapes/finalizers
  - vmnodescrapes/status
  - vmpodscrapes
  - vmpodscrapes/finalizers
  - vmpodscrapes/status
  - vmprobes
  - vmprobes/finalizers
  - vmprobes/status
  - vmrules
  - vmrules/finalizers
  - vmrules/status
  - vmscrapeconfigs
  - vmscrapeconfigs/finalizers
  - vmscrapeconfigs/status
  - vmservicescrapes
  - vmservicescrapes/finalizers
  - vmservicescrapes/status
  - vmsingles
  - vmsingles/finalizers
  - vmsingles/status
  - vmstaticscrapes
  - vmstaticscrapes/finalizers
  - vmstaticscrapes/status
  - vmusers
  - vmusers/finalizers
  - vmusers/status
  - vtclusters
  - vtclusters/finalizers
  - vtclusters/status
  - vtsingles
  - vtsingles/finalizers
  - vtsingles/status
  verbs:
  - '*'
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: release-name-victoria-metrics-operator-victoriametrics-admin
  labels:
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
    rbac.authorization.k8s.io/aggregate-to-admin: "true"
rules: 
  - apiGroups:
    - operator.victoriametrics.com
    resources:
    - vlagents
    - vlagents/finalizers
    - vlagents/status
    - vlclusters
    - vlclusters/finalizers
    - vlclusters/status
    - vlogs
    - vlogs/finalizers
    - vlogs/status
    - vlsingles
    - vlsingles/finalizers
    - vlsingles/status
    - vmagents
    - vmagents/finalizers
    - vmagents/status
    - vmalertmanagerconfigs
    - vmalertmanagerconfigs/finalizers
    - vmalertmanagerconfigs/status
    - vmalertmanagers
    - vmalertmanagers/finalizers
    - vmalertmanagers/status
    - vmalerts
    - vmalerts/finalizers
    - vmalerts/status
    - vmanomalies
    - vmanomalies/finalizers
    - vmanomalies/status
    - vmauths
    - vmauths/finalizers
    - vmauths/status
    - vmclusters
    - vmclusters/finalizers
    - vmclusters/status
    - vmdistributed
    - vmdistributed/finalizers
    - vmdistributed/status
    - vmnodescrapes
    - vmnodescrapes/finalizers
    - vmnodescrapes/status
    - vmpodscrapes
    - vmpodscrapes/finalizers
    - vmpodscrapes/status
    - vmprobes
    - vmprobes/finalizers
    - vmprobes/status
    - vmrules
    - vmrules/finalizers
    - vmrules/status
    - vmscrapeconfigs
    - vmscrapeconfigs/finalizers
    - vmscrapeconfigs/status
    - vmservicescrapes
    - vmservicescrapes/finalizers
    - vmservicescrapes/status
    - vmsingles
    - vmsingles/finalizers
    - vmsingles/status
    - vmstaticscrapes
    - vmstaticscrapes/finalizers
    - vmstaticscrapes/status
    - vmusers
    - vmusers/finalizers
    - vmusers/status
    - vtclusters
    - vtclusters/finalizers
    - vtclusters/status
    - vtsingles
    - vtsingles/finalizers
    - vtsingles/status
    verbs:
    - '*'
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: release-name-victoria-metrics-operator-victoriametrics-view
  labels:
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
    rbac.authorization.k8s.io/aggregate-to-view: "true"
rules: 
  - apiGroups:
    - operator.victoriametrics.com
    resources:
    - vlagents
    - vlagents/finalizers
    - vlagents/status
    - vlclusters
    - vlclusters/finalizers
    - vlclusters/status
    - vlogs
    - vlogs/finalizers
    - vlogs/status
    - vlsingles
    - vlsingles/finalizers
    - vlsingles/status
    - vmagents
    - vmagents/finalizers
    - vmagents/status
    - vmalertmanagerconfigs
    - vmalertmanagerconfigs/finalizers
    - vmalertmanagerconfigs/status
    - vmalertmanagers
    - vmalertmanagers/finalizers
    - vmalertmanagers/status
    - vmalerts
    - vmalerts/finalizers
    - vmalerts/status
    - vmanomalies
    - vmanomalies/finalizers
    - vmanomalies/status
    - vmauths
    - vmauths/finalizers
    - vmauths/status
    - vmclusters
    - vmclusters/finalizers
    - vmclusters/status
    - vmdistributed
    - vmdistributed/finalizers
    - vmdistributed/status
    - vmnodescrapes
    - vmnodescrapes/finalizers
    - vmnodescrapes/status
    - vmpodscrapes
    - vmpodscrapes/finalizers
    - vmpodscrapes/status
    - vmprobes
    - vmprobes/finalizers
    - vmprobes/status
    - vmrules
    - vmrules/finalizers
    - vmrules/status
    - vmscrapeconfigs
    - vmscrapeconfigs/finalizers
    - vmscrapeconfigs/status
    - vmservicescrapes
    - vmservicescrapes/finalizers
    - vmservicescrapes/status
    - vmsingles
    - vmsingles/finalizers
    - vmsingles/status
    - vmstaticscrapes
    - vmstaticscrapes/finalizers
    - vmstaticscrapes/status
    - vmusers
    - vmusers/finalizers
    - vmusers/status
    - vtclusters
    - vtclusters/finalizers
    - vtclusters/status
    - vtsingles
    - vtsingles/finalizers
    - vtsingles/status
    verbs:
    - get
    - list
    - watch
---
# Source: victoria-metrics-k8s-stack/charts/grafana/templates/clusterrolebinding.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: release-name-grafana-clusterrolebinding
  labels:
    helm.sh/chart: grafana-11.3.2
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "12.4.1"
subjects:
  - kind: ServiceAccount
    name: release-name-grafana
    namespace: default
roleRef:
  kind: ClusterRole
  name: release-name-grafana-clusterrole
  apiGroup: rbac.authorization.k8s.io
---
# Source: victoria-metrics-k8s-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:    
    helm.sh/chart: kube-state-metrics-7.2.1
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: kube-state-metrics
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.18.0"
  name: release-name-kube-state-metrics
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: release-name-kube-state-metrics
subjects:
- kind: ServiceAccount
  name: release-name-kube-state-metrics
  namespace: default
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/crb.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: release-name-victoria-metrics-operator
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
subjects:
- kind: ServiceAccount
  name: release-name-victoria-metrics-operator
  namespace: default
roleRef:
  kind: ClusterRole
  name: release-name-victoria-metrics-operator
  apiGroup: rbac.authorization.k8s.io
---
# Source: victoria-metrics-k8s-stack/charts/grafana/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: release-name-grafana
  namespace: default
  labels:
    helm.sh/chart: grafana-11.3.2
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "12.4.1"
rules: []
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: release-name-victoria-metrics-operator
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
  namespace: default
rules:
- apiGroups:
  - coordination.k8s.io
  verbs:
  - create
  - get
  - update
  resources:
  - leases
---
# Source: victoria-metrics-k8s-stack/charts/grafana/templates/rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: release-name-grafana
  namespace: default
  labels:
    helm.sh/chart: grafana-11.3.2
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "12.4.1"
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: release-name-grafana
subjects:
- kind: ServiceAccount
  name: release-name-grafana
  namespace: default
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/rb.yaml
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: release-name-victoria-metrics-operator
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
subjects:
- kind: ServiceAccount
  name: release-name-victoria-metrics-operator
  namespace: default
roleRef:
  kind: Role
  name: release-name-victoria-metrics-operator
  apiGroup: rbac.authorization.k8s.io
---
# Source: victoria-metrics-k8s-stack/charts/grafana/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-grafana
  namespace: default
  labels:
    helm.sh/chart: grafana-11.3.2
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "12.4.1"
spec:
  type: ClusterIP
  ports:
    - name: service
      port: 80
      protocol: TCP
      targetPort: grafana
  selector:
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: release-name
---
# Source: victoria-metrics-k8s-stack/charts/kube-state-metrics/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-kube-state-metrics
  namespace: default
  labels:    
    helm.sh/chart: kube-state-metrics-7.2.1
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: kube-state-metrics
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.18.0"
  annotations:
    prometheus.io/scrape: 'true'
spec:
  type: "ClusterIP"
  ports:
  - name: http
    protocol: TCP
    port: 8080
    targetPort: http
  
  selector:    
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
---
# Source: victoria-metrics-k8s-stack/charts/prometheus-node-exporter/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-prometheus-node-exporter
  namespace: default
  labels:
    helm.sh/chart: prometheus-node-exporter-4.52.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: prometheus-node-exporter
    app.kubernetes.io/name: prometheus-node-exporter
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.10.2"
    jobLabel: node-exporter
  annotations:
    prometheus.io/scrape: "true"
spec:
  type: ClusterIP
  ports:
    - port: 9100
      targetPort: 9100
      protocol: TCP
      name: metrics
  selector:
    app.kubernetes.io/name: prometheus-node-exporter
    app.kubernetes.io/instance: release-name
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
  name: release-name-victoria-metrics-operator
spec:
  type: ClusterIP
  ports:
    - name: http
      port: 8080
      targetPort: http
      protocol: TCP
    - name: webhook
      port: 9443
      targetPort: webhook
  selector: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/name: victoria-metrics-operator
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: v1
kind: Service
metadata:
  name: vmks-core-dns
  namespace: kube-system
  labels: 
    app: vmks-core-dns
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
    jobLabel: core-dns
spec:
  clusterIP: None
  ports:
    - name: http-metrics
      port: 9153
      protocol: TCP
      targetPort: 9153
  selector: 
    k8s-app: kube-dns
  type: ClusterIP
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: v1
kind: Service
metadata:
  name: vmks-kube-controller-manager
  namespace: kube-system
  labels: 
    app: vmks-kube-controller-manager
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
    jobLabel: kube-controller-manager
spec:
  clusterIP: None
  ports:
    - name: http-metrics
      port: 10257
      protocol: TCP
      targetPort: 10257
  selector: 
    component: kube-controller-manager
  type: ClusterIP
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: v1
kind: Service
metadata:
  name: vmks-kube-etcd
  namespace: kube-system
  labels: 
    app: vmks-kube-etcd
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
    jobLabel: kube-etcd
spec:
  clusterIP: None
  ports:
    - name: http-metrics
      port: 2379
      protocol: TCP
      targetPort: 2379
  selector: 
    component: etcd
  type: ClusterIP
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: v1
kind: Service
metadata:
  name: vmks-kube-scheduler
  namespace: kube-system
  labels: 
    app: vmks-kube-scheduler
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
    jobLabel: kube-scheduler
spec:
  clusterIP: None
  ports:
    - name: http-metrics
      port: 10259
      protocol: TCP
      targetPort: 10259
  selector: 
    component: kube-scheduler
  type: ClusterIP
---
# Source: victoria-metrics-k8s-stack/charts/prometheus-node-exporter/templates/daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: release-name-prometheus-node-exporter
  namespace: default
  labels:
    helm.sh/chart: prometheus-node-exporter-4.52.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: prometheus-node-exporter
    app.kubernetes.io/name: prometheus-node-exporter
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.10.2"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: prometheus-node-exporter
      app.kubernetes.io/instance: release-name
  revisionHistoryLimit: 10
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      annotations:
        cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
      labels:
        helm.sh/chart: prometheus-node-exporter-4.52.0
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: metrics
        app.kubernetes.io/part-of: prometheus-node-exporter
        app.kubernetes.io/name: prometheus-node-exporter
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "1.10.2"
    spec:
      automountServiceAccountToken: false
      securityContext:
        fsGroup: 65534
        runAsGroup: 65534
        runAsNonRoot: true
        runAsUser: 65534
      serviceAccountName: release-name-prometheus-node-exporter
      containers:
        - name: node-exporter
          image: quay.io/prometheus/node-exporter:v1.10.2
          imagePullPolicy: IfNotPresent
          args:
            - --path.procfs=/host/proc
            - --path.sysfs=/host/sys
            - --path.rootfs=/host/root
            - --path.udev.data=/host/root/run/udev/data
            - --web.listen-address=[$(HOST_IP)]:9100
            - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/)
            - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|erofs|sysfs|tracefs)$
          securityContext:
            readOnlyRootFilesystem: true
          env:
            - name: HOST_IP
              value: 0.0.0.0
          ports:
            - name: metrics
              containerPort: 9100
              protocol: TCP
          livenessProbe:
            failureThreshold: 3
            httpGet:
              httpHeaders:
              path: /
              port: metrics
              scheme: HTTP
            initialDelaySeconds: 0
            periodSeconds: 15
            successThreshold: 1
            timeoutSeconds: 3
          readinessProbe:
            failureThreshold: 3
            httpGet:
              httpHeaders:
              path: /
              port: metrics
              scheme: HTTP
            initialDelaySeconds: 0
            periodSeconds: 15
            successThreshold: 1
            timeoutSeconds: 2
          volumeMounts:
            - name: proc
              mountPath: /host/proc
              readOnly:  true
            - name: sys
              mountPath: /host/sys
              readOnly: true
            - name: root
              mountPath: /host/root
              mountPropagation: HostToContainer
              readOnly: true
      hostNetwork: true
      hostPID: true
      hostIPC: false
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: eks.amazonaws.com/compute-type
                operator: NotIn
                values:
                - fargate
              - key: type
                operator: NotIn
                values:
                - virtual-kubelet
      nodeSelector:
        kubernetes.io/os: linux
      tolerations:
        - effect: NoSchedule
          operator: Exists
      volumes:
        - name: proc
          hostPath:
            path: /proc
        - name: sys
          hostPath:
            path: /sys
        - name: root
          hostPath:
            path: /
---
# Source: victoria-metrics-k8s-stack/charts/grafana/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-grafana
  namespace: default
  labels:
    helm.sh/chart: grafana-11.3.2
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "12.4.1"
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app.kubernetes.io/name: grafana
      app.kubernetes.io/instance: release-name
  strategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        helm.sh/chart: grafana-11.3.2
        app.kubernetes.io/name: grafana
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "12.4.1"
      annotations:
        checksum/config: f38afa2f98a6e27bc60ac444fd9f0e5e6851b7b99775c7a56b2a5de52750234c
        checksum/sc-dashboard-provider-config: 89ec8c9c059a6a61286174130a15d183be69b7a486acc8437afbaf5d790a0ddd
        checksum/secret: 0349ca37b44d79c6ebe499af4bd6440843429ad3d7f9fa4ef72d3ac2c93a9d56
        kubectl.kubernetes.io/default-container: grafana
    spec:
      
      serviceAccountName: release-name-grafana
      automountServiceAccountToken: true
      shareProcessNamespace: false
      securityContext:
        fsGroup: 472
        runAsGroup: 472
        runAsNonRoot: true
        runAsUser: 472
      enableServiceLinks: true
      containers:
        - name: grafana-sc-dashboard
          image: "quay.io/kiwigrid/k8s-sidecar:2.5.0"
          imagePullPolicy: IfNotPresent
          env:
            - name: METHOD
              value: WATCH
            - name: LABEL
              value: "grafana_dashboard"
            - name: LABEL_VALUE
              value: "1"
            - name: FOLDER
              value: "/var/lib/grafana/dashboards/default"
            - name: RESOURCE
              value: "both"
            - name: REQ_USERNAME
              valueFrom:
                secretKeyRef:
                  name: release-name-grafana
                  key: admin-user
            - name: REQ_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: release-name-grafana
                  key: admin-password
            - name: REQ_URL
              value: http://localhost:3000/api/admin/provisioning/dashboards/reload
            - name: REQ_METHOD
              value: POST
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            seccompProfile:
              type: RuntimeDefault
          volumeMounts:
            - name: sc-dashboard-volume
              mountPath: "/var/lib/grafana/dashboards"
        - name: grafana-sc-datasources
          image: "quay.io/kiwigrid/k8s-sidecar:2.5.0"
          imagePullPolicy: IfNotPresent
          env:
            - name: METHOD
              value: WATCH
            - name: LABEL
              value: "grafana_datasource"
            - name: LABEL_VALUE
              value: "1"
            - name: FOLDER
              value: "/etc/grafana/provisioning/datasources"
            - name: RESOURCE
              value: "both"
            - name: REQ_USERNAME
              valueFrom:
                secretKeyRef:
                  name: release-name-grafana
                  key: admin-user
            - name: REQ_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: release-name-grafana
                  key: admin-password
            - name: REQ_URL
              value: http://localhost:3000/api/admin/provisioning/datasources/reload
            - name: REQ_METHOD
              value: POST
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            seccompProfile:
              type: RuntimeDefault
          volumeMounts:
            - name: sc-datasources-volume
              mountPath: "/etc/grafana/provisioning/datasources"
        - name: grafana
          image: "docker.io/grafana:12.4.1"
          imagePullPolicy: IfNotPresent
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            privileged: false
            seccompProfile:
              type: RuntimeDefault
          volumeMounts:
            - name: config
              mountPath: "/etc/grafana/grafana.ini"
              subPath: grafana.ini
            - name: storage
              mountPath: "/var/lib/grafana"
            - name: search
              mountPath: "/var/lib/grafana-search"
            - name: sc-dashboard-volume
              mountPath: "/var/lib/grafana/dashboards"
            - name: sc-dashboard-provider
              mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml"
              subPath: provider.yaml
            - name: sc-datasources-volume
              mountPath: "/etc/grafana/provisioning/datasources"
          ports:
            - name: grafana
              containerPort: 3000
              protocol: TCP
            - name: gossip-tcp
              containerPort: 9094
              protocol: TCP
            - name: gossip-udp
              containerPort: 9094
              protocol: UDP
            - name: profiling
              containerPort: 6060
              protocol: TCP
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: GF_SECURITY_ADMIN_USER
              valueFrom:
                secretKeyRef:
                  name: release-name-grafana
                  key: admin-user
            - name: GF_SECURITY_ADMIN_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: release-name-grafana
                  key: admin-password
            - name: GF_PATHS_DATA
              value: /var/lib/grafana/
            - name: GF_PATHS_LOGS
              value: /var/log/grafana
            - name: GF_PATHS_PLUGINS
              value: /var/lib/grafana/plugins
            - name: GF_PATHS_PROVISIONING
              value: /etc/grafana/provisioning
            - name: GF_UNIFIED_STORAGE_INDEX_PATH
              value: /var/lib/grafana-search/bleve
          livenessProbe:
            failureThreshold: 10
            httpGet:
              path: /api/health
              port: grafana
            initialDelaySeconds: 60
            timeoutSeconds: 30
          readinessProbe:
            httpGet:
              path: /api/health
              port: grafana
      volumes:
        - name: config
          configMap:
            name: release-name-grafana
        - name: storage
          emptyDir: {}
        - name: search
          emptyDir: {}
        - name: sc-dashboard-volume
          emptyDir: {}
        - name: sc-dashboard-provider
          configMap:
            name: release-name-grafana-config-dashboards
        - name: sc-datasources-volume
          emptyDir: {}
---
# Source: victoria-metrics-k8s-stack/charts/kube-state-metrics/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-kube-state-metrics
  namespace: default
  labels:    
    helm.sh/chart: kube-state-metrics-7.2.1
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: kube-state-metrics
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.18.0"
spec:
  selector:
    matchLabels:      
      app.kubernetes.io/name: kube-state-metrics
      app.kubernetes.io/instance: release-name
  replicas: 1
  strategy:
    type: RollingUpdate
  revisionHistoryLimit: 10
  template:
    metadata:
      labels:        
        helm.sh/chart: kube-state-metrics-7.2.1
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: metrics
        app.kubernetes.io/part-of: kube-state-metrics
        app.kubernetes.io/name: kube-state-metrics
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.18.0"
    spec:
      automountServiceAccountToken: true
      hostNetwork: false
      serviceAccountName: release-name-kube-state-metrics
      securityContext:
        fsGroup: 65534
        runAsGroup: 65534
        runAsNonRoot: true
        runAsUser: 65534
        seccompProfile:
          type: RuntimeDefault
      dnsPolicy: ClusterFirst
      containers:
      - name: kube-state-metrics
        args:
        - --port=8080
        - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpointslices,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments
        imagePullPolicy: IfNotPresent
        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.18.0
        ports:
        - containerPort: 8080
          name: http
        - containerPort: 8081
          name: metrics
        livenessProbe:
          failureThreshold: 3
          httpGet:
            httpHeaders:
            path: /livez
            port: http
            scheme: HTTP
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        readinessProbe:
          failureThreshold: 3
          httpGet:
            httpHeaders:
            path: /readyz
            port: metrics
            scheme: HTTP
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        resources:
          {}
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/server.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-victoria-metrics-operator
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
spec:
  replicas: 1
  selector:
    matchLabels: 
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/name: victoria-metrics-operator
  template:
    metadata:
      labels: 
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/name: victoria-metrics-operator
    spec:
      automountServiceAccountToken: true
      securityContext: 
        fsGroup: 2000
        runAsNonRoot: true
        runAsUser: 1000
      serviceAccountName: release-name-victoria-metrics-operator
      containers:
        - name: operator
          image: operator:v0.68.3
          imagePullPolicy: IfNotPresent
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: OPERATOR_NAME
              value: victoria-metrics-operator
            - name: VM_ENABLEDPROMETHEUSCONVERTEROWNERREFERENCES
              value: "false"
          args:
            - --zap-log-level=info
            - --leader-elect
            - --health-probe-bind-address=:8081
            - --webhook.enable=true
          ports:
            - name: http
              containerPort: 8080
              protocol: TCP
            - name: probe
              containerPort: 8081
              protocol: TCP
            - name: webhook
              containerPort: 9443
              protocol: TCP
          readinessProbe: 
            failureThreshold: 3
            httpGet:
              path: /health
              port: probe
              scheme: HTTP
            initialDelaySeconds: 5
            periodSeconds: 15
            timeoutSeconds: 5
          livenessProbe: 
            failureThreshold: 3
            initialDelaySeconds: 5
            periodSeconds: 15
            tcpSocket:
              port: probe
            timeoutSeconds: 5
          volumeMounts:
            - mountPath: /tmp/k8s-webhook-server/serving-certs
              name: cert
              readOnly: true
          resources: 
            {}
          securityContext: 
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
      terminationGracePeriodSeconds: 30
      volumes:
        - name: cert
          secret:
            defaultMode: 420
            secretName: release-name-victoria-metrics-operator-validation
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmagent/vmagent.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAgent
metadata:
  name: vmks
  namespace: default
  labels: 
    app.kubernetes.io/component: vmks-vmagent
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  externalLabels: {}
  extraArgs:
    promscrape.dropOriginalLabels: "true"
    promscrape.streamParse: "true"
  image:
    tag: v1.138.0
  port: "8429"
  remoteWrite:
  - url: http://vmsingle-vmks.default.svc.cluster.local.:8428/api/v1/write
  scrapeInterval: 20s
  selectAllByDefault: true
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/vmalert.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAlert
metadata:
  name: vmks
  namespace: default
  labels: 
    app.kubernetes.io/component: vmks-vmalert
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  configMaps: []
  datasource:
    url: http://vmsingle-vmks.default.svc.cluster.local.:8428
  evaluationInterval: 20s
  externalLabels: {}
  extraArgs:
    external.alert.source: explore?left={"datasource":"VictoriaMetrics","queries":[{"expr":{{
      .Expr|jsonEscape|queryEscape }},"refId":"A"}],"range":{"from":"{{
      .ActiveAt.UnixMilli }}","to":"now"}}
    external.url: http://grafana.domain.com
    http.pathPrefix: /
    remoteWrite.disablePathAppend: "true"
  image:
    tag: v1.138.0
  notifiers:
  - url: http://vmalertmanager-vmks-0.vmalertmanager-vmks.default.svc.cluster.local.:9093
  port: "8080"
  remoteRead:
    url: http://vmsingle-vmks.default.svc.cluster.local.:8428
  remoteWrite:
    url: http://vmsingle-vmks.default.svc.cluster.local.:8428/api/v1/write
  selectAllByDefault: true
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalertmanager/vmalertmanager.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAlertmanager
metadata:
  name: vmks
  namespace: default
  labels: 
    app.kubernetes.io/component: vmks-alertmanager
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  configSecret: vmalertmanager-vmks
  externalURL: ""
  image:
    tag: v0.28.1
  port: "9093"
  replicaCount: 1
  routePrefix: /
  selectAllByDefault: true
  templates:
  - key: monzo.tmpl
    name: vmalertmanager-vmks-monzo-tpl
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMNodeScrape
metadata:
  name: vmks-cadvisor
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
  honorLabels: true
  honorTimestamps: false
  interval: 30s
  metricRelabelConfigs:
  - action: labeldrop
    regex: (uid)
  - action: labeldrop
    regex: (id|name)
  - action: drop
    regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count)
    source_labels:
    - __name__
  path: /metrics/cadvisor
  relabelConfigs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  - sourceLabels:
    - __metrics_path__
    targetLabel: metrics_path
  - replacement: kubelet
    targetLabel: job
  scheme: https
  scrapeTimeout: 5s
  selector: {}
  tlsConfig:
    caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    insecureSkipVerify: true
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMNodeScrape
metadata:
  name: vmks-kubelet
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
  honorLabels: true
  honorTimestamps: false
  interval: 30s
  metricRelabelConfigs:
  - action: labeldrop
    regex: (uid)
  - action: labeldrop
    regex: (id|name)
  - action: drop
    regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count)
    source_labels:
    - __name__
  relabelConfigs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  - sourceLabels:
    - __metrics_path__
    targetLabel: metrics_path
  - replacement: kubelet
    targetLabel: job
  scheme: https
  scrapeTimeout: 5s
  selector: {}
  tlsConfig:
    caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    insecureSkipVerify: true
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMNodeScrape
metadata:
  name: vmks-probes
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
  honorLabels: true
  honorTimestamps: false
  interval: 30s
  metricRelabelConfigs:
  - action: labeldrop
    regex: (uid)
  - action: labeldrop
    regex: (id|name)
  - action: drop
    regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count)
    source_labels:
    - __name__
  path: /metrics/probes
  relabelConfigs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  - sourceLabels:
    - __metrics_path__
    targetLabel: metrics_path
  - replacement: kubelet
    targetLabel: job
  scheme: https
  scrapeTimeout: 5s
  selector: {}
  tlsConfig:
    caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    insecureSkipVerify: true
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMNodeScrape
metadata:
  name: vmks-resources
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
  honorLabels: true
  honorTimestamps: false
  interval: 30s
  metricRelabelConfigs:
  - action: labeldrop
    regex: (uid)
  - action: labeldrop
    regex: (id|name)
  - action: drop
    regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count)
    source_labels:
    - __name__
  path: /metrics/resource
  relabelConfigs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  - sourceLabels:
    - __metrics_path__
    targetLabel: metrics_path
  - replacement: kubelet
    targetLabel: job
  scheme: https
  scrapeTimeout: 5s
  selector: {}
  tlsConfig:
    caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    insecureSkipVerify: true
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-etcd
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: etcd
    params: {}
    rules:
    - alert: etcdMembersDown
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": members are down ({{ $value }}).'
        summary: etcd cluster members are down.
      expr: max(sum(up{job=~".*etcd.*"} ==bool 0) without(instance) or count(sum(rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s]))
        without(instance) > 0.01) without(To)) without(endpoint) > 0
      labels: {}
    - alert: etcdInsufficientMembers
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": insufficient members ({{ $value
          }}).'
        summary: etcd cluster has insufficient number of members.
      expr: sum(up{job=~".*etcd.*"} ==bool 1) without(instance) < ((count(up{job=~".*etcd.*"})
        without(instance) + 1) / 2)
      labels: {}
    - alert: etcdNoLeader
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance }} has
          no leader.'
        summary: etcd cluster has no leader.
      expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
      labels: {}
    - alert: etcdHighNumberOfLeaderChanges
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }} leader changes within
          the last 15 minutes. Frequent elections may be a sign of insufficient resources,
          high network latency, or disruptions by other components and should be investigated.'
        summary: etcd cluster has high number of leader changes.
      expr: increase((max(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) without(instance)
        or (0 * absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"})))[15m:1m])
        >= 4
      labels: {}
    - alert: etcdHighNumberOfFailedGRPCRequests
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{
          $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster has high number of failed grpc requests.
      expr: ((100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*",grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m]))
        without(grpc_type,grpc_code)) / sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m]))
        without(grpc_type,grpc_code)) > 1
      labels: {}
    - alert: etcdHighNumberOfFailedGRPCRequests
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{
          $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster has high number of failed grpc requests.
      expr: ((100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*",grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m]))
        without(grpc_type,grpc_code)) / sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m]))
        without(grpc_type,grpc_code)) > 5
      labels: {}
    - alert: etcdGRPCRequestsSlow
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests
          is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method
          }} method.'
        summary: etcd grpc requests are slow
      expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*",grpc_method!="Defragment",grpc_type="unary"}[5m]))
        without(grpc_type)) > 0.15
      labels: {}
    - alert: etcdMemberCommunicationSlow
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": member communication with {{ $labels.To
          }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster member communication is slow.
      expr: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.15
      labels: {}
    - alert: etcdHighNumberOfFailedProposals
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures within
          the last 30 minutes on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster has high number of proposal failures.
      expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
      labels: {}
    - alert: etcdHighFsyncDurations
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations
          are {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster 99th percentile fsync durations are too high.
      expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.5
      labels: {}
    - alert: etcdHighFsyncDurations
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations
          are {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster 99th percentile fsync durations are too high.
      expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 1
      labels: {}
    - alert: etcdHighCommitDurations
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations
          {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster 99th percentile commit durations are too high.
      expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.25
      labels: {}
    - alert: etcdDatabaseQuotaLowSpace
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": database size exceeds the defined
          quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota
          as the writes to etcd will be disabled when it is full.'
        summary: etcd cluster database is running full.
      expr: ((last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))
        * 100) > 95
      labels: {}
    - alert: etcdExcessiveDatabaseGrowth
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": Predicting running out of disk space
          in the next four hours, based on write observations within the past four hours
          on etcd instance {{ $labels.instance }}, please check as it might be disruptive.'
        summary: etcd cluster database growing very fast.
      expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 14400)
        > etcd_server_quota_backend_bytes{job=~".*etcd.*"}
      labels: {}
    - alert: etcdDatabaseHighFragmentationRatio
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": database size in use on instance
          {{ $labels.instance }} is {{ $value | humanizePercentage }} of the actual allocated
          disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused
          fragmented disk space.'
        runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation
        summary: etcd database size in use is less than 50% of the actual allocated storage.
      expr: ((last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m])
        / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5)
        and (etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600)
      labels: {}
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-general.rules
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: general.rules
    params: {}
    rules:
    - alert: TargetDown
      annotations:
        description: '{{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service
          }} targets in {{ $labels.namespace }} namespace are down.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/targetdown
        summary: One or more targets are unreachable.
      expr: (100 * (count(up == 0) by(cluster,job,namespace,service) / count(up) by(cluster,job,namespace,service)))
        > 10
      for: 10m
      labels:
        severity: warning
    - alert: Watchdog
      annotations:
        description: |
          This is an alert meant to ensure that the entire alerting pipeline is functional.
          This alert is always firing, therefore it should always be firing in Alertmanager
          and always fire against a receiver. There are integrations with various notification
          mechanisms that send a notification when this alert is not firing. For example the
          "DeadMansSnitch" integration in PagerDuty.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/watchdog
        summary: An alert that should always be firing to certify that Alertmanager is working
          properly.
      expr: vector(1)
      labels:
        severity: none
    - alert: InfoInhibitor
      annotations:
        description: |
          This is an alert that is used to inhibit info alerts.
          By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with
          other alerts.
          This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a
          severity of 'warning' or 'critical' starts firing on the same namespace.
          This alert should be routed to a null receiver and configured to inhibit alerts with severity="info".
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/infoinhibitor
        summary: Info-level alert inhibition.
      expr: group(ALERTS{severity="info"} == 1) by(namespace,cluster) unless on(namespace,cluster)
        group(ALERTS{alertname!="InfoInhibitor",alertstate="firing",severity=~"warning|critical"}
        == 1) by(namespace,cluster)
      labels:
        severity: none
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-k8s.rules.containercpulimits
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: k8s.rules.container_cpu_limits
    params: {}
    rules:
    - annotations: {}
      expr: kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"}
        * on(namespace,pod,cluster) group_left() max(kube_pod_status_phase{phase=~"Pending|Running"}
        == 1) by(namespace,pod,cluster)
      labels: {}
      record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits
    - annotations: {}
      expr: sum(sum(max(kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"})
        by(namespace,pod,container,cluster) * on(namespace,pod,cluster) group_left() max(kube_pod_status_phase{phase=~"Pending|Running"}
        == 1) by(namespace,pod,cluster)) by(namespace,pod,cluster)) by(namespace,cluster)
      labels: {}
      record: namespace_cpu:kube_pod_container_resource_limits:sum
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-k8s.rules.containercpurequests
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: k8s.rules.container_cpu_requests
    params: {}
    rules:
    - annotations: {}
      expr: kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"}
        * on(namespace,pod,cluster) group_left() max(kube_pod_status_phase{phase=~"Pending|Running"}
        == 1) by(namespace,pod,cluster)
      labels: {}
      record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests
    - annotations: {}
      expr: sum(sum(max(kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"})
        by(namespace,pod,container,cluster) * on(namespace,pod,cluster) group_left() max(kube_pod_status_phase{phase=~"Pending|Running"}
        == 1) by(namespace,pod,cluster)) by(namespace,pod,cluster)) by(namespace,cluster)
      labels: {}
      record: namespace_cpu:kube_pod_container_resource_requests:sum
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-k8s.rules.containercpuusagesecondstotal
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: k8s.rules.container_cpu_usage_seconds_total
    params: {}
    rules:
    - annotations: {}
      expr: sum(rate(container_cpu_usage_seconds_total{job="kubelet",metrics_path="/metrics/cadvisor",image!=""}[5m]))
        by(cluster,namespace,pod,container) * on(cluster,namespace,pod) group_left(node)
        topk(1, max(kube_pod_info{node!=""}) by(cluster,namespace,pod,node)) by(cluster,namespace,pod)
      labels: {}
      record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m
    - annotations: {}
      expr: sum(irate(container_cpu_usage_seconds_total{job="kubelet",metrics_path="/metrics/cadvisor",image!=""}[5m]))
        by(cluster,namespace,pod,container) * on(cluster,namespace,pod) group_left(node)
        topk(1, max(kube_pod_info{node!=""}) by(cluster,namespace,pod,node)) by(cluster,namespace,pod)
      labels: {}
      record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-k8s.rules.containermemorycache
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: k8s.rules.container_memory_cache
    params: {}
    rules:
    - annotations: {}
      expr: container_memory_cache{job="kubelet",metrics_path="/metrics/cadvisor",image!=""}
        * on(cluster,namespace,pod) group_left(node) topk(1, max(kube_pod_info{node!=""})
        by(cluster,namespace,pod,node)) by(cluster,namespace,pod)
      labels: {}
      record: node_namespace_pod_container:container_memory_cache
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-k8s.rules.containermemorylimits
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: k8s.rules.container_memory_limits
    params: {}
    rules:
    - annotations: {}
      expr: kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"}
        * on(namespace,pod,cluster) group_left() max(kube_pod_status_phase{phase=~"Pending|Running"}
        == 1) by(namespace,pod,cluster)
      labels: {}
      record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits
    - annotations: {}
      expr: sum(sum(max(kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"})
        by(namespace,pod,container,cluster) * on(namespace,pod,cluster) group_left() max(kube_pod_status_phase{phase=~"Pending|Running"}
        == 1) by(namespace,pod,cluster)) by(namespace,pod,cluster)) by(namespace,cluster)
      labels: {}
      record: namespace_memory:kube_pod_container_resource_limits:sum
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-k8s.rules.containermemoryrequests
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: k8s.rules.container_memory_requests
    params: {}
    rules:
    - annotations: {}
      expr: kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"}
        * on(namespace,pod,cluster) group_left() max(kube_pod_status_phase{phase=~"Pending|Running"}
        == 1) by(namespace,pod,cluster)
      labels: {}
      record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests
    - annotations: {}
      expr: sum(sum(max(kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"})
        by(namespace,pod,container,cluster) * on(namespace,pod,cluster) group_left() max(kube_pod_status_phase{phase=~"Pending|Running"}
        == 1) by(namespace,pod,cluster)) by(namespace,pod,cluster)) by(namespace,cluster)
      labels: {}
      record: namespace_memory:kube_pod_container_resource_requests:sum
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-k8s.rules.containermemoryrss
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: k8s.rules.container_memory_rss
    params: {}
    rules:
    - annotations: {}
      expr: container_memory_rss{job="kubelet",metrics_path="/metrics/cadvisor",image!=""}
        * on(cluster,namespace,pod) group_left(node) topk(1, max(kube_pod_info{node!=""})
        by(cluster,namespace,pod,node)) by(cluster,namespace,pod)
      labels: {}
      record: node_namespace_pod_container:container_memory_rss
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-k8s.rules.containermemoryswap
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: k8s.rules.container_memory_swap
    params: {}
    rules:
    - annotations: {}
      expr: container_memory_swap{job="kubelet",metrics_path="/metrics/cadvisor",image!=""}
        * on(cluster,namespace,pod) group_left(node) topk(1, max(kube_pod_info{node!=""})
        by(cluster,namespace,pod,node)) by(cluster,namespace,pod)
      labels: {}
      record: node_namespace_pod_container:container_memory_swap
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-k8s.rules.containermemoryworkingsetbytes
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: k8s.rules.container_memory_working_set_bytes
    params: {}
    rules:
    - annotations: {}
      expr: container_memory_working_set_bytes{job="kubelet",metrics_path="/metrics/cadvisor",image!=""}
        * on(cluster,namespace,pod) group_left(node) topk(1, max(kube_pod_info{node!=""})
        by(cluster,namespace,pod,node)) by(cluster,namespace,pod)
      labels: {}
      record: node_namespace_pod_container:container_memory_working_set_bytes
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-k8s.rules.podowner
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: k8s.rules.pod_owner
    params: {}
    rules:
    - annotations: {}
      expr: max(label_replace(label_replace(kube_pod_owner{job="kube-state-metrics",owner_kind="ReplicaSet"},
        "replicaset", "$1", "owner_name", "(.*)") * on(cluster,replicaset,namespace) group_left(owner_name)
        topk(1, max(kube_replicaset_owner{job="kube-state-metrics",owner_kind=""}) by(cluster,replicaset,namespace,owner_name))
        by(cluster,replicaset,namespace), "workload", "$1", "replicaset", "(.*)")) by(cluster,namespace,workload,pod)
      labels:
        workload_type: replicaset
      record: namespace_workload_pod:kube_pod_owner:relabel
    - annotations: {}
      expr: max(label_replace(label_replace(kube_pod_owner{job="kube-state-metrics",owner_kind="ReplicaSet"},
        "replicaset", "$1", "owner_name", "(.*)") * on(replicaset,namespace,cluster) group_left(owner_name)
        topk(1, max(kube_replicaset_owner{job="kube-state-metrics",owner_kind="Deployment"})
        by(cluster,replicaset,namespace,owner_name)) by(cluster,replicaset,namespace), "workload",
        "$1", "owner_name", "(.*)")) by(cluster,namespace,workload,pod)
      labels:
        workload_type: deployment
      record: namespace_workload_pod:kube_pod_owner:relabel
    - annotations: {}
      expr: max(label_replace(kube_pod_owner{job="kube-state-metrics",owner_kind="DaemonSet"},
        "workload", "$1", "owner_name", "(.*)")) by(cluster,namespace,workload,pod)
      labels:
        workload_type: daemonset
      record: namespace_workload_pod:kube_pod_owner:relabel
    - annotations: {}
      expr: max(label_replace(kube_pod_owner{job="kube-state-metrics",owner_kind="StatefulSet"},
        "workload", "$1", "owner_name", "(.*)")) by(cluster,namespace,workload,pod)
      labels:
        workload_type: statefulset
      record: namespace_workload_pod:kube_pod_owner:relabel
    - annotations: {}
      expr: group(label_join(group(label_join(kube_pod_owner{job="kube-state-metrics",owner_kind="Job"},
        "job_name", "", "owner_name")) by(cluster,namespace,job_name,pod,owner_name) * on(cluster,namespace,job_name)
        group_left() group(kube_job_owner{job="kube-state-metrics",owner_kind=~"Pod|"})
        by(cluster,namespace,job_name), "workload", "", "owner_name")) by(cluster,namespace,workload,pod)
      labels:
        workload_type: job
      record: namespace_workload_pod:kube_pod_owner:relabel
    - annotations: {}
      expr: max(label_replace(kube_pod_owner{job="kube-state-metrics",owner_kind="",owner_name=""},
        "workload", "$1", "pod", "(.+)")) by(cluster,namespace,workload,pod)
      labels:
        workload_type: barepod
      record: namespace_workload_pod:kube_pod_owner:relabel
    - annotations: {}
      expr: max(label_replace(kube_pod_owner{job="kube-state-metrics",owner_kind="Node"},
        "workload", "$1", "pod", "(.+)")) by(cluster,namespace,workload,pod)
      labels:
        workload_type: staticpod
      record: namespace_workload_pod:kube_pod_owner:relabel
    - annotations: {}
      expr: group(label_join(label_join(group(label_join(kube_pod_owner{job="kube-state-metrics",owner_kind="Job"},
        "job_name", "", "owner_name")) by(cluster,namespace,job_name,pod) * on(cluster,namespace,job_name)
        group_left(owner_kind,owner_name) group(kube_job_owner{job="kube-state-metrics",owner_kind!="Pod",owner_kind!=""})
        by(cluster,namespace,job_name,owner_kind,owner_name), "workload", "", "owner_name"),
        "workload_type", "", "owner_kind") or label_replace(label_replace(label_replace(kube_pod_owner{job="kube-state-metrics",owner_kind="ReplicaSet"},
        "replicaset", "$1", "owner_name", "(.+)") * on(cluster,namespace,replicaset) group_left(owner_kind,owner_name)
        group(kube_replicaset_owner{job="kube-state-metrics",owner_kind!="Deployment",owner_kind!=""})
        by(cluster,namespace,replicaset,owner_kind,owner_name), "workload", "$1", "owner_name",
        "(.+)") or label_replace(group(kube_pod_owner{job="kube-state-metrics",owner_kind!="ReplicaSet",owner_kind!="DaemonSet",owner_kind!="StatefulSet",owner_kind!="Job",owner_kind!="Node",owner_kind!=""})
        by(cluster,namespace,pod,owner_name,owner_kind), "workload", "$1", "owner_name",
        "(.+)"), "workload_type", "$1", "owner_kind", "(.+)")) by(cluster,namespace,workload,workload_type,pod)
      labels: {}
      record: namespace_workload_pod:kube_pod_owner:relabel
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kube-apiserver-availability.rules
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - interval: 3m
    name: kube-apiserver-availability.rules
    params: {}
    rules:
    - annotations: {}
      expr: (avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24) * 30
      labels: {}
      record: code_verb:apiserver_request_total:increase30d
    - annotations: {}
      expr: sum(code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) by(cluster,code)
      labels:
        verb: read
      record: code:apiserver_request_total:increase30d
    - annotations: {}
      expr: sum(code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
        by(cluster,code)
      labels:
        verb: write
      record: code:apiserver_request_total:increase30d
    - annotations: {}
      expr: sum(increase(apiserver_request_sli_duration_seconds_bucket[1h])) by(cluster,verb,scope,le)
      labels: {}
      record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h
    - annotations: {}
      expr: sum((avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d])
        * 24) * 30) by(cluster,verb,scope,le)
      labels: {}
      record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d
    - annotations: {}
      expr: sum(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h{le="+Inf"})
        by(cluster,verb,scope)
      labels: {}
      record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h
    - annotations: {}
      expr: sum(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{le="+Inf"})
        by(cluster,verb,scope)
      labels: {}
      record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d
    - annotations: {}
      expr: 1 - ((((sum(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
        by(cluster) - sum(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le=~"1(\\.0)?"}
        or vector(0)) by(cluster)) + (sum(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"LIST|GET"})
        by(cluster) - ((sum(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le=~"1(\\.0)?"}
        or vector(0)) by(cluster) + sum(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le=~"5(\\.0)?"}
        or vector(0)) by(cluster)) + sum(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le=~"30(\\.0)?"}
        or vector(0)) by(cluster)))) + sum(code:apiserver_request_total:increase30d{code=~"5.."}
        or vector(0)) by(cluster)) / sum(code:apiserver_request_total:increase30d) by(cluster))
      labels:
        verb: all
      record: apiserver_request:availability30d
    - annotations: {}
      expr: 1 - (((sum(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"LIST|GET"})
        by(cluster) - ((sum(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le=~"1(\\.0)?"}
        or vector(0)) by(cluster) + sum(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le=~"5(\\.0)?"}
        or vector(0)) by(cluster)) + sum(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le=~"30(\\.0)?"}
        or vector(0)) by(cluster))) + sum(code:apiserver_request_total:increase30d{verb="read",code=~"5.."}
        or vector(0)) by(cluster)) / sum(code:apiserver_request_total:increase30d{verb="read"})
        by(cluster))
      labels:
        verb: read
      record: apiserver_request:availability30d
    - annotations: {}
      expr: 1 - (((sum(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
        by(cluster) - sum(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le=~"1(\\.0)?"}
        or vector(0)) by(cluster)) + sum(code:apiserver_request_total:increase30d{verb="write",code=~"5.."}
        or vector(0)) by(cluster)) / sum(code:apiserver_request_total:increase30d{verb="write"})
        by(cluster))
      labels:
        verb: write
      record: apiserver_request:availability30d
    - annotations: {}
      expr: sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) by(cluster,code,resource)
      labels:
        verb: read
      record: code_resource:apiserver_request_total:rate5m
    - annotations: {}
      expr: sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
        by(cluster,code,resource)
      labels:
        verb: write
      record: code_resource:apiserver_request_total:rate5m
    - annotations: {}
      expr: sum(increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h]))
        by(cluster,code,verb)
      labels: {}
      record: code_verb:apiserver_request_total:increase1h
    - annotations: {}
      expr: sum(increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h]))
        by(cluster,code,verb)
      labels: {}
      record: code_verb:apiserver_request_total:increase1h
    - annotations: {}
      expr: sum(increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h]))
        by(cluster,code,verb)
      labels: {}
      record: code_verb:apiserver_request_total:increase1h
    - annotations: {}
      expr: sum(increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h]))
        by(cluster,code,verb)
      labels: {}
      record: code_verb:apiserver_request_total:increase1h
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kube-apiserver-burnrate.rules
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kube-apiserver-burnrate.rules
    params: {}
    rules:
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
        by(cluster) - (((sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[1d]))
        by(cluster) or vector(0)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[1d]))
        by(cluster)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[1d]))
        by(cluster))) + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d]))
        by(cluster)
      labels:
        verb: read
      record: apiserver_request:burnrate1d
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
        by(cluster) - (((sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[1h]))
        by(cluster) or vector(0)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[1h]))
        by(cluster)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[1h]))
        by(cluster))) + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h]))
        by(cluster)
      labels:
        verb: read
      record: apiserver_request:burnrate1h
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
        by(cluster) - (((sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[2h]))
        by(cluster) or vector(0)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[2h]))
        by(cluster)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[2h]))
        by(cluster))) + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h]))
        by(cluster)
      labels:
        verb: read
      record: apiserver_request:burnrate2h
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
        by(cluster) - (((sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[30m]))
        by(cluster) or vector(0)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[30m]))
        by(cluster)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[30m]))
        by(cluster))) + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m]))
        by(cluster)
      labels:
        verb: read
      record: apiserver_request:burnrate30m
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
        by(cluster) - (((sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[3d]))
        by(cluster) or vector(0)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[3d]))
        by(cluster)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[3d]))
        by(cluster))) + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d]))
        by(cluster)
      labels:
        verb: read
      record: apiserver_request:burnrate3d
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
        by(cluster) - (((sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[5m]))
        by(cluster) or vector(0)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[5m]))
        by(cluster)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[5m]))
        by(cluster))) + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m]))
        by(cluster)
      labels:
        verb: read
      record: apiserver_request:burnrate5m
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
        by(cluster) - (((sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[6h]))
        by(cluster) or vector(0)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[6h]))
        by(cluster)) + sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[6h]))
        by(cluster))) + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h]))
        by(cluster)
      labels:
        verb: read
      record: apiserver_request:burnrate6h
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
        by(cluster) - sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[1d]))
        by(cluster)) + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d]))
        by(cluster)
      labels:
        verb: write
      record: apiserver_request:burnrate1d
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
        by(cluster) - sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[1h]))
        by(cluster)) + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))
        by(cluster)
      labels:
        verb: write
      record: apiserver_request:burnrate1h
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
        by(cluster) - sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[2h]))
        by(cluster)) + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h]))
        by(cluster)
      labels:
        verb: write
      record: apiserver_request:burnrate2h
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
        by(cluster) - sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[30m]))
        by(cluster)) + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))
        by(cluster)
      labels:
        verb: write
      record: apiserver_request:burnrate30m
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
        by(cluster) - sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[3d]))
        by(cluster)) + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d]))
        by(cluster)
      labels:
        verb: write
      record: apiserver_request:burnrate3d
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
        by(cluster) - sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[5m]))
        by(cluster)) + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
        by(cluster)
      labels:
        verb: write
      record: apiserver_request:burnrate5m
    - annotations: {}
      expr: ((sum(rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
        by(cluster) - sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[6h]))
        by(cluster)) + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h]))
        by(cluster)) / sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))
        by(cluster)
      labels:
        verb: write
      record: apiserver_request:burnrate6h
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kube-apiserver-histogram.rules
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kube-apiserver-histogram.rules
    params: {}
    rules:
    - annotations: {}
      expr: histogram_quantile(0.99, sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
        by(cluster,le,resource)) > 0
      labels:
        quantile: "0.99"
        verb: read
      record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile
    - annotations: {}
      expr: histogram_quantile(0.99, sum(rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
        by(cluster,le,resource)) > 0
      labels:
        quantile: "0.99"
        verb: write
      record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kube-apiserver-slos
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kube-apiserver-slos
    params: {}
    rules:
    - alert: KubeAPIErrorBudgetBurn
      annotations:
        description: The API server is burning too much error budget on cluster {{ $labels.cluster
          }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
        summary: The API server is burning too much error budget.
      expr: (sum(apiserver_request:burnrate1h) by(cluster) > 0.14400000000000002) and on(cluster)
        (sum(apiserver_request:burnrate5m) by(cluster) > 0.14400000000000002)
      for: 2m
      labels:
        long: 1h
        severity: critical
        short: 5m
    - alert: KubeAPIErrorBudgetBurn
      annotations:
        description: The API server is burning too much error budget on cluster {{ $labels.cluster
          }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
        summary: The API server is burning too much error budget.
      expr: (sum(apiserver_request:burnrate6h) by(cluster) > 0.06) and on(cluster) (sum(apiserver_request:burnrate30m)
        by(cluster) > 0.06)
      for: 15m
      labels:
        long: 6h
        severity: critical
        short: 30m
    - alert: KubeAPIErrorBudgetBurn
      annotations:
        description: The API server is burning too much error budget on cluster {{ $labels.cluster
          }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
        summary: The API server is burning too much error budget.
      expr: (sum(apiserver_request:burnrate1d) by(cluster) > 0.03) and on(cluster) (sum(apiserver_request:burnrate2h)
        by(cluster) > 0.03)
      for: 1h
      labels:
        long: 1d
        severity: warning
        short: 2h
    - alert: KubeAPIErrorBudgetBurn
      annotations:
        description: The API server is burning too much error budget on cluster {{ $labels.cluster
          }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
        summary: The API server is burning too much error budget.
      expr: (sum(apiserver_request:burnrate3d) by(cluster) > 0.01) and on(cluster) (sum(apiserver_request:burnrate6h)
        by(cluster) > 0.01)
      for: 3h
      labels:
        long: 3d
        severity: warning
        short: 6h
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kube-prometheus-node-recording.rules
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kube-prometheus-node-recording.rules
    params: {}
    rules:
    - annotations: {}
      expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m]))
        by(instance,cluster)
      labels: {}
      record: instance:node_cpu:rate:sum
    - annotations: {}
      expr: sum(rate(node_network_receive_bytes_total[3m])) by(instance,cluster)
      labels: {}
      record: instance:node_network_receive_bytes:rate:sum
    - annotations: {}
      expr: sum(rate(node_network_transmit_bytes_total[3m])) by(instance,cluster)
      labels: {}
      record: instance:node_network_transmit_bytes:rate:sum
    - annotations: {}
      expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m]))
        without(cpu,mode) / on(instance,cluster) group_left() count(sum(node_cpu_seconds_total)
        by(instance,cpu,cluster)) by(instance,cluster)
      labels: {}
      record: instance:node_cpu:ratio
    - annotations: {}
      expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m]))
        by(cluster)
      labels: {}
      record: cluster:node_cpu:sum_rate5m
    - annotations: {}
      expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) by(instance,cpu,cluster))
        by(cluster)
      labels: {}
      record: cluster:node_cpu:ratio
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kube-scheduler.rules
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kube-scheduler.rules
    params: {}
    rules:
    - annotations: {}
      expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_attempt_duration_seconds_bucket{job="kube-scheduler"}[5m]))
        without(instance,pod))
      labels:
        quantile: "0.99"
      record: cluster_quantile:scheduler_scheduling_attempt_duration_seconds:histogram_quantile
    - annotations: {}
      expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m]))
        without(instance,pod))
      labels:
        quantile: "0.99"
      record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
    - annotations: {}
      expr: histogram_quantile(0.99, sum(rate(scheduler_pod_scheduling_sli_duration_seconds_bucket{job="kube-scheduler"}[5m]))
        without(instance,pod))
      labels:
        quantile: "0.99"
      record: cluster_quantile:scheduler_pod_scheduling_sli_duration_seconds:histogram_quantile
    - annotations: {}
      expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_attempt_duration_seconds_bucket{job="kube-scheduler"}[5m]))
        without(instance,pod))
      labels:
        quantile: "0.9"
      record: cluster_quantile:scheduler_scheduling_attempt_duration_seconds:histogram_quantile
    - annotations: {}
      expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m]))
        without(instance,pod))
      labels:
        quantile: "0.9"
      record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
    - annotations: {}
      expr: histogram_quantile(0.9, sum(rate(scheduler_pod_scheduling_sli_duration_seconds_bucket{job="kube-scheduler"}[5m]))
        without(instance,pod))
      labels:
        quantile: "0.9"
      record: cluster_quantile:scheduler_pod_scheduling_sli_duration_seconds:histogram_quantile
    - annotations: {}
      expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_attempt_duration_seconds_bucket{job="kube-scheduler"}[5m]))
        without(instance,pod))
      labels:
        quantile: "0.5"
      record: cluster_quantile:scheduler_scheduling_attempt_duration_seconds:histogram_quantile
    - annotations: {}
      expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m]))
        without(instance,pod))
      labels:
        quantile: "0.5"
      record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
    - annotations: {}
      expr: histogram_quantile(0.5, sum(rate(scheduler_pod_scheduling_sli_duration_seconds_bucket{job="kube-scheduler"}[5m]))
        without(instance,pod))
      labels:
        quantile: "0.5"
      record: cluster_quantile:scheduler_pod_scheduling_sli_duration_seconds:histogram_quantile
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kube-state-metrics
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kube-state-metrics
    params: {}
    rules:
    - alert: KubeStateMetricsListErrors
      annotations:
        description: kube-state-metrics is experiencing errors at an elevated rate in list
          operations. This is likely causing it to not be able to expose metrics about Kubernetes
          objects correctly or at all.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricslisterrors
        summary: kube-state-metrics is experiencing errors in list operations.
      expr: (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m]))
        by(cluster) / sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m]))
        by(cluster)) > 0.01
      for: 15m
      labels:
        severity: critical
    - alert: KubeStateMetricsWatchErrors
      annotations:
        description: kube-state-metrics is experiencing errors at an elevated rate in watch
          operations. This is likely causing it to not be able to expose metrics about Kubernetes
          objects correctly or at all.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricswatcherrors
        summary: kube-state-metrics is experiencing errors in watch operations.
      expr: (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m]))
        by(cluster) / sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m]))
        by(cluster)) > 0.01
      for: 15m
      labels:
        severity: critical
    - alert: KubeStateMetricsShardingMismatch
      annotations:
        description: kube-state-metrics pods are running with different --total-shards configuration,
          some Kubernetes objects may be exposed multiple times or not exposed at all.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardingmismatch
        summary: kube-state-metrics sharding is misconfigured.
      expr: stdvar(kube_state_metrics_total_shards{job="kube-state-metrics"}) by(cluster)
        != 0
      for: 15m
      labels:
        severity: critical
    - alert: KubeStateMetricsShardsMissing
      annotations:
        description: kube-state-metrics shards are missing, some Kubernetes objects are
          not being exposed.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardsmissing
        summary: kube-state-metrics shards are missing.
      expr: (((2 ^ max(kube_state_metrics_total_shards{job="kube-state-metrics"}) by(cluster))
        - 1) - sum(2 ^ max(kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) by(cluster,shard_ordinal))
        by(cluster)) != 0
      for: 15m
      labels:
        severity: critical
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kubelet.rules
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kubelet.rules
    params: {}
    rules:
    - annotations: {}
      expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet",metrics_path="/metrics"}[5m]))
        by(cluster,instance,le) * on(cluster,instance) group_left(node) max(kubelet_node_name{job="kubelet",metrics_path="/metrics"})
        by(cluster,instance,node))
      labels:
        quantile: "0.99"
      record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
    - annotations: {}
      expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet",metrics_path="/metrics"}[5m]))
        by(cluster,instance,le) * on(cluster,instance) group_left(node) max(kubelet_node_name{job="kubelet",metrics_path="/metrics"})
        by(cluster,instance,node))
      labels:
        quantile: "0.9"
      record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
    - annotations: {}
      expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet",metrics_path="/metrics"}[5m]))
        by(cluster,instance,le) * on(cluster,instance) group_left(node) max(kubelet_node_name{job="kubelet",metrics_path="/metrics"})
        by(cluster,instance,node))
      labels:
        quantile: "0.5"
      record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kubernetes-apps
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kubernetes-apps
    params: {}
    rules:
    - alert: KubePodCrashLooping
      annotations:
        description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
          }}) is in waiting state (reason: "CrashLoopBackOff") on cluster {{ $labels.cluster
          }}.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping
        summary: Pod is crash looping.
      expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff",namespace=~".*",job="kube-state-metrics"}[5m])
        >= 1
      for: 15m
      labels:
        severity: warning
    - alert: KubePodNotReady
      annotations:
        description: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready
          state for longer than 15 minutes on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
        summary: Pod has been in a non-ready state for more than 15 minutes.
      expr: sum(max(kube_pod_status_phase{namespace=~".*",job="kube-state-metrics",phase=~"Pending|Unknown"})
        by(namespace,pod,job,cluster) * on(namespace,pod,cluster) group_left(owner_kind)
        topk(1, max(kube_pod_owner{owner_kind!="Job"}) by(namespace,pod,owner_kind,cluster))
        by(namespace,pod,cluster)) by(namespace,pod,job,cluster) > 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeDeploymentGenerationMismatch
      annotations:
        description: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment
          }} does not match, this indicates that the Deployment has failed but has not been
          rolled back on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentgenerationmismatch
        summary: Deployment generation mismatch due to possible roll-back
      expr: kube_deployment_status_observed_generation{namespace=~".*",job="kube-state-metrics"}
        != kube_deployment_metadata_generation{namespace=~".*",job="kube-state-metrics"}
      for: 15m
      labels:
        severity: warning
    - alert: KubeDeploymentReplicasMismatch
      annotations:
        description: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not
          matched the expected number of replicas for longer than 15 minutes on cluster
          {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch
        summary: Deployment has not matched the expected number of replicas.
      expr: (kube_deployment_spec_replicas{namespace=~".*",job="kube-state-metrics"} > kube_deployment_status_replicas_available{namespace=~".*",job="kube-state-metrics"})
        and (changes(kube_deployment_status_replicas_updated{namespace=~".*",job="kube-state-metrics"}[10m])
        == 0)
      for: 15m
      labels:
        severity: warning
    - alert: KubeDeploymentRolloutStuck
      annotations:
        description: Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment
          }} is not progressing for longer than 15 minutes on cluster {{ $labels.cluster
          }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentrolloutstuck
        summary: Deployment rollout is not progressing.
      expr: kube_deployment_status_condition{condition="Progressing",status="false",namespace=~".*",job="kube-state-metrics"}
        != 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeStatefulSetReplicasMismatch
      annotations:
        description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not
          matched the expected number of replicas for longer than 15 minutes on cluster
          {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetreplicasmismatch
        summary: StatefulSet has not matched the expected number of replicas.
      expr: (kube_statefulset_status_replicas_ready{namespace=~".*",job="kube-state-metrics"}
        != kube_statefulset_replicas{namespace=~".*",job="kube-state-metrics"}) and (changes(kube_statefulset_status_replicas_updated{namespace=~".*",job="kube-state-metrics"}[10m])
        == 0)
      for: 15m
      labels:
        severity: warning
    - alert: KubeStatefulSetGenerationMismatch
      annotations:
        description: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset
          }} does not match, this indicates that the StatefulSet has failed but has not
          been rolled back on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetgenerationmismatch
        summary: StatefulSet generation mismatch due to possible roll-back
      expr: kube_statefulset_status_observed_generation{namespace=~".*",job="kube-state-metrics"}
        != kube_statefulset_metadata_generation{namespace=~".*",job="kube-state-metrics"}
      for: 15m
      labels:
        severity: warning
    - alert: KubeStatefulSetUpdateNotRolledOut
      annotations:
        description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update
          has not been rolled out on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout
        summary: StatefulSet update has not been rolled out.
      expr: (max(kube_statefulset_status_current_revision{namespace=~".*",job="kube-state-metrics"}
        unless kube_statefulset_status_update_revision{namespace=~".*",job="kube-state-metrics"})
        by(namespace,statefulset,job,cluster) * on(namespace,statefulset,job,cluster) (kube_statefulset_replicas{namespace=~".*",job="kube-state-metrics"}
        != kube_statefulset_status_replicas_updated{namespace=~".*",job="kube-state-metrics"}))
        and on(namespace,statefulset,job,cluster) (changes(kube_statefulset_status_replicas_updated{namespace=~".*",job="kube-state-metrics"}[5m])
        == 0)
      for: 15m
      labels:
        severity: warning
    - alert: KubeDaemonSetRolloutStuck
      annotations:
        description: DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished
          or progressed for at least 15m on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck
        summary: DaemonSet rollout is stuck.
      expr: ((((kube_daemonset_status_current_number_scheduled{namespace=~".*",job="kube-state-metrics"}
        != kube_daemonset_status_desired_number_scheduled{namespace=~".*",job="kube-state-metrics"})
        or (kube_daemonset_status_number_misscheduled{namespace=~".*",job="kube-state-metrics"}
        != 0)) or (kube_daemonset_status_updated_number_scheduled{namespace=~".*",job="kube-state-metrics"}
        != kube_daemonset_status_desired_number_scheduled{namespace=~".*",job="kube-state-metrics"}))
        or (kube_daemonset_status_number_available{namespace=~".*",job="kube-state-metrics"}
        != kube_daemonset_status_desired_number_scheduled{namespace=~".*",job="kube-state-metrics"}))
        and (changes(kube_daemonset_status_updated_number_scheduled{namespace=~".*",job="kube-state-metrics"}[5m])
        == 0)
      for: 15m
      labels:
        severity: warning
    - alert: KubeContainerWaiting
      annotations:
        description: 'pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container
          {{ $labels.container}} has been in waiting state for longer than 1 hour. (reason:
          "{{ $labels.reason }}") on cluster {{ $labels.cluster }}.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting
        summary: Pod container waiting longer than 1 hour
      expr: kube_pod_container_status_waiting_reason{reason!="CrashLoopBackOff",namespace=~".*",job="kube-state-metrics"}
        > 0
      for: 1h
      labels:
        severity: warning
    - alert: KubeDaemonSetNotScheduled
      annotations:
        description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset
          }} are not scheduled on cluster {{ $labels.cluster }}.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetnotscheduled
        summary: DaemonSet pods are not scheduled.
      expr: (kube_daemonset_status_desired_number_scheduled{namespace=~".*",job="kube-state-metrics"}
        - kube_daemonset_status_current_number_scheduled{namespace=~".*",job="kube-state-metrics"})
        > 0
      for: 10m
      labels:
        severity: warning
    - alert: KubeDaemonSetMisScheduled
      annotations:
        description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset
          }} are running where they are not supposed to run on cluster {{ $labels.cluster
          }}.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetmisscheduled
        summary: DaemonSet pods are misscheduled.
      expr: kube_daemonset_status_number_misscheduled{namespace=~".*",job="kube-state-metrics"}
        > 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeJobNotCompleted
      annotations:
        description: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than
          {{ "43200" | humanizeDuration }} to complete on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted
        summary: Job did not complete in time
      expr: (time() - max(kube_job_status_start_time{namespace=~".*",job="kube-state-metrics"}
        and (kube_job_status_active{namespace=~".*",job="kube-state-metrics"} > 0)) by(namespace,job_name,cluster))
        > 43200
      labels:
        severity: warning
    - alert: KubeJobFailed
      annotations:
        description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete.
          Removing failed job after investigation should clear this alert on cluster {{
          $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed
        summary: Job failed to complete.
      expr: kube_job_failed{namespace=~".*",job="kube-state-metrics"} > 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeHpaReplicasMismatch
      annotations:
        description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }}
          has not matched the desired number of replicas for longer than 15 minutes on cluster
          {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpareplicasmismatch
        summary: HPA has not matched desired number of replicas.
      expr: (((kube_horizontalpodautoscaler_status_desired_replicas{namespace=~".*",job="kube-state-metrics"}
        != kube_horizontalpodautoscaler_status_current_replicas{namespace=~".*",job="kube-state-metrics"})
        and (kube_horizontalpodautoscaler_status_current_replicas{namespace=~".*",job="kube-state-metrics"}
        > kube_horizontalpodautoscaler_spec_min_replicas{namespace=~".*",job="kube-state-metrics"}))
        and (kube_horizontalpodautoscaler_status_current_replicas{namespace=~".*",job="kube-state-metrics"}
        < kube_horizontalpodautoscaler_spec_max_replicas{namespace=~".*",job="kube-state-metrics"}))
        and (changes(kube_horizontalpodautoscaler_status_current_replicas{namespace=~".*",job="kube-state-metrics"}[15m])
        == 0)
      for: 15m
      labels:
        severity: warning
    - alert: KubeHpaMaxedOut
      annotations:
        description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }}
          has been running at max replicas for longer than 15 minutes on cluster {{ $labels.cluster
          }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpamaxedout
        summary: HPA is running at max replicas
      expr: (kube_horizontalpodautoscaler_status_current_replicas{namespace=~".*",job="kube-state-metrics"}
        == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~".*",job="kube-state-metrics"})
        and on(namespace,horizontalpodautoscaler,cluster) (kube_horizontalpodautoscaler_spec_max_replicas{namespace=~".*",job="kube-state-metrics"}
        != kube_horizontalpodautoscaler_spec_min_replicas{namespace=~".*",job="kube-state-metrics"})
      for: 15m
      labels:
        severity: warning
    - alert: KubePdbNotEnoughHealthyPods
      annotations:
        description: PDB {{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.poddisruptionbudget
          }} expects {{ $value }} more healthy pods. The desired number of healthy pods
          has not been met for at least 15m.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepdbnotenoughhealthypods
        summary: PDB does not have enough healthy pods.
      expr: (kube_poddisruptionbudget_status_desired_healthy{namespace=~".*",job="kube-state-metrics"}
        - kube_poddisruptionbudget_status_current_healthy{namespace=~".*",job="kube-state-metrics"})
        > 0
      for: 15m
      labels:
        severity: warning
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kubernetes-resources
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kubernetes-resources
    params: {}
    rules:
    - alert: KubeCPUOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }} has overcommitted CPU resource requests
          for Pods by {{ printf "%.2f" $value }} CPU shares and cannot tolerate node failure.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuovercommit
        summary: Cluster has overcommitted CPU resource requests.
      expr: (((sum(namespace_cpu:kube_pod_container_resource_requests:sum) by(cluster) -
        sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by(cluster))
        > 0) and (count(max(kube_node_role{job="kube-state-metrics",role="control-plane"})
        by(cluster,node)) by(cluster) < 3)) or ((sum(namespace_cpu:kube_pod_container_resource_requests:sum)
        by(cluster) - ((sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
        by(cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"})
        by(cluster)) > 0)) > 0)
      for: 10m
      labels:
        severity: warning
    - alert: KubeMemoryOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }} has overcommitted memory resource requests
          for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit
        summary: Cluster has overcommitted memory resource requests.
      expr: (((sum(namespace_memory:kube_pod_container_resource_requests:sum) by(cluster)
        - sum(kube_node_status_allocatable{job="kube-state-metrics",resource="memory"})
        by(cluster)) > 0) and (count(max(kube_node_role{job="kube-state-metrics",role="control-plane"})
        by(cluster,node)) by(cluster) < 3)) or ((sum(namespace_memory:kube_pod_container_resource_requests:sum)
        by(cluster) - ((sum(kube_node_status_allocatable{job="kube-state-metrics",resource="memory"})
        by(cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="memory"})
        by(cluster)) > 0)) > 0)
      for: 10m
      labels:
        severity: warning
    - alert: KubeCPUQuotaOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }} has overcommitted CPU resource requests
          for Namespaces.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit
        summary: Cluster has overcommitted CPU resource requests.
      expr: (sum(min(kube_resourcequota{job="kube-state-metrics",type="hard",resource=~"(cpu|requests.cpu)"})
        without(resource)) by(cluster) / sum(kube_node_status_allocatable{resource="cpu",job="kube-state-metrics"})
        by(cluster)) > 1.5
      for: 5m
      labels:
        severity: warning
    - alert: KubeMemoryQuotaOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }} has overcommitted memory resource requests
          for Namespaces.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit
        summary: Cluster has overcommitted memory resource requests.
      expr: (sum(min(kube_resourcequota{job="kube-state-metrics",type="hard",resource=~"(memory|requests.memory)"})
        without(resource)) by(cluster) / sum(kube_node_status_allocatable{resource="memory",job="kube-state-metrics"})
        by(cluster)) > 1.5
      for: 5m
      labels:
        severity: warning
    - alert: KubeQuotaAlmostFull
      annotations:
        description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
          }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaalmostfull
        summary: Namespace quota is going to be full.
      expr: ((max(kube_resourcequota{job="kube-state-metrics",type="used"}) without(instance,job,type)
        / on(cluster,namespace,resource,resourcequota) group_left() (max(kube_resourcequota{job="kube-state-metrics",type="hard"})
        without(instance,job,type) > 0)) > 0.9) < 1
      for: 15m
      labels:
        severity: info
    - alert: KubeQuotaFullyUsed
      annotations:
        description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
          }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotafullyused
        summary: Namespace quota is fully used.
      expr: (max(kube_resourcequota{job="kube-state-metrics",type="used"}) without(instance,job,type)
        / on(cluster,namespace,resource,resourcequota) group_left() (max(kube_resourcequota{job="kube-state-metrics",type="hard"})
        without(instance,job,type) > 0)) == 1
      for: 15m
      labels:
        severity: info
    - alert: KubeQuotaExceeded
      annotations:
        description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
          }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaexceeded
        summary: Namespace quota has exceeded the limits.
      expr: (max(kube_resourcequota{job="kube-state-metrics",type="used"}) without(instance,job,type)
        / on(cluster,namespace,resource,resourcequota) group_left() (max(kube_resourcequota{job="kube-state-metrics",type="hard"})
        without(instance,job,type) > 0)) > 1
      for: 15m
      labels:
        severity: warning
    - alert: CPUThrottlingHigh
      annotations:
        description: '{{ $value | humanizePercentage }} throttling of CPU in namespace {{
          $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod
          }} on cluster {{ $labels.cluster }}.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh
        summary: Processes experience elevated CPU throttling.
      expr: (sum(topk(1, increase(container_cpu_cfs_throttled_periods_total{container!="",job="kubelet",metrics_path="/metrics/cadvisor"}[5m]))
        by(cluster,namespace,pod,container,instance)) without(id,metrics_path,name,image,endpoint,job,node)
        / on(cluster,namespace,pod,container,instance) group_left() sum(topk(1, increase(container_cpu_cfs_periods_total{job="kubelet",metrics_path="/metrics/cadvisor"}[5m]))
        by(cluster,namespace,pod,container,instance)) without(id,metrics_path,name,image,endpoint,job,node))
        > 0.25
      for: 15m
      labels:
        severity: info
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kubernetes-storage
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kubernetes-storage
    params: {}
    rules:
    - alert: KubePersistentVolumeFillingUp
      annotations:
        description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }}
          in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{
          . }} {{- end }} is only {{ $value | humanizePercentage }} free.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
        summary: PersistentVolume is filling up.
      expr: ((((kubelet_volume_stats_available_bytes{namespace=~".*",job="kubelet",metrics_path="/metrics"}
        / kubelet_volume_stats_capacity_bytes{namespace=~".*",job="kubelet",metrics_path="/metrics"})
        < 0.03) and (kubelet_volume_stats_used_bytes{namespace=~".*",job="kubelet",metrics_path="/metrics"}
        > 0)) unless on(cluster,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"}
        == 1)) unless on(cluster,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"}
        == 1)
      for: 1m
      labels:
        severity: critical
    - alert: KubePersistentVolumeFillingUp
      annotations:
        description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim
          }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster
          {{ . }} {{- end }} is expected to fill up within four days. Currently {{ $value
          | humanizePercentage }} is available.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
        summary: PersistentVolume is filling up.
      expr: (((((kubelet_volume_stats_available_bytes{namespace=~".*",job="kubelet",metrics_path="/metrics"}
        / kubelet_volume_stats_capacity_bytes{namespace=~".*",job="kubelet",metrics_path="/metrics"})
        < 0.15) and (kubelet_volume_stats_used_bytes{namespace=~".*",job="kubelet",metrics_path="/metrics"}
        > 0)) and (predict_linear(kubelet_volume_stats_available_bytes{namespace=~".*",job="kubelet",metrics_path="/metrics"}[6h],
        345600) < 0)) unless on(cluster,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"}
        == 1)) unless on(cluster,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"}
        == 1)
      for: 1h
      labels:
        severity: warning
    - alert: KubePersistentVolumeInodesFillingUp
      annotations:
        description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }}
          in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{
          . }} {{- end }} only has {{ $value | humanizePercentage }} free inodes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup
        summary: PersistentVolumeInodes are filling up.
      expr: ((((kubelet_volume_stats_inodes_free{namespace=~".*",job="kubelet",metrics_path="/metrics"}
        / kubelet_volume_stats_inodes{namespace=~".*",job="kubelet",metrics_path="/metrics"})
        < 0.03) and (kubelet_volume_stats_inodes_used{namespace=~".*",job="kubelet",metrics_path="/metrics"}
        > 0)) unless on(cluster,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"}
        == 1)) unless on(cluster,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"}
        == 1)
      for: 1m
      labels:
        severity: critical
    - alert: KubePersistentVolumeInodesFillingUp
      annotations:
        description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim
          }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster
          {{ . }} {{- end }} is expected to run out of inodes within four days. Currently
          {{ $value | humanizePercentage }} of its inodes are free.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup
        summary: PersistentVolumeInodes are filling up.
      expr: (((((kubelet_volume_stats_inodes_free{namespace=~".*",job="kubelet",metrics_path="/metrics"}
        / kubelet_volume_stats_inodes{namespace=~".*",job="kubelet",metrics_path="/metrics"})
        < 0.15) and (kubelet_volume_stats_inodes_used{namespace=~".*",job="kubelet",metrics_path="/metrics"}
        > 0)) and (predict_linear(kubelet_volume_stats_inodes_free{namespace=~".*",job="kubelet",metrics_path="/metrics"}[6h],
        345600) < 0)) unless on(cluster,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"}
        == 1)) unless on(cluster,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"}
        == 1)
      for: 1h
      labels:
        severity: warning
    - alert: KubePersistentVolumeErrors
      annotations:
        description: The persistent volume {{ $labels.persistentvolume }} {{ with $labels.cluster
          -}} on Cluster {{ . }} {{- end }} has status {{ $labels.phase }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeerrors
        summary: PersistentVolume is having issues with provisioning.
      expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"}
        > 0
      for: 5m
      labels:
        severity: critical
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kubernetes-system-apiserver
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kubernetes-system-apiserver
    params: {}
    rules:
    - alert: KubeClientCertificateExpiration
      annotations:
        description: A client certificate used to authenticate to kubernetes apiserver is
          expiring in less than 7.0 days on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration
        summary: Client certificate is about to expire.
      expr: (histogram_quantile(0.01, sum(rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))
        without(namespace,service,endpoint)) < 604800) and on(job,cluster,instance) (apiserver_client_certificate_expiration_seconds_count{job="apiserver"}
        > 0)
      for: 5m
      labels:
        severity: warning
    - alert: KubeClientCertificateExpiration
      annotations:
        description: A client certificate used to authenticate to kubernetes apiserver is
          expiring in less than 24.0 hours on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration
        summary: Client certificate is about to expire.
      expr: (histogram_quantile(0.01, sum(rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))
        without(namespace,service,endpoint)) < 86400) and on(job,cluster,instance) (apiserver_client_certificate_expiration_seconds_count{job="apiserver"}
        > 0)
      for: 5m
      labels:
        severity: critical
    - alert: KubeAggregatedAPIErrors
      annotations:
        description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name }}
          has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors
        summary: Kubernetes aggregated API has reported errors.
      expr: sum(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m]))
        by(cluster,instance,name,reason) > 0
      for: 10m
      labels:
        severity: warning
    - alert: KubeAggregatedAPIDown
      annotations:
        description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }}
          has been only {{ $value | humanize }}% available over the last 10m on cluster
          {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown
        summary: Kubernetes aggregated API is down.
      expr: ((1 - max(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))
        by(name,namespace,cluster)) * 100) < 85
      for: 5m
      labels:
        severity: warning
    - alert: KubeAPIDown
      annotations:
        description: KubeAPI has disappeared from Prometheus target discovery.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapidown
        summary: Target disappeared from Prometheus target discovery.
      expr: absent(up{job="apiserver"})
      for: 15m
      labels:
        severity: critical
    - alert: KubeAPITerminatedRequests
      annotations:
        description: The kubernetes apiserver has terminated {{ $value | humanizePercentage
          }} of its incoming requests on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests
        summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage
          }} of its incoming requests.
      expr: (sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) by(cluster)
        / (sum(rate(apiserver_request_total{job="apiserver"}[10m])) by(cluster) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m]))
        by(cluster))) > 0.20
      for: 5m
      labels:
        severity: warning
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kubernetes-system-controller-manager
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kubernetes-system-controller-manager
    params: {}
    rules:
    - alert: KubeControllerManagerDown
      annotations:
        description: KubeControllerManager has disappeared from Prometheus target discovery.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontrollermanagerdown
        summary: Target disappeared from Prometheus target discovery.
      expr: absent(up{job="kube-controller-manager"})
      for: 15m
      labels:
        severity: critical
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kubernetes-system-kubelet
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kubernetes-system-kubelet
    params: {}
    rules:
    - alert: KubeNodeNotReady
      annotations:
        description: '{{ $labels.node }} has been unready for more than 15 minutes on cluster
          {{ $labels.cluster }}.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodenotready
        summary: Node is not ready.
      expr: (kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"}
        == 0) and on(cluster,node) (kube_node_spec_unschedulable{job="kube-state-metrics"}
        == 0)
      for: 15m
      labels:
        severity: warning
    - alert: KubeNodePressure
      annotations:
        description: '{{ $labels.node }} on cluster {{ $labels.cluster }} has active Condition
          {{ $labels.condition }}. This is caused by resource usage exceeding eviction thresholds.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodepressure
        summary: Node has as active Condition.
      expr: (kube_node_status_condition{job="kube-state-metrics",condition=~"(MemoryPressure|DiskPressure|PIDPressure)",status="true"}
        == 1) and on(cluster,node) (kube_node_spec_unschedulable{job="kube-state-metrics"}
        == 0)
      for: 10m
      labels:
        severity: info
    - alert: KubeNodeUnreachable
      annotations:
        description: '{{ $labels.node }} is unreachable and some workloads may be rescheduled
          on cluster {{ $labels.cluster }}.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodeunreachable
        summary: Node is unreachable.
      expr: (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"}
        unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"})
        == 1
      for: 15m
      labels:
        severity: warning
    - alert: KubeletTooManyPods
      annotations:
        description: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage
          }} of its Pod capacity on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubelettoomanypods
        summary: Kubelet is running at capacity.
      expr: ((max(kubelet_running_pods{job="kubelet",metrics_path="/metrics"} > 1) by(cluster,instance)
        * on(cluster,instance) group_left(node) max(kubelet_node_name{job="kubelet",metrics_path="/metrics"})
        by(cluster,instance,node)) / on(cluster,node) group_left() max(kube_node_status_capacity{job="kube-state-metrics",resource="pods"}
        != 1) by(cluster,node)) > 0.95
      for: 15m
      labels:
        severity: info
    - alert: KubeNodeReadinessFlapping
      annotations:
        description: The readiness status of node {{ $labels.node }} has changed {{ $value
          }} times in the last 15 minutes on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping
        summary: Node readiness status is flapping.
      expr: (sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready"}[15m]))
        by(cluster,node) > 2) and on(cluster,node) (kube_node_spec_unschedulable{job="kube-state-metrics"}
        == 0)
      for: 15m
      labels:
        severity: warning
    - alert: KubeNodeEviction
      annotations:
        description: Node {{ $labels.node }} on {{ $labels.cluster }} is evicting Pods due
          to {{ $labels.eviction_signal }}.  Eviction occurs when eviction thresholds are
          crossed, typically caused by Pods exceeding RAM/ephemeral-storage limits.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodeeviction
        summary: Node is evicting pods.
      expr: (sum(rate(kubelet_evictions{job="kubelet",metrics_path="/metrics"}[15m])) by(cluster,eviction_signal,instance)
        * on(cluster,instance) group_left(node) max(kubelet_node_name{job="kubelet",metrics_path="/metrics"})
        by(cluster,instance,node)) > 0
      for: 0s
      labels:
        severity: info
    - alert: KubeletPlegDurationHigh
      annotations:
        description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration
          of {{ $value }} seconds on node {{ $labels.node }} on cluster {{ $labels.cluster
          }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletplegdurationhigh
        summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist.
      expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"}
        >= 10
      for: 5m
      labels:
        severity: warning
    - alert: KubeletPodStartUpLatencyHigh
      annotations:
        description: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds
          on node {{ $labels.node }} on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletpodstartuplatencyhigh
        summary: Kubelet Pod startup latency is too high.
      expr: (histogram_quantile(0.99, sum(topk(1, rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet",metrics_path="/metrics"}[5m]))
        by(cluster,instance,le,operation_type)) by(cluster,instance,le)) * on(cluster,instance)
        group_left(node) topk(1, kubelet_node_name{job="kubelet",metrics_path="/metrics"})
        by(cluster,instance,node)) > 60
      for: 15m
      labels:
        severity: warning
    - alert: KubeletClientCertificateExpiration
      annotations:
        description: Client certificate for Kubelet on node {{ $labels.node }} expires in
          {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration
        summary: Kubelet client certificate is about to expire.
      expr: kubelet_certificate_manager_client_ttl_seconds < 604800
      labels:
        severity: warning
    - alert: KubeletClientCertificateExpiration
      annotations:
        description: Client certificate for Kubelet on node {{ $labels.node }} expires in
          {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration
        summary: Kubelet client certificate is about to expire.
      expr: kubelet_certificate_manager_client_ttl_seconds < 86400
      labels:
        severity: critical
    - alert: KubeletServerCertificateExpiration
      annotations:
        description: Server certificate for Kubelet on node {{ $labels.node }} expires in
          {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration
        summary: Kubelet server certificate is about to expire.
      expr: kubelet_certificate_manager_server_ttl_seconds < 604800
      labels:
        severity: warning
    - alert: KubeletServerCertificateExpiration
      annotations:
        description: Server certificate for Kubelet on node {{ $labels.node }} expires in
          {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration
        summary: Kubelet server certificate is about to expire.
      expr: kubelet_certificate_manager_server_ttl_seconds < 86400
      labels:
        severity: critical
    - alert: KubeletClientCertificateRenewalErrors
      annotations:
        description: Kubelet on node {{ $labels.node }} has failed to renew its client certificate
          ({{ $value | humanize }} errors in the last 5 minutes) on cluster {{ $labels.cluster
          }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificaterenewalerrors
        summary: Kubelet has failed to renew its client certificate.
      expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeletServerCertificateRenewalErrors
      annotations:
        description: Kubelet on node {{ $labels.node }} has failed to renew its server certificate
          ({{ $value | humanize }} errors in the last 5 minutes) on cluster {{ $labels.cluster
          }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificaterenewalerrors
        summary: Kubelet has failed to renew its server certificate.
      expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeletDown
      annotations:
        description: Kubelet has disappeared from Prometheus target discovery on cluster
          {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletdown
        summary: Target disappeared from Prometheus target discovery.
      expr: count(kube_node_info{job="kube-state-metrics"}) by(cluster) unless on(cluster)
        count(up{job="kubelet",metrics_path="/metrics"} == 1) by(cluster)
      for: 15m
      labels:
        severity: critical
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kubernetes-system-scheduler
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kubernetes-system-scheduler
    params: {}
    rules:
    - alert: KubeSchedulerDown
      annotations:
        description: KubeScheduler has disappeared from Prometheus target discovery.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeschedulerdown
        summary: Target disappeared from Prometheus target discovery.
      expr: absent(up{job="kube-scheduler"})
      for: 15m
      labels:
        severity: critical
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-kubernetes-system
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: kubernetes-system
    params: {}
    rules:
    - alert: KubeVersionMismatch
      annotations:
        description: There are {{ $value }} different semantic versions of Kubernetes components
          running on cluster {{ $labels.cluster }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeversionmismatch
        summary: Different semantic versions of Kubernetes components running.
      expr: count(count(label_replace(kubernetes_build_info{job!~"kube-dns|coredns"}, "git_version",
        "$1", "git_version", "(v[0-9]*.[0-9]*).*")) by(git_version,cluster)) by(cluster)
        > 1
      for: 15m
      labels:
        severity: warning
    - alert: KubeClientErrors
      annotations:
        description: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance
          }}' is experiencing {{ $value | humanizePercentage }} errors on cluster {{ $labels.cluster
          }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclienterrors
        summary: Kubernetes API server client is experiencing errors.
      expr: (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by(cluster,instance,job,namespace)
        / sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by(cluster,instance,job,namespace))
        > 0.01
      for: 15m
      labels:
        severity: warning
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-node-exporter.rules
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: node-exporter.rules
    params: {}
    rules:
    - annotations: {}
      expr: count(node_cpu_seconds_total{job="node-exporter",mode="idle"}) without(cpu,mode)
      labels: {}
      record: instance:node_num_cpu:sum
    - annotations: {}
      expr: 1 - avg(sum(rate(node_cpu_seconds_total{job="node-exporter",mode=~"idle|iowait|steal"}[5m]))
        without(mode)) without(cpu)
      labels: {}
      record: instance:node_cpu_utilisation:rate5m
    - annotations: {}
      expr: node_load1{job="node-exporter"} / instance:node_num_cpu:sum{job="node-exporter"}
      labels: {}
      record: instance:node_load1_per_cpu:ratio
    - annotations: {}
      expr: 1 - ((node_memory_MemAvailable_bytes{job="node-exporter"} or (((node_memory_Buffers_bytes{job="node-exporter"}
        + node_memory_Cached_bytes{job="node-exporter"}) + node_memory_MemFree_bytes{job="node-exporter"})
        + node_memory_Slab_bytes{job="node-exporter"})) / node_memory_MemTotal_bytes{job="node-exporter"})
      labels: {}
      record: instance:node_memory_utilisation:ratio
    - annotations: {}
      expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m])
      labels: {}
      record: instance:node_vmstat_pgmajfault:rate5m
    - annotations: {}
      expr: rate(node_disk_io_time_seconds_total{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
      labels: {}
      record: instance_device:node_disk_io_time_seconds:rate5m
    - annotations: {}
      expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
      labels: {}
      record: instance_device:node_disk_io_time_weighted_seconds:rate5m
    - annotations: {}
      expr: sum(rate(node_network_receive_bytes_total{job="node-exporter",device!="lo"}[5m]))
        without(device)
      labels: {}
      record: instance:node_network_receive_bytes_excluding_lo:rate5m
    - annotations: {}
      expr: sum(rate(node_network_transmit_bytes_total{job="node-exporter",device!="lo"}[5m]))
        without(device)
      labels: {}
      record: instance:node_network_transmit_bytes_excluding_lo:rate5m
    - annotations: {}
      expr: sum(rate(node_network_receive_drop_total{job="node-exporter",device!="lo"}[5m]))
        without(device)
      labels: {}
      record: instance:node_network_receive_drop_excluding_lo:rate5m
    - annotations: {}
      expr: sum(rate(node_network_transmit_drop_total{job="node-exporter",device!="lo"}[5m]))
        without(device)
      labels: {}
      record: instance:node_network_transmit_drop_excluding_lo:rate5m
    - annotations: {}
      expr: sum(rate(node_network_receive_bytes_total{job="node-exporter",device!~"lo|veth.+"}[5m]))
        without(device)
      labels: {}
      record: instance:node_network_receive_bytes_physical:rate5m
    - annotations: {}
      expr: sum(rate(node_network_transmit_bytes_total{job="node-exporter",device!~"lo|veth.+"}[5m]))
        without(device)
      labels: {}
      record: instance:node_network_transmit_bytes_physical:rate5m
    - annotations: {}
      expr: sum(rate(node_network_receive_drop_total{job="node-exporter",device!~"lo|veth.+"}[5m]))
        without(device)
      labels: {}
      record: instance:node_network_receive_drop_physical:rate5m
    - annotations: {}
      expr: sum(rate(node_network_transmit_drop_total{job="node-exporter",device!~"lo|veth.+"}[5m]))
        without(device)
      labels: {}
      record: instance:node_network_transmit_drop_physical:rate5m
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-node-exporter
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: node-exporter
    params: {}
    rules:
    - alert: NodeFilesystemSpaceFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
          }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space
          left and is filling up.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
        summary: Filesystem is predicted to run out of space within the next 24 hours.
      expr: ((((node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}
        / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""}) * 100)
        < 15) and (predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h],
        86400) < 0)) and (node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""}
        == 0)
      for: 1h
      labels:
        severity: warning
    - alert: NodeFilesystemSpaceFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
          }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space
          left and is filling up fast.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
        summary: Filesystem is predicted to run out of space within the next 4 hours.
      expr: ((((node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}
        / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""}) * 100)
        < 10) and (predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h],
        14400) < 0)) and (node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""}
        == 0)
      for: 1h
      labels:
        severity: critical
    - alert: NodeFilesystemAlmostOutOfSpace
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
          }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space
          left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
        summary: Filesystem has less than 5% space left.
      expr: (((node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}
        / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""}) * 100)
        < 5) and (node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""}
        == 0)
      for: 30m
      labels:
        severity: warning
    - alert: NodeFilesystemAlmostOutOfSpace
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
          }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space
          left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
        summary: Filesystem has less than 3% space left.
      expr: (((node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}
        / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""}) * 100)
        < 3) and (node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""}
        == 0)
      for: 30m
      labels:
        severity: critical
    - alert: NodeFilesystemFilesFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
          }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes
          left and is filling up.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
        summary: Filesystem is predicted to run out of inodes within the next 24 hours.
      expr: ((((node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}
        / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""}) * 100) <
        40) and (predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h],
        86400) < 0)) and (node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""}
        == 0)
      for: 1h
      labels:
        severity: warning
    - alert: NodeFilesystemFilesFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
          }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes
          left and is filling up fast.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
        summary: Filesystem is predicted to run out of inodes within the next 4 hours.
      expr: ((((node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}
        / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""}) * 100) <
        20) and (predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h],
        14400) < 0)) and (node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""}
        == 0)
      for: 1h
      labels:
        severity: critical
    - alert: NodeFilesystemAlmostOutOfFiles
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
          }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes
          left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
        summary: Filesystem has less than 5% inodes left.
      expr: (((node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}
        / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""}) * 100) <
        5) and (node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""}
        == 0)
      for: 1h
      labels:
        severity: warning
    - alert: NodeFilesystemAlmostOutOfFiles
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
          }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes
          left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
        summary: Filesystem has less than 3% inodes left.
      expr: (((node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}
        / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""}) * 100) <
        3) and (node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""}
        == 0)
      for: 1h
      labels:
        severity: critical
    - alert: NodeNetworkReceiveErrs
      annotations:
        description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
          {{ printf "%.0f" $value }} receive errors in the last two minutes.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworkreceiveerrs
        summary: Network interface is reporting many receive errors.
      expr: (rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]))
        > 0.01
      for: 1h
      labels:
        severity: warning
    - alert: NodeNetworkTransmitErrs
      annotations:
        description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
          {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworktransmiterrs
        summary: Network interface is reporting many transmit errors.
      expr: (rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]))
        > 0.01
      for: 1h
      labels:
        severity: warning
    - alert: NodeHighNumberConntrackEntriesUsed
      annotations:
        description: '{{ $labels.instance }} {{ $value | humanizePercentage }} of conntrack
          entries are used.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesused
        summary: Number of conntrack are getting close to the limit.
      expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit)
        > 0.75
      labels:
        severity: warning
    - alert: NodeTextFileCollectorScrapeError
      annotations:
        description: Node Exporter text file collector on {{ $labels.instance }} failed
          to scrape.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodetextfilecollectorscrapeerror
        summary: Node Exporter text file collector failed to scrape.
      expr: node_textfile_scrape_error{job="node-exporter"} == 1
      labels:
        severity: warning
    - alert: NodeClockSkewDetected
      annotations:
        description: Clock at {{ $labels.instance }} is out of sync by more than 0.05s.
          Ensure NTP is configured correctly on this host.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetected
        summary: Clock skew detected.
      expr: ((node_timex_offset_seconds{job="node-exporter"} > 0.05) and (deriv(node_timex_offset_seconds{job="node-exporter"}[5m])
        >= 0)) or ((node_timex_offset_seconds{job="node-exporter"} < -0.05) and (deriv(node_timex_offset_seconds{job="node-exporter"}[5m])
        <= 0))
      for: 10m
      labels:
        severity: warning
    - alert: NodeClockNotSynchronising
      annotations:
        description: Clock at {{ $labels.instance }} is not synchronising. Ensure NTP is
          configured on this host.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclocknotsynchronising
        summary: Clock not synchronising.
      expr: (min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0) and (node_timex_maxerror_seconds{job="node-exporter"}
        >= 16)
      for: 10m
      labels:
        severity: warning
    - alert: NodeRAIDDegraded
      annotations:
        description: RAID array '{{ $labels.device }}' at {{ $labels.instance }} is in degraded
          state due to one or more disks failures. Number of spare drives is insufficient
          to fix issue automatically.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegraded
        summary: RAID Array is degraded.
      expr: (node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}
        - ignoring(state) node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"})
        > 0
      for: 15m
      labels:
        severity: critical
    - alert: NodeRAIDDiskFailure
      annotations:
        description: At least one device in RAID array at {{ $labels.instance }} failed.
          Array '{{ $labels.device }}' needs attention and possibly a disk swap.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailure
        summary: Failed device in RAID array.
      expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}
        > 0
      labels:
        severity: warning
    - alert: NodeFileDescriptorLimit
      annotations:
        description: File descriptors limit at {{ $labels.instance }} is currently at {{
          printf "%.2f" $value }}%.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
        summary: Kernel is predicted to exhaust file descriptors limit soon.
      expr: ((node_filefd_allocated{job="node-exporter"} * 100) / node_filefd_maximum{job="node-exporter"})
        > 70
      for: 15m
      labels:
        severity: warning
    - alert: NodeFileDescriptorLimit
      annotations:
        description: File descriptors limit at {{ $labels.instance }} is currently at {{
          printf "%.2f" $value }}%.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
        summary: Kernel is predicted to exhaust file descriptors limit soon.
      expr: ((node_filefd_allocated{job="node-exporter"} * 100) / node_filefd_maximum{job="node-exporter"})
        > 90
      for: 15m
      labels:
        severity: critical
    - alert: NodeCPUHighUsage
      annotations:
        description: |
          CPU usage at {{ $labels.instance }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodecpuhighusage
        summary: High CPU usage.
      expr: (sum(avg(rate(node_cpu_seconds_total{job="node-exporter",mode!~"idle|iowait"}[2m]))
        without(cpu)) without(mode) * 100) > 90
      for: 15m
      labels:
        severity: info
    - alert: NodeSystemSaturation
      annotations:
        description: |
          System load per core at {{ $labels.instance }} has been above 2 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
          This might indicate this instance resources saturation and can cause it becoming unresponsive.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemsaturation
        summary: System saturated, load per core is very high.
      expr: (node_load1{job="node-exporter"} / count(node_cpu_seconds_total{job="node-exporter",mode="idle"})
        without(cpu,mode)) > 2
      for: 15m
      labels:
        severity: warning
    - alert: NodeMemoryMajorPagesFaults
      annotations:
        description: |
          Memory major pages are occurring at very high rate at {{ $labels.instance }}, 500 major page faults per second for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
          Please check that there is enough memory available at this instance.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememorymajorpagesfaults
        summary: Memory major page faults are occurring at very high rate.
      expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500
      for: 15m
      labels:
        severity: warning
    - alert: NodeMemoryHighUtilization
      annotations:
        description: |
          Memory is filling up at {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememoryhighutilization
        summary: Host is running out of memory.
      expr: (100 - ((node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"})
        * 100)) > 90
      for: 15m
      labels:
        severity: warning
    - alert: NodeDiskIOSaturation
      annotations:
        description: |
          Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}.
          This symptom might indicate disk saturation.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodediskiosaturation
        summary: Disk IO queue is high.
      expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
        > 10
      for: 30m
      labels:
        severity: warning
    - alert: NodeSystemdServiceFailed
      annotations:
        description: Systemd service {{ $labels.name }} has entered failed state at {{ $labels.instance
          }}
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicefailed
        summary: Systemd service has entered failed state.
      expr: node_systemd_unit_state{job="node-exporter",state="failed"} == 1
      for: 5m
      labels:
        severity: warning
    - alert: NodeSystemdServiceCrashlooping
      annotations:
        description: Systemd service {{ $labels.name }} has being restarted too many times
          at {{ $labels.instance }} for the last 15 minutes. Please check if service is
          crash looping.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicecrashlooping
        summary: Systemd service keeps restaring, possibly crash looping.
      expr: increase(node_systemd_service_restart_total{job="node-exporter"}[5m]) > 2
      for: 15m
      labels:
        severity: warning
    - alert: NodeBondingDegraded
      annotations:
        description: Bonding interface {{ $labels.master }} on {{ $labels.instance }} is
          in degraded state due to one or more slave failures.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodebondingdegraded
        summary: Bonding interface is degraded.
      expr: (node_bonding_slaves{job="node-exporter"} - node_bonding_active{job="node-exporter"})
        != 0
      for: 5m
      labels:
        severity: warning
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-node-network
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: node-network
    params: {}
    rules:
    - alert: NodeNetworkInterfaceFlapping
      annotations:
        description: Network interface "{{ $labels.device }}" changing its up status often
          on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/nodenetworkinterfaceflapping
        summary: Network interface is often changing its status
      expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2
      for: 2m
      labels:
        severity: warning
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-node.rules
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: node.rules
    params: {}
    rules:
    - annotations: {}
      expr: topk(1, max(label_replace(kube_pod_info{job="kube-state-metrics",node!=""},
        "pod", "$1", "pod", "(.*)")) by(cluster,node,namespace,pod)) by(cluster,namespace,pod)
      labels: {}
      record: 'node_namespace_pod:kube_pod_info:'
    - annotations: {}
      expr: count(node_cpu_seconds_total{mode="idle",job="node-exporter"} * on(cluster,namespace,pod)
        group_left(node) topk(1, node_namespace_pod:kube_pod_info:) by(cluster,namespace,pod))
        by(cluster,node)
      labels: {}
      record: node:node_num_cpu:sum
    - annotations: {}
      expr: sum(node_memory_MemAvailable_bytes{job="node-exporter"} or (((node_memory_Buffers_bytes{job="node-exporter"}
        + node_memory_Cached_bytes{job="node-exporter"}) + node_memory_MemFree_bytes{job="node-exporter"})
        + node_memory_Slab_bytes{job="node-exporter"})) by(cluster)
      labels: {}
      record: :node_memory_MemAvailable_bytes:sum
    - annotations: {}
      expr: avg(sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m]))
        without(mode)) by(cluster,node)
      labels: {}
      record: node:node_cpu_utilization:ratio_rate5m
    - annotations: {}
      expr: avg(node:node_cpu_utilization:ratio_rate5m) by(cluster)
      labels: {}
      record: cluster:node_cpu:ratio_rate5m
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-vm-health
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: vm-health
    params: {}
    rules:
    - alert: TooManyRestarts
      annotations:
        description: |
          Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes. It might be crashlooping.
        summary: '{{ $labels.job }} too many restarts (instance {{ $labels.instance }})'
      expr: changes(process_start_time_seconds{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"}[15m])
        > 2
      labels:
        severity: critical
    - alert: ServiceDown
      annotations:
        description: '{{ $labels.instance }} of job {{ $labels.job }} has been down for
          more than 2 minutes.'
        summary: Service {{ $labels.job }} is down on {{ $labels.instance }}
      expr: up{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"}
        == 0
      for: 2m
      labels:
        severity: critical
    - alert: ProcessNearFDLimits
      annotations:
        description: |
          Exhausting OS file descriptors limit can cause severe degradation of the process.
          Consider to increase the limit as fast as possible.
        summary: Number of free file descriptors is less than 100 for "{{ $labels.job }}"("{{
          $labels.instance }}") for the last 5m
      expr: (process_max_fds - process_open_fds) < 100
      for: 5m
      labels:
        severity: critical
    - alert: TooHighMemoryUsage
      annotations:
        description: |
          Too high memory usage may result into multiple issues such as OOMs or degraded performance.
          Consider to either increase available memory or decrease the load on the process.
        summary: It is more than 80% of memory used by "{{ $labels.job }}"("{{ $labels.instance
          }}")
      expr: (min_over_time(process_resident_memory_anon_bytes[10m]) / vm_available_memory_bytes)
        > 0.8
      for: 5m
      labels:
        severity: critical
    - alert: TooHighCPUUsage
      annotations:
        description: |
          Too high CPU usage may be a sign of insufficient resources and make process unstable. Consider to either increase available CPU resources or decrease the load on the process.
        summary: More than 90% of CPU is used by "{{ $labels.job }}"("{{ $labels.instance
          }}") during the last 5m
      expr: (rate(process_cpu_seconds_total[5m]) / process_cpu_cores_available) > 0.9
      for: 5m
      labels:
        severity: critical
    - alert: TooHighGoroutineSchedulingLatency
      annotations:
        description: |
          Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of insufficient CPU resources or CPU throttling. Verify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.
        summary: '"{{ $labels.job }}"("{{ $labels.instance }}") has insufficient CPU resources
          for >15m'
      expr: histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"}[5m]))
        by(le,job,instance,cluster)) > 0.1
      for: 15m
      labels:
        severity: critical
    - alert: TooManyLogs
      annotations:
        description: |
          The job \"{{ $labels.job }}\" ({{ $labels.instance }}) generated {{ $value }} log messages with the level higher than info for the last 5 minutes. Check the logs for the given target. Check also the \"location\" label at the vm_log_messages_total metric if -loggerLevel command-line flag is set to value other than INFO. This label contains code locations responsible for generating log messages suppressed by -loggerLevel.
        summary: Too many logs are generated for job "{{ $labels.job }}" ({{ $labels.instance
          }})
      expr: sum(increase(vm_log_messages_total{level!="info"}[5m])) without(app_version,location,is_printed)
        > 0
      for: 15m
      labels:
        severity: warning
    - alert: TooManyTSIDMisses
      annotations:
        description: |
          Unexpected TSID misses for \"{{ $labels.job }}\" ({{ $labels.instance }}) for the last 15 minutes.
          If this happens after unclean shutdown of VictoriaMetrics process (via \"kill -9\", OOM or power off),
          then this is OK - the alert must go away in a few minutes after the restart.
          Otherwise this may point to the corruption of index data.
        summary: Unexpected TSID misses for job "{{ $labels.job }}" ({{ $labels.instance
          }}) for the last 15 minutes
      expr: increase(vm_missing_tsids_for_metric_id_total[5m]) > 0
      for: 15m
      labels:
        severity: critical
    - alert: ConcurrentInsertsHitTheLimit
      annotations:
        description: "The limit of concurrent inserts on instance {{ $labels.instance }}
          depends on the number of CPUs.\nUsually, when component constantly hits the limit
          it is likely the component is overloaded and requires more CPU.\nIn some cases
          for components like vmagent or vminsert the alert might trigger if there are too
          many clients\nmaking write attempts. If vmagent's or vminsert's CPU usage and
          network saturation are at normal level, then \nit might be worth adjusting `-maxConcurrentInserts`
          cmd-line flag.\n"
        summary: '{{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting
          concurrent inserts limit'
      expr: avg_over_time(vm_concurrent_insert_current[1m]) >= vm_concurrent_insert_capacity
      for: 15m
      labels:
        severity: warning
    - alert: IndexDBRecordsDrop
      annotations:
        description: "VictoriaMetrics could skip registering new timeseries during ingestion
          if they fail the validation process. \nFor example, `reason=too_long_item` means
          that time series cannot exceed 64KB. Please, reduce the number \nof labels or
          label values for such series. Or enforce these limits via `-maxLabelsPerTimeseries`
          and \n`-maxLabelValueLen` command-line flags.\n"
        summary: IndexDB skipped registering items during data ingestion with reason={{
          $labels.reason }}.
      expr: increase(vm_indexdb_items_dropped_total[5m]) > 0
      labels:
        severity: critical
    - alert: RowsRejectedOnIngestion
      annotations:
        description: 'Ingested rows on instance "{{ $labels.instance }}" are rejected due
          to the following reason: "{{ $labels.reason }}"'
        summary: Some rows are rejected on "{{ $labels.instance }}" on ingestion attempt
      expr: rate(vm_rows_ignored_total[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: TooHighQueryLoad
      annotations:
        description: |
          Instance {{ $labels.instance }} ({{ $labels.job }}) is failing to serve read queries during last 15m.
          Concurrency limit `-search.maxConcurrentRequests` was reached on this instance and extra queries were
          put into the queue for `-search.maxQueueDuration` interval. But even after waiting in the queue these queries weren't served.
          This happens if instance is overloaded with the current workload, or datasource is too slow to respond.
          Possible solutions are the following:
          * reduce the query load;
          * increase compute resources or number of replicas;
          * adjust limits `-search.maxConcurrentRequests` and `-search.maxQueueDuration`.
          See more at https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries.
        summary: Read queries fail with timeout for {{ $labels.job }} on instance {{ $labels.instance
          }}
      expr: increase(vm_concurrent_select_limit_timeout_total[5m]) > 0
      for: 15m
      labels:
        severity: warning
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-vmagent
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - concurrency: 2
    interval: 30s
    name: vmagent
    params: {}
    rules:
    - alert: PersistentQueueIsDroppingData
      annotations:
        dashboard: '{{ $externalURL }}/d/G7Z9GzMGz?viewPanel=49&var-instance={{ $labels.instance
          }}'
        description: Vmagent dropped {{ $value | humanize1024 }} from persistent queue on
          instance {{ $labels.instance }} for the last 10m.
        summary: Instance {{ $labels.instance }} is dropping data from persistent queue
      expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without(path) > 0
      for: 10m
      labels:
        severity: critical
    - alert: RejectedRemoteWriteDataBlocksAreDropped
      annotations:
        dashboard: '{{ $externalURL }}/d/G7Z9GzMGz?viewPanel=79&var-instance={{ $labels.instance
          }}'
        description: Job "{{ $labels.job }}" on instance {{ $labels.instance }} drops the
          rejected by remote-write server data blocks. Check the logs to find the reason
          for rejects.
        summary: Vmagent is dropping data blocks that are rejected by remote storage
      expr: sum(increase(vmagent_remotewrite_packets_dropped_total[5m])) without(url) >
        0
      for: 15m
      labels:
        severity: warning
    - alert: TooManyScrapeErrors
      annotations:
        dashboard: '{{ $externalURL }}/d/G7Z9GzMGz?viewPanel=31&var-instance={{ $labels.instance
          }}'
        description: Job "{{ $labels.job }}" on instance {{ $labels.instance }} fails to
          scrape targets for last 15m
        summary: Vmagent fails to scrape one or more targets
      expr: increase(vm_promscrape_scrapes_failed_total[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: ScrapePoolHasNoTargets
      annotations:
        description: Vmagent "{{ $labels.job }}" has scrape_pool "{{ $labels.scrape_job
          }}" with 0 discovered targets. It is likely a misconfiguration. Please follow
          https://docs.victoriametrics.com/victoriametrics/vmagent/#debugging-scrape-targets
          to troubleshoot the scraping config.
        summary: Vmagent has scrape_pool with 0 configured/discovered targets
      expr: sum(vm_promscrape_scrape_pool_targets) without(status,instance,pod) == 0
      for: 30m
      labels:
        severity: warning
    - alert: TooManyWriteErrors
      annotations:
        dashboard: '{{ $externalURL }}/d/G7Z9GzMGz?viewPanel=77&var-instance={{ $labels.instance
          }}'
        description: Job "{{ $labels.job }}" on instance {{ $labels.instance }} responds
          with errors to write requests for last 15m.
        summary: Vmagent responds with too many errors on data ingestion protocols
      expr: (sum(increase(vm_ingestserver_request_errors_total[5m])) without(name,net,type)
        + sum(increase(vmagent_http_request_errors_total[5m])) without(path,protocol)) >
        0
      for: 15m
      labels:
        severity: warning
    - alert: TooManyRemoteWriteErrors
      annotations:
        dashboard: '{{ $externalURL }}/d/G7Z9GzMGz?viewPanel=61&var-instance={{ $labels.instance
          }}'
        description: |-
          Vmagent fails to push data via remote write protocol to destination "{{ $labels.url }}"
           Ensure that destination is up and reachable.
        summary: Job "{{ $labels.job }}" on instance {{ $labels.instance }} fails to push
          to remote storage
      expr: rate(vmagent_remotewrite_retries_count_total[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: RemoteWriteConnectionIsSaturated
      annotations:
        dashboard: '{{ $externalURL }}/d/G7Z9GzMGz?viewPanel=84&var-instance={{ $labels.instance
          }}'
        description: |-
          The remote write connection between vmagent "{{ $labels.job }}" (instance {{ $labels.instance }}) and destination "{{ $labels.url }}" is saturated by more than 90% and vmagent won't be able to keep up.
           There could be the following reasons for this:
           * vmagent can't send data fast enough through the existing network connections. Increase `-remoteWrite.queues` cmd-line flag value to establish more connections per destination.
           * remote destination can't accept data fast enough. Check if remote destination has enough resources for processing.
        summary: Remote write connection from "{{ $labels.job }}" (instance {{ $labels.instance
          }}) to {{ $labels.url }} is saturated
      expr: (rate(vmagent_remotewrite_send_duration_seconds_total[5m]) / vmagent_remotewrite_queues)
        > 0.9
      for: 15m
      labels:
        severity: warning
    - alert: PersistentQueueForWritesIsSaturated
      annotations:
        dashboard: '{{ $externalURL }}/d/G7Z9GzMGz?viewPanel=98&var-instance={{ $labels.instance
          }}'
        description: Persistent queue writes for vmagent "{{ $labels.job }}" (instance {{
          $labels.instance }}) are saturated by more than 90% and vmagent won't be able
          to keep up with flushing data on disk. In this case, consider to decrease load
          on the vmagent or improve the disk throughput.
        summary: Persistent queue writes for instance {{ $labels.instance }} are saturated
      expr: rate(vm_persistentqueue_write_duration_seconds_total[5m]) > 0.9
      for: 15m
      labels:
        severity: warning
    - alert: PersistentQueueForReadsIsSaturated
      annotations:
        dashboard: '{{ $externalURL }}/d/G7Z9GzMGz?viewPanel=99&var-instance={{ $labels.instance
          }}'
        description: Persistent queue reads for vmagent "{{ $labels.job }}" (instance {{
          $labels.instance }}) are saturated by more than 90% and vmagent won't be able
          to keep up with reading data from the disk. In this case, consider to decrease
          load on the vmagent or improve the disk throughput.
        summary: Persistent queue reads for instance {{ $labels.instance }} are saturated
      expr: rate(vm_persistentqueue_read_duration_seconds_total[5m]) > 0.9
      for: 15m
      labels:
        severity: warning
    - alert: SeriesLimitHourReached
      annotations:
        dashboard: '{{ $externalURL }}/d/G7Z9GzMGz?viewPanel=88&var-instance={{ $labels.instance
          }}'
        description: Max series limit set via -remoteWrite.maxHourlySeries flag is close
          to reaching the max value. Then samples for new time series will be dropped instead
          of sending them to remote storage systems.
        summary: Instance {{ $labels.instance }} reached 90% of the limit
      expr: (vmagent_hourly_series_limit_current_series / vmagent_hourly_series_limit_max_series)
        > 0.9
      labels:
        severity: critical
    - alert: SeriesLimitDayReached
      annotations:
        dashboard: '{{ $externalURL }}/d/G7Z9GzMGz?viewPanel=90&var-instance={{ $labels.instance
          }}'
        description: Max series limit set via -remoteWrite.maxDailySeries flag is close
          to reaching the max value. Then samples for new time series will be dropped instead
          of sending them to remote storage systems.
        summary: Instance {{ $labels.instance }} reached 90% of the limit
      expr: (vmagent_daily_series_limit_current_series / vmagent_daily_series_limit_max_series)
        > 0.9
      labels:
        severity: critical
    - alert: ConfigurationReloadFailure
      annotations:
        description: Configuration hot-reload failed for vmagent on instance {{ $labels.instance
          }}. Check vmagent's logs for detailed error message.
        summary: Configuration reload failed for vmagent instance {{ $labels.instance }}
      expr: (vm_promscrape_config_last_reload_successful != 1) or (vmagent_relabel_config_last_reload_successful
        != 1)
      labels:
        severity: warning
    - alert: StreamAggrFlushTimeout
      annotations:
        description: 'Stream aggregation process can''t keep up with the load and might
          produce incorrect aggregation results. Check logs for more details. Possible solutions:
          increase aggregation interval; aggregate smaller number of series; reduce samples''
          ingestion rate to stream aggregation.'
        summary: Streaming aggregation at "{{ $labels.job }}" (instance {{ $labels.instance
          }}) can't be finished within the configured aggregation interval.
      expr: increase(vm_streamaggr_flush_timeouts_total[5m]) > 0
      labels:
        severity: warning
    - alert: StreamAggrDedupFlushTimeout
      annotations:
        description: 'Deduplication process can''t keep up with the load and might produce
          incorrect results. Check docs https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication
          and logs for more details. Possible solutions: increase deduplication interval;
          deduplicate smaller number of series; reduce samples'' ingestion rate.'
        summary: Deduplication "{{ $labels.job }}" (instance {{ $labels.instance }}) can't
          be finished within configured deduplication interval.
      expr: increase(vm_streamaggr_dedup_flush_timeouts_total[5m]) > 0
      labels:
        severity: warning
    - alert: PersistentQueueRunsOutOfSpaceIn12Hours
      annotations:
        description: RemoteWrite destination ({{ $labels.url }}) is unavailable or unable
          to receive data in a timely manner, so the persistent queue size is growing. Once
          the available space is exhausted, some samples will be discarded and cause incident.
          Please check the health of remoteWrite destination ({{ $labels.url }}).
        summary: Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{
          $labels.job }}) will run out of space in 12 hours.
      expr: (((sum(vm_persistentqueue_free_disk_space_bytes) by(job,instance,path,cluster)
        / (sum(deriv(vm_persistentqueue_bytes_pending[1m])) by(job,instance,path,cluster)
        > 0)) * on(job,instance,path,cluster) group_left(url) ((vmagent_remotewrite_pending_data_bytes
        * 0) + 1)) < 43200) > 0
      for: 10m
      labels:
        severity: warning
    - alert: PersistentQueueRunsOutOfSpaceIn4Hours
      annotations:
        description: RemoteWrite destination ({{ $labels.url }}) is unavailable or unable
          to receive data in a timely manner, so the persistent queue size is growing. Once
          the available space is exhausted, some samples will be discarded and cause incident.
          Please check the health of remoteWrite destination ({{ $labels.url }}).
        summary: Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{
          $labels.job }}) will run out of space in 4 hours.
      expr: (((sum(vm_persistentqueue_free_disk_space_bytes) by(job,instance,path,cluster)
        / (sum(deriv(vm_persistentqueue_bytes_pending[1m])) by(job,instance,path,cluster)
        > 0)) * on(job,instance,path,cluster) group_left(url) ((vmagent_remotewrite_pending_data_bytes
        * 0) + 1)) < 14400) > 0
      for: 10m
      labels:
        severity: critical
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-vmalert
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - interval: 30s
    name: vmalert
    params: {}
    rules:
    - alert: ConfigurationReloadFailure
      annotations:
        description: Configuration hot-reload failed for vmalert on instance {{ $labels.instance
          }}. Check vmalert's logs for detailed error message.
        summary: Configuration reload failed for vmalert instance {{ $labels.instance }}
      expr: vmalert_config_last_reload_successful != 1
      labels:
        severity: warning
    - alert: AlertingRulesError
      annotations:
        dashboard: '{{ $externalURL }}/d/LzldHAVnz?viewPanel=13&var-instance={{ $labels.instance
          }}&var-file={{ $labels.file }}&var-group={{ $labels.group }}'
        description: Alerting rules execution is failing for "{{ $labels.alertname }}" from
          group "{{ $labels.group }}" in file "{{ $labels.file }}". Check vmalert's logs
          for detailed error message.
        summary: Alerting rules are failing for vmalert instance {{ $labels.instance }}
      expr: sum(increase(vmalert_alerting_rules_errors_total[5m])) without(id) > 0
      for: 5m
      labels:
        severity: warning
    - alert: RecordingRulesError
      annotations:
        dashboard: '{{ $externalURL }}/d/LzldHAVnz?viewPanel=30&var-instance={{ $labels.instance
          }}&var-file={{ $labels.file }}&var-group={{ $labels.group }}'
        description: Recording rules execution is failing for "{{ $labels.recording }}"
          from group "{{ $labels.group }}" in file "{{ $labels.file }}". Check vmalert's
          logs for detailed error message.
        summary: Recording rules are failing for vmalert instance {{ $labels.instance }}
      expr: sum(increase(vmalert_recording_rules_errors_total[5m])) without(id) > 0
      for: 5m
      labels:
        severity: warning
    - alert: RecordingRulesNoData
      annotations:
        dashboard: '{{ $externalURL }}/d/LzldHAVnz?viewPanel=33&var-file={{ $labels.file
          }}&var-group={{ $labels.group }}'
        description: Recording rule "{{ $labels.recording }}" from group "{{ $labels.group
          }} in file "{{ $labels.file }}" produces 0 samples over the last 30min. It might
          be caused by a misconfiguration or incorrect query expression.
        summary: Recording rule {{ $labels.recording }} ({{ $labels.group }}) produces no
          data
      expr: sum(vmalert_recording_rules_last_evaluation_samples) without(id) < 1
      for: 30m
      labels:
        severity: info
    - alert: TooManyMissedIterations
      annotations:
        description: vmalert instance {{ $labels.instance }} is missing rules evaluations
          for group "{{ $labels.group }}" in file "{{ $labels.file }}". The group evaluation
          time takes longer than the configured evaluation interval. This may result in
          missed alerting notifications or recording rules samples. Try increasing evaluation
          interval or concurrency of group "{{ $labels.group }}". See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups.
          If rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries.
        summary: vmalert instance {{ $labels.instance }} is missing rules evaluations
      expr: increase(vmalert_iteration_missed_total[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: RemoteWriteErrors
      annotations:
        description: vmalert instance {{ $labels.instance }} is failing to push metrics
          generated via alerting or recording rules to the configured remote write URL.
          Check vmalert's logs for detailed error message.
        summary: vmalert instance {{ $labels.instance }} is failing to push metrics to remote
          write URL
      expr: increase(vmalert_remotewrite_errors_total[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: RemoteWriteDroppingData
      annotations:
        description: vmalert instance {{ $labels.instance }} is failing to send results
          of alerting or recording rules to the configured remote write URL. This may result
          into gaps in recording rules or alerts state. Check vmalert's logs for detailed
          error message.
        summary: vmalert instance {{ $labels.instance }} is dropping data sent to remote
          write URL
      expr: increase(vmalert_remotewrite_dropped_rows_total[5m]) > 0
      for: 5m
      labels:
        severity: critical
    - alert: AlertmanagerErrors
      annotations:
        description: vmalert instance {{ $labels.instance }} is failing to send alert notifications
          to "{{ $labels.addr }}". Check vmalert's logs for detailed error message.
        summary: vmalert instance {{ $labels.instance }} is failing to send notifications
          to Alertmanager
      expr: increase(vmalert_alerts_send_errors_total[5m]) > 0
      for: 15m
      labels:
        severity: warning
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-vmcluster
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - concurrency: 2
    interval: 30s
    name: vmcluster
    params: {}
    rules:
    - alert: DiskRunsOutOfSpaceIn3Days
      annotations:
        dashboard: '{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=20&var-instance={{ $labels.instance
          }}'
        description: |-
          Taking into account current ingestion rate, free disk space will be enough only for {{ $value | humanizeDuration }} on instance {{ $labels.instance }}.
           Consider to limit the ingestion rate, decrease retention or scale the disk space up if possible.
        summary: Instance {{ $labels.instance }} will run out of disk space in 3 days
      expr: ((sum(vm_free_disk_space_bytes) without(path) / (((rate(vm_rows_added_to_storage_total[1d])
        - sum(rate(vm_deduplicated_samples_total[1d])) without(type)) * (sum(vm_data_size_bytes{type!~"indexdb.*"})
        without(type) / sum(vm_rows{type!~"indexdb.*"}) without(type))) + (rate(vm_new_timeseries_created_total[1d])
        * (sum(vm_data_size_bytes{type="indexdb/file"}) without(type) / sum(vm_rows{type="indexdb/file"})
        without(type))))) < 259200) > 0
      for: 30m
      labels:
        severity: critical
    - alert: NodeBecomesReadonlyIn3Days
      annotations:
        dashboard: '{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=20&var-instance={{ $labels.instance
          }}'
        description: |-
          Taking into account current ingestion rate, free disk space and -storage.minFreeDiskSpaceBytes instance {{ $labels.instance }} will remain writable for {{ $value | humanizeDuration }}.
           Consider to limit the ingestion rate, decrease retention or scale the disk space up if possible.
        summary: Instance {{ $labels.instance }} will become read-only in 3 days
      expr: ((sum(vm_free_disk_space_bytes - vm_free_disk_space_limit_bytes) without(path)
        / (((rate(vm_rows_added_to_storage_total[1d]) - sum(rate(vm_deduplicated_samples_total[1d]))
        without(type)) * (sum(vm_data_size_bytes{type!~"indexdb.*"}) without(type) / sum(vm_rows{type!~"indexdb.*"})
        without(type))) + (rate(vm_new_timeseries_created_total[1d]) * (sum(vm_data_size_bytes{type="indexdb/file"})
        without(type) / sum(vm_rows{type="indexdb/file"}) without(type))))) < 259200) >
        0
      for: 30m
      labels:
        severity: warning
    - alert: DiskRunsOutOfSpace
      annotations:
        dashboard: '{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=20&var-instance={{ $labels.instance
          }}'
        description: |-
          Disk utilisation on instance {{ $labels.instance }} is more than 80%.
           Having less than 20% of free disk space could cripple merges processes and overall performance. Consider to limit the ingestion rate, decrease retention or scale the disk space if possible.
        summary: Instance {{ $labels.instance }} (job={{ $labels.job }}) will run out of
          disk space soon
      expr: (sum(vm_data_size_bytes) by(job,instance,cluster) / (sum(vm_free_disk_space_bytes)
        by(job,instance,cluster) + sum(vm_data_size_bytes) by(job,instance,cluster))) >
        0.8
      for: 30m
      labels:
        severity: critical
    - alert: RequestErrorsToAPI
      annotations:
        dashboard: '{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance
          }}'
        description: Requests to path {{ $labels.path }} are receiving errors. Please verify
          if clients are sending correct requests.
        summary: Too many errors served for {{ $labels.job }} path {{ $labels.path }} (instance
          {{ $labels.instance }})
      expr: increase(vm_http_request_errors_total[5m]) > 0
      for: 15m
      labels:
        severity: warning
        show_at: dashboard
    - alert: RPCErrors
      annotations:
        dashboard: '{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=44&var-instance={{ $labels.instance
          }}'
        description: |-
          RPC errors are interconnection errors between cluster components.
           Possible reasons for errors are misconfiguration, overload, network blips or unreachable components.
        summary: Too many RPC errors for {{ $labels.job }} (instance {{ $labels.instance
          }})
      expr: ((sum(increase(vm_rpc_connection_errors_total[5m])) by(job,instance,cluster)
        + sum(increase(vm_rpc_dial_errors_total[5m])) by(job,instance,cluster)) + sum(increase(vm_rpc_handshake_errors_total[5m]))
        by(job,instance,cluster)) > 0
      for: 15m
      labels:
        severity: warning
        show_at: dashboard
    - alert: TooHighChurnRate
      annotations:
        dashboard: '{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=102'
        description: |-
          VM constantly creates new time series.
           This effect is known as Churn Rate.
           High Churn Rate tightly connected with database performance and may result in unexpected OOM's or slow queries.
        summary: Churn rate is more than 10% for the last 15m
      expr: (sum(rate(vm_new_timeseries_created_total[5m])) by(job,cluster) / sum(rate(vm_rows_inserted_total[5m]))
        by(job,cluster)) > 0.1
      for: 15m
      labels:
        severity: warning
    - alert: TooHighChurnRate24h
      annotations:
        dashboard: '{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=102'
        description: |-
          The number of created new time series over last 24h is 3x times higher than current number of active series.
           This effect is known as Churn Rate.
           High Churn Rate tightly connected with database performance and may result in unexpected OOM's or slow queries.
        summary: Too high number of new series created over last 24h
      expr: sum(increase(vm_new_timeseries_created_total[24h])) by(job,cluster) > (sum(vm_cache_entries{type="storage/hour_metric_ids"})
        by(job,cluster) * 3)
      for: 15m
      labels:
        severity: warning
    - alert: TooHighSlowInsertsRate
      annotations:
        dashboard: '{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=108'
        description: High rate of slow inserts may be a sign of resource exhaustion for
          the current load. It is likely more RAM is needed for optimal handling of the
          current number of active time series. See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183
        summary: Percentage of slow inserts is more than 5% for the last 15m
      expr: (sum(rate(vm_slow_row_inserts_total[5m])) by(job,cluster) / sum(rate(vm_rows_inserted_total[5m]))
        by(job,cluster)) > 0.05
      for: 15m
      labels:
        severity: warning
    - alert: VminsertVmstorageConnectionIsSaturated
      annotations:
        dashboard: '{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=139&var-instance={{ $labels.instance
          }}'
        description: |-
          The connection between vminsert (instance {{ $labels.instance }}) and vmstorage (instance {{ $labels.addr }}) is saturated by more than 90% and vminsert won't be able to keep up.
           This usually means that more vminsert or vmstorage nodes must be added to the cluster in order to increase the total number of vminsert -> vmstorage links.
        summary: Connection between vminsert on {{ $labels.instance }} and vmstorage on
          {{ $labels.addr }} is saturated
      expr: rate(vm_rpc_send_duration_seconds_total[5m]) > 0.9
      for: 15m
      labels:
        severity: warning
        show_at: dashboard
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-vmoperator
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - name: vmoperator
    params: {}
    rules:
    - alert: LogErrors
      annotations:
        dashboard: '{{ $externalURL }}/d/1H179hunk/victoriametrics-operator?ds={{ $labels.dc
          }}&orgId=1&viewPanel=16'
        description: 'Operator has too many errors at logs: {{ $value}}, check operator
          logs'
        summary: 'Too many errors at logs of operator: {{ $value}}'
      expr: sum(rate(operator_log_messages_total{level="error",job=~".*((victoria.*)|vm)-?operator"}[5m]))
        by(cluster) > 0
      for: 15m
      labels:
        severity: warning
        show_at: dashboard
    - alert: ReconcileErrors
      annotations:
        dashboard: '{{ $externalURL }}/d/1H179hunk/victoriametrics-operator?ds={{ $labels.dc
          }}&orgId=1&viewPanel=10'
        description: 'Operator cannot parse response from k8s api server, possible bug:
          {{ $value }}, check operator logs'
        summary: 'Too many errors at reconcile loop of operator: {{ $value}}'
      expr: sum(rate(controller_runtime_reconcile_errors_total{job=~".*((victoria.*)|vm)-?operator"}[5m]))
        by(cluster) > 0
      for: 10m
      labels:
        severity: warning
        show_at: dashboard
    - alert: HighQueueDepth
      annotations:
        dashboard: '{{ $externalURL }}/d/1H179hunk/victoriametrics-operator?ds={{ $labels.dc
          }}&orgId=1&viewPanel=20'
        description: 'Operator cannot handle reconciliation load for controller: `{{- $labels.name
          }}`, current depth: {{ $value }}'
        summary: 'Too many `{{- $labels.name }}` in queue: {{ $value }}'
      expr: sum(workqueue_depth{job=~".*((victoria.*)|vm)-?operator",name=~"(vmagent|vmalert|vmalertmanager|vmauth|vmcluster|vmnodescrape|vmpodscrape|vmprobe|vmrule|vmservicescrape|vmsingle|vmstaticscrape)"})
        by(name,cluster) > 10
      for: 15m
      labels:
        severity: warning
        show_at: dashboard
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmalert/rule.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  namespace: default
  name: vmks-vmsingle
  labels: 
    app: vmks
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v0.28.1
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec:
  groups:
  - concurrency: 2
    interval: 30s
    name: vmsingle
    params: {}
    rules:
    - alert: DiskRunsOutOfSpaceIn3Days
      annotations:
        dashboard: '{{ $externalURL }}/d/wNf0q_kZk?viewPanel=53&var-instance={{ $labels.instance
          }}'
        description: |-
          Taking into account current ingestion rate, free disk space will be enough only for {{ $value | humanizeDuration }} on instance {{ $labels.instance }}.
           Consider to limit the ingestion rate, decrease retention or scale the disk space if possible.
        summary: Instance {{ $labels.instance }} will run out of disk space soon
      expr: ((sum(vm_free_disk_space_bytes) without(path) / (((rate(vm_rows_added_to_storage_total[1d])
        - sum(rate(vm_deduplicated_samples_total[1d])) without(type)) * (sum(vm_data_size_bytes{type!~"indexdb.*"})
        without(type) / sum(vm_rows{type!~"indexdb.*"}) without(type))) + (rate(vm_new_timeseries_created_total[1d])
        * (sum(vm_data_size_bytes{type="indexdb/file"}) without(type) / sum(vm_rows{type="indexdb/file"})
        without(type))))) < 259200) > 0
      for: 30m
      labels:
        severity: critical
    - alert: NodeBecomesReadonlyIn3Days
      annotations:
        dashboard: '{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=53&var-instance={{ $labels.instance
          }}'
        description: |-
          Taking into account current ingestion rate and free disk space instance {{ $labels.instance }} is writable for {{ $value | humanizeDuration }}.
           Consider to limit the ingestion rate, decrease retention or scale the disk space up if possible.
        summary: Instance {{ $labels.instance }} will become read-only in 3 days
      expr: ((sum(vm_free_disk_space_bytes - vm_free_disk_space_limit_bytes) without(path)
        / (((rate(vm_rows_added_to_storage_total[1d]) - sum(rate(vm_deduplicated_samples_total[1d]))
        without(type)) * (sum(vm_data_size_bytes{type!~"indexdb.*"}) without(type) / sum(vm_rows{type!~"indexdb.*"})
        without(type))) + (rate(vm_new_timeseries_created_total[1d]) * (sum(vm_data_size_bytes{type="indexdb/file"})
        without(type) / sum(vm_rows{type="indexdb/file"}) without(type))))) < 259200) >
        0
      for: 30m
      labels:
        severity: warning
    - alert: DiskRunsOutOfSpace
      annotations:
        dashboard: '{{ $externalURL }}/d/wNf0q_kZk?viewPanel=53&var-instance={{ $labels.instance
          }}'
        description: |-
          Disk utilisation on instance {{ $labels.instance }} is more than 80%.
           Having less than 20% of free disk space could cripple merge processes and overall performance. Consider to limit the ingestion rate, decrease retention or scale the disk space if possible.
        summary: Instance {{ $labels.instance }} (job={{ $labels.job }}) will run out of
          disk space soon
      expr: (sum(vm_data_size_bytes) by(job,instance,cluster) / (sum(vm_free_disk_space_bytes)
        by(job,instance,cluster) + sum(vm_data_size_bytes) by(job,instance,cluster))) >
        0.8
      for: 30m
      labels:
        severity: critical
    - alert: RequestErrorsToAPI
      annotations:
        dashboard: '{{ $externalURL }}/d/wNf0q_kZk?viewPanel=35&var-instance={{ $labels.instance
          }}'
        description: Requests to path {{ $labels.path }} are receiving errors. Please verify
          if clients are sending correct requests.
        summary: Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance
          }})
      expr: increase(vm_http_request_errors_total[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: TooHighChurnRate
      annotations:
        dashboard: '{{ $externalURL }}/d/wNf0q_kZk?viewPanel=66&var-instance={{ $labels.instance
          }}'
        description: |-
          VM constantly creates new time series on "{{ $labels.instance }}".
           This effect is known as Churn Rate.
           High Churn Rate is tightly connected with database performance and may result in unexpected OOM's or slow queries.
        summary: Churn rate is more than 10% on "{{ $labels.instance }}" for the last 15m
      expr: (sum(rate(vm_new_timeseries_created_total[5m])) by(instance,cluster) / sum(rate(vm_rows_inserted_total[5m]))
        by(instance,cluster)) > 0.1
      for: 15m
      labels:
        severity: warning
    - alert: TooHighChurnRate24h
      annotations:
        dashboard: '{{ $externalURL }}/d/wNf0q_kZk?viewPanel=66&var-instance={{ $labels.instance
          }}'
        description: |-
          The number of created new time series over last 24h is 3x times higher than current number of active series on "{{ $labels.instance }}".
           This effect is known as Churn Rate.
           High Churn Rate is tightly connected with database performance and may result in unexpected OOM's or slow queries.
        summary: Too high number of new series on "{{ $labels.instance }}" created over
          last 24h
      expr: sum(increase(vm_new_timeseries_created_total[24h])) by(instance,cluster) > (sum(vm_cache_entries{type="storage/hour_metric_ids"})
        by(instance,cluster) * 3)
      for: 15m
      labels:
        severity: warning
    - alert: TooHighSlowInsertsRate
      annotations:
        dashboard: '{{ $externalURL }}/d/wNf0q_kZk?viewPanel=68&var-instance={{ $labels.instance
          }}'
        description: High rate of slow inserts on "{{ $labels.instance }}" may be a sign
          of resource exhaustion for the current load. It is likely more RAM is needed for
          optimal handling of the current number of active time series. See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183
        summary: Percentage of slow inserts is more than 5% on "{{ $labels.instance }}"
          for the last 15m
      expr: (sum(rate(vm_slow_row_inserts_total[5m])) by(instance,cluster) / sum(rate(vm_rows_inserted_total[5m]))
        by(instance,cluster)) > 0.05
      for: 15m
      labels:
        severity: warning
    - alert: MetadataCacheUtilizationIsTooHigh
      annotations:
        description: Metadata cache stores meta information about ingested time series -
          see https://docs.victoriametrics.com/victoriametrics/#metrics-metadata. When cache
          is overutilized, the oldest entries will be dropped out automatically. It may
          result into incomplete response for /api/v1/metadata API calls. It doesn't impact
          regular queries or alerts. Cache size is controlled via -storage.maxMetadataStorageSize
          cmd-line flag.
        summary: Metadata cache capacity on {{ $labels.instance }} (job={{ $labels.job }})
          is utilized for more than 95% for the last 15min
      expr: (vm_metrics_metadata_storage_size_bytes / vm_metrics_metadata_storage_max_size_bytes)
        > 0.95
      for: 15m
      labels:
        severity: warning
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/monitor.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
  name: release-name-victoria-metrics-operator
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
spec:
  selector:
    matchLabels: 
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/name: victoria-metrics-operator
  endpoints:
    - port: http
  namespaceSelector:
    matchNames:
      - default
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
  name: vmks-core-dns
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    port: http-metrics
  jobLabel: jobLabel
  namespaceSelector:
    matchNames:
    - kube-system
  selector:
    matchLabels:
      app: vmks-core-dns
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/name: vmks
      jobLabel: core-dns
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
  name: vmks-grafana
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  endpoints:
  - port: 'service'
  selector:
    matchLabels:
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/name: 'grafana'
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
  name: vmks-kube-state-metrics
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  endpoints:
  - honorLabels: true
    metricRelabelConfigs:
    - action: labeldrop
      regex: (uid|container_id|image_id)
    port: http
  jobLabel: app.kubernetes.io/name
  selector:
    matchLabels:
      app.kubernetes.io/instance: 'release-name'
      app.kubernetes.io/name: 'kube-state-metrics'
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
  name: vmks-kube-api-server
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    port: https
    scheme: https
    tlsConfig:
      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      serverName: kubernetes
  jobLabel: component
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      component: apiserver
      provider: kubernetes
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
  name: vmks-kube-controller-manager
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    port: http-metrics
    scheme: https
    tlsConfig:
      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      serverName: kubernetes
  jobLabel: jobLabel
  namespaceSelector:
    matchNames:
    - kube-system
  selector:
    matchLabels:
      app: vmks-kube-controller-manager
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/name: vmks
      jobLabel: kube-controller-manager
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
  name: vmks-kube-etcd
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    port: http-metrics
    scheme: https
    tlsConfig:
      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  jobLabel: jobLabel
  namespaceSelector:
    matchNames:
    - kube-system
  selector:
    matchLabels:
      app: vmks-kube-etcd
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/name: vmks
      jobLabel: kube-etcd
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
  name: vmks-kube-scheduler
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    port: http-metrics
    scheme: https
    tlsConfig:
      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  jobLabel: jobLabel
  namespaceSelector:
    matchNames:
    - kube-system
  selector:
    matchLabels:
      app: vmks-kube-scheduler
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/name: vmks
      jobLabel: kube-scheduler
---
# Source: victoria-metrics-k8s-stack/templates/servicemonitors.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
  name: vmks-prometheus-node-exporter
  namespace: default
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  endpoints:
  - metricRelabelConfigs:
    - action: drop
      regex: /var/lib/kubelet/pods.+
      source_labels:
      - mountpoint
    port: metrics
  jobLabel: jobLabel
  selector:
    matchLabels:
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/name: 'prometheus-node-exporter'
---
# Source: victoria-metrics-k8s-stack/templates/victoria-metrics-operator/vmsingle/vmsingle.yml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMSingle
metadata:
  name: vmks
  namespace: default
  labels: 
    app.kubernetes.io/component: vmks-vmsingle
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: vmks
    app.kubernetes.io/version: v1.138.0
    helm.sh/chart: victoria-metrics-k8s-stack-0.72.5
spec: 
  extraArgs:
    vmalert.proxyURL: http://vmalert-vmks.default.svc.cluster.local.:8080
  image:
    tag: v1.138.0
  port: "8428"
  replicaCount: 1
  retentionPeriod: "1"
  storage:
    accessModes:
    - ReadWriteOnce
    resources:
      requests:
        storage: 20Gi
---
# Source: victoria-metrics-k8s-stack/charts/victoria-metrics-operator/templates/webhook.yaml
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
  name:  release-name-victoria-metrics-operator-admission
  labels: 
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: victoria-metrics-operator
    app.kubernetes.io/version: v0.68.3
    helm.sh/chart: victoria-metrics-operator-0.59.3
webhooks:
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1-vlagent
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vlagents.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vlagents
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1-vlcluster
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vlclusters.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vlclusters
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vlogs
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vlogs.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vlogs
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1-vlsingle
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vlsingles.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vlsingles
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmagent
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmagents.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmagents
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmalertmanagerconfig
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmalertmanagerconfigs.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmalertmanagerconfigs
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmalertmanager
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmalertmanagers.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmalertmanagers
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmalert
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmalerts.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmalerts
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1-vmanomaly
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmanomalies.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmanomalies
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmauth
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmauths.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmauths
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmcluster
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmclusters.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmclusters
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1alpha1-vmdistributed
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmdistributed.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1alpha1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmdistributed
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmnodescrape
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmnodescrapes.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmnodescrapes
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmpodscrape
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmpodscrapes.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmpodscrapes
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmprobe
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmprobes.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmprobes
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmrule
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmrules.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmrules
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmscrapeconfig
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmscrapeconfigs.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmscrapeconfigs
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmservicescrape
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmservicescrapes.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmservicescrapes
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmsingle
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmsingles.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmsingles
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmstaticscrape
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmstaticscrapes.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmstaticscrapes
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1beta1-vmuser
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vmusers.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vmusers
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1-vtcluster
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vtclusters.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vtclusters
  - clientConfig:
      service:
        namespace: default
        name: release-name-victoria-metrics-operator
        path: /validate-operator-victoriametrics-com-v1-vtsingle
        port: 9443
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUlQTEo4dE5Lc2E3dHNhV0tEL3lPbTR3RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qWXdOVEV4TURVMU5qTXlXaGNOTXpZdwpOVEE0TURVMU5qTXlXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTVIxeXZTa1Y3VWNLUGlld1pscXp0VWhuRk5HZ1AyZ0NlbUoKRHByMno2d1hLYTFjNm44cUtyUnpJS2I4aTBPVzRCN0NZcEFHblhiN3ZaRzNaMzk3VWJxNTJaTU54dzRybmpQKwo2ODlDMkl2eEpTOW5WUkQ2b3VBV2pUL2llTmlyS2VWUmIrb0R1M25qdUVTWmUwaVZldHQzMUMvUElDTjg0MFhCCnBrdHFUalIxSXB6a2JWenlNUjJBMFp0SjNDc1g3U3FYRkhPaVhZV1IzVzB2TVNRSFk1bDdPcVBZTXRhYTZPRjQKNUJxNmQvQnovMTgrS3poeTNJdUNwc2RZTE9kUE5RajJ6SVYzWG5ZVUxFWkZqMkRMbVlGUDJHL2tad0x3R1EySwp0QW9hSURxS0hoWU5WTmJQNGtoQWRVVDlNY0ovU2duNDUwOXI5cTQ5K3c1cjRTZTk4T01DQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlM3d0t1VWRTZjFzOGJBRE9zakl1b0xHaWFkQmpBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFqYWo4dW5JZFJHT3piM05QcFVWM011M1h6M09BY0U1Y1lLenB1WXN5ClZNcmZjb2Q4b1dURVNkTE5hU1dIS0xEQ2hsa0RLSEdMOXNoc2liQlZvOTNwbFNGZG1ocCs5aS9iMDlBUi9kNzYKTzhTalJwbklDemxiNEloSXN0TFFXNFhSTWVJSEh1WDdMV0lQbEJOT25IUnp1OXIvVkJIc3J3OTRnRXpmeE5acwo5YlFQVE5YZFFNcVFBd1l3aUZOVzFaUnczYWQwMXU4MVRBcFZIZDZ5V0YzcW05SlE3MmhaNXcrUjJLdHkxcTlHCkdGa0l3eVpHMUpXWTU2UmYwd095b1RKRXlMNHFRSnprRHFtelAwNkNPOEFyazJmSCtQRUMzWS9HNzM5Q3VKV2kKSExtVUJwb3BFbnBid05qdmt5c2c4V0hyM2kzYS92NlVyUllZck1wSU1Rc056Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    failurePolicy: Fail
    name: 'vtsingles.operator.victoriametrics.com'
    admissionReviewVersions:
      - v1
    sideEffects: None
    objectSelector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: NotIn
          values: [victoria-metrics-operator]
    rules:
      - apiGroups:
          - operator.victoriametrics.com
        apiVersions:
          - v1
        operations:
          - CREATE
          - UPDATE
        resources:
          - vtsingles
