From 30af3165e45b5831f97f826250f57aac7f7a01a5 Mon Sep 17 00:00:00 2001 From: admrene Date: Sat, 25 Apr 2026 14:43:45 +0000 Subject: [PATCH] init commit --- grafana-alerting/README.md | 25 + grafana-alerting/contact-points.n8n.yaml | 12 + .../grafana-alerting-traefik.configmap.yaml | 507 + ...metheus-stack-values-alerting-snippet.yaml | 12 + grafana-alerting/traefik-alert-rules.yaml | 487 + .../00-namespaces.yaml | 14 + live-monitoring-deployments/README.md | 31 + .../alertmanager/01-core.yaml | 88 + .../alertmanager/02-services.yaml | 91 + .../alertmanager/03-statefulset.yaml | 317 + .../grafana/01-rbac.yaml | 104 + .../grafana/02-core-config.yaml | 83 + .../grafana/03-services.yaml | 78 + .../grafana/04-statefulset.yaml | 333 + .../grafana/05-provisioning-configmaps.yaml | 8158 +++++++++++++++++ live-monitoring-deployments/loki/01-core.yaml | 71 + .../loki/02-services.yaml | 50 + .../loki/03-statefulset.yaml | 100 + .../prometheus/01-rbac.yaml | 102 + .../prometheus/02-config.yaml | 3198 +++++++ .../prometheus/03-services.yaml | 80 + .../prometheus/04-statefulset.yaml | 302 + .../promtail/01-rbac.yaml | 39 + .../promtail/02-config.yaml | 50 + .../promtail/03-daemonset.yaml | 84 + traefik-grafana-dashboard.configmap.yaml | 2213 +++++ 26 files changed, 16629 insertions(+) create mode 100644 grafana-alerting/README.md create mode 100644 grafana-alerting/contact-points.n8n.yaml create mode 100644 grafana-alerting/grafana-alerting-traefik.configmap.yaml create mode 100644 grafana-alerting/kube-prometheus-stack-values-alerting-snippet.yaml create mode 100644 grafana-alerting/traefik-alert-rules.yaml create mode 100644 live-monitoring-deployments/00-namespaces.yaml create mode 100644 live-monitoring-deployments/README.md create mode 100644 live-monitoring-deployments/alertmanager/01-core.yaml create mode 100644 live-monitoring-deployments/alertmanager/02-services.yaml create mode 100644 live-monitoring-deployments/alertmanager/03-statefulset.yaml create mode 100644 live-monitoring-deployments/grafana/01-rbac.yaml create mode 100644 live-monitoring-deployments/grafana/02-core-config.yaml create mode 100644 live-monitoring-deployments/grafana/03-services.yaml create mode 100644 live-monitoring-deployments/grafana/04-statefulset.yaml create mode 100644 live-monitoring-deployments/grafana/05-provisioning-configmaps.yaml create mode 100644 live-monitoring-deployments/loki/01-core.yaml create mode 100644 live-monitoring-deployments/loki/02-services.yaml create mode 100644 live-monitoring-deployments/loki/03-statefulset.yaml create mode 100644 live-monitoring-deployments/prometheus/01-rbac.yaml create mode 100644 live-monitoring-deployments/prometheus/02-config.yaml create mode 100644 live-monitoring-deployments/prometheus/03-services.yaml create mode 100644 live-monitoring-deployments/prometheus/04-statefulset.yaml create mode 100644 live-monitoring-deployments/promtail/01-rbac.yaml create mode 100644 live-monitoring-deployments/promtail/02-config.yaml create mode 100644 live-monitoring-deployments/promtail/03-daemonset.yaml create mode 100644 traefik-grafana-dashboard.configmap.yaml diff --git a/grafana-alerting/README.md b/grafana-alerting/README.md new file mode 100644 index 0000000..79dd874 --- /dev/null +++ b/grafana-alerting/README.md @@ -0,0 +1,25 @@ +# Grafana Alerting Provisioning + +Dieses Verzeichnis enthaelt die aktuell in Grafana gepflegten Traefik-Alerts als file-based Provisioning-Dateien. + +## Dateien + +- `traefik-alert-rules.yaml`: Export der Rule Group `traefik-prometheus` im Folder `Traefik Alerts` +- `contact-points.n8n.yaml`: Kontaktpunkt `n8n` als Provisioning-Datei +- `grafana-alerting-traefik.configmap.yaml`: Kubernetes-ConfigMap mit beiden Dateien +- `kube-prometheus-stack-values-alerting-snippet.yaml`: Helm-Values-Snippet zum Einhaengen der ConfigMap in Grafana + +## Wichtiger Hinweis + +Grafana exportiert Secret-Felder wie Webhook-Passwoerter nur redigiert. Ersetze deshalb in `contact-points.n8n.yaml` und in der ConfigMap den Platzhalter `__REPLACE_WITH_N8N_WEBHOOK_PASSWORD__` durch das echte Secret. + +## Nutzung + +1. Passe den Passwort-Platzhalter an. +2. Wende die ConfigMap an. +3. Erweitere die `kube-prometheus-stack`-Values um das Mount-Snippet. +4. Rolle Grafana neu aus oder fuehre ein Helm-Upgrade durch. + +Die Regeln referenzieren den Receiver direkt ueber `notification_settings.receiver: n8n`, daher ist kein separater Policy-Export notwendig. + +File-provisioned Alerting-Ressourcen sind in der Grafana-UI anschliessend read-only. diff --git a/grafana-alerting/contact-points.n8n.yaml b/grafana-alerting/contact-points.n8n.yaml new file mode 100644 index 0000000..982b542 --- /dev/null +++ b/grafana-alerting/contact-points.n8n.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 +contactPoints: + - orgId: 1 + name: n8n + receivers: + - uid: dfjuxb8oyjh8ge + type: webhook + settings: + httpMethod: POST + url: https://n8n.apps.rsitservice.de/webhook/alerting + password: __REPLACE_WITH_N8N_WEBHOOK_PASSWORD__ + disableResolveMessage: false diff --git a/grafana-alerting/grafana-alerting-traefik.configmap.yaml b/grafana-alerting/grafana-alerting-traefik.configmap.yaml new file mode 100644 index 0000000..bb9de03 --- /dev/null +++ b/grafana-alerting/grafana-alerting-traefik.configmap.yaml @@ -0,0 +1,507 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-alerting-traefik + namespace: kube-mon +data: + traefik-alert-rules.yaml: | + apiVersion: 1 + groups: + - orgId: 1 + name: traefik-prometheus + folder: Traefik Alerts + interval: 1m + rules: + - uid: traefik-instance-down + title: Traefik Instance Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: max by (pod, instance) (up{job="traefik"} == 0) + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + range: true + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + settings: + mode: dropNN + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.5 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: traefik-overview + panelId: 5 + noDataState: OK + execErrState: Error + for: 5m + annotations: + __dashboardUid__: traefik-overview + __panelId__: "5" + description: Die Traefik-Instanz {{ $labels.pod }} auf {{ $labels.instance }} liefert seit 5 Minuten keine erfolgreichen Prometheus-Scrapes. + summary: Traefik-Instanz {{ $labels.pod }} ist nicht erreichbar + labels: + category: availability + cluster: microk8s + severity: critical + source: traefik + team: ops + isPaused: false + notification_settings: + receiver: n8n + - uid: traefik-websecure-5xx-ratio-high + title: Traefik Websecure 5xx Ratio High + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (100 * sum by (entrypoint) (rate(traefik_entrypoint_requests_total{entrypoint="websecure",code=~"5.."}[5m])) / clamp_min(sum by (entrypoint) (rate(traefik_entrypoint_requests_total{entrypoint="websecure"}[5m])), 0.001)) and on(entrypoint) (sum by (entrypoint) (rate(traefik_entrypoint_requests_total{entrypoint="websecure"}[5m])) > 0.3) + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + range: true + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + settings: + mode: dropNN + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 15 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: traefik-overview + panelId: 6 + noDataState: OK + execErrState: Error + for: 10m + annotations: + __dashboardUid__: traefik-overview + __panelId__: "6" + description: Die 5xx-Quote auf {{ $labels.entrypoint }} liegt seit 10 Minuten ueber 15% (aktuell {{ printf "%.2f" $values.B.Value }}%). + summary: Traefik Websecure 5xx-Quote ist erhoeht + labels: + category: errors + cluster: microk8s + severity: warning + source: traefik + team: ops + isPaused: false + notification_settings: + receiver: n8n + - uid: traefik-router-5xx-ratio-critical + title: Traefik Router 5xx Ratio Critical + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (100 * sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal",code=~"5.."}[5m])) / clamp_min(sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal"}[5m])), 0.001)) and on(router) (sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal"}[5m])) > 0.1) + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + range: true + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + settings: + mode: dropNN + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 50 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: traefik-overview + panelId: 11 + noDataState: OK + execErrState: Error + for: 10m + annotations: + __dashboardUid__: traefik-overview + __panelId__: "11" + description: Der Router {{ $labels.router }} liegt seit 10 Minuten ueber 50% 5xx-Quote bei relevanter Last (aktuell {{ printf "%.2f" $values.B.Value }}%). + summary: Traefik-Router {{ $labels.router }} produziert viele 5xx + labels: + category: errors + cluster: microk8s + severity: critical + source: traefik + team: ops + isPaused: false + notification_settings: + receiver: n8n + - uid: traefik-router-p95-latency-high + title: Traefik Router P95 Latency High + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (1000 * histogram_quantile(0.95, sum by (router, le) (rate(traefik_router_request_duration_seconds_bucket{router!~"ping@internal|prometheus@internal"}[5m])))) and on(router) (sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal"}[5m])) > 0.05) + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + range: true + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + settings: + mode: dropNN + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 3000 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: traefik-overview + panelId: 44 + noDataState: OK + execErrState: Error + for: 10m + annotations: + __dashboardUid__: traefik-overview + __panelId__: "44" + description: Der Router {{ $labels.router }} hat seit 10 Minuten eine p95-Latenz ueber 3000 ms (aktuell {{ printf "%.0f" $values.B.Value }} ms). + summary: Traefik-Router {{ $labels.router }} ist langsam + labels: + category: latency + cluster: microk8s + severity: warning + source: traefik + team: ops + isPaused: false + notification_settings: + receiver: n8n + - uid: traefik-cert-expiry-soon + title: Traefik Certificate Expiry Soon + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (max by (cn) (traefik_tls_certs_not_after) - time()) / 86400 + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + range: true + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + settings: + mode: dropNN + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 30 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: traefik-overview + panelId: 46 + noDataState: OK + execErrState: Error + for: 1h + annotations: + __dashboardUid__: traefik-overview + __panelId__: "46" + description: Das Zertifikat {{ $labels.cn }} laeuft in weniger als 30 Tagen ab (aktuell {{ printf "%.1f" $values.B.Value }} Tage Restlaufzeit). + summary: Traefik-Zertifikat {{ $labels.cn }} laeuft bald ab + labels: + category: certificate + cluster: microk8s + severity: warning + source: traefik + team: ops + isPaused: false + notification_settings: + receiver: n8n + contact-points.n8n.yaml: | + apiVersion: 1 + contactPoints: + - orgId: 1 + name: n8n + receivers: + - uid: dfjuxb8oyjh8ge + type: webhook + settings: + httpMethod: POST + url: https://n8n.apps.rsitservice.de/webhook/alerting + password: __REPLACE_WITH_N8N_WEBHOOK_PASSWORD__ + disableResolveMessage: false diff --git a/grafana-alerting/kube-prometheus-stack-values-alerting-snippet.yaml b/grafana-alerting/kube-prometheus-stack-values-alerting-snippet.yaml new file mode 100644 index 0000000..80fe9ac --- /dev/null +++ b/grafana-alerting/kube-prometheus-stack-values-alerting-snippet.yaml @@ -0,0 +1,12 @@ +grafana: + extraConfigmapMounts: + - name: grafana-alerting-traefik-rules + configMap: grafana-alerting-traefik + subPath: traefik-alert-rules.yaml + mountPath: /etc/grafana/provisioning/alerting/traefik-alert-rules.yaml + readOnly: true + - name: grafana-alerting-traefik-contact-point + configMap: grafana-alerting-traefik + subPath: contact-points.n8n.yaml + mountPath: /etc/grafana/provisioning/alerting/contact-points.n8n.yaml + readOnly: true diff --git a/grafana-alerting/traefik-alert-rules.yaml b/grafana-alerting/traefik-alert-rules.yaml new file mode 100644 index 0000000..4b20407 --- /dev/null +++ b/grafana-alerting/traefik-alert-rules.yaml @@ -0,0 +1,487 @@ +apiVersion: 1 +groups: + - orgId: 1 + name: traefik-prometheus + folder: Traefik Alerts + interval: 1m + rules: + - uid: traefik-instance-down + title: Traefik Instance Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: max by (pod, instance) (up{job="traefik"} == 0) + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + range: true + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + settings: + mode: dropNN + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.5 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: traefik-overview + panelId: 5 + noDataState: OK + execErrState: Error + for: 5m + annotations: + __dashboardUid__: traefik-overview + __panelId__: "5" + description: Die Traefik-Instanz {{ $labels.pod }} auf {{ $labels.instance }} liefert seit 5 Minuten keine erfolgreichen Prometheus-Scrapes. + summary: Traefik-Instanz {{ $labels.pod }} ist nicht erreichbar + labels: + category: availability + cluster: microk8s + severity: critical + source: traefik + team: ops + isPaused: false + notification_settings: + receiver: n8n + - uid: traefik-websecure-5xx-ratio-high + title: Traefik Websecure 5xx Ratio High + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (100 * sum by (entrypoint) (rate(traefik_entrypoint_requests_total{entrypoint="websecure",code=~"5.."}[5m])) / clamp_min(sum by (entrypoint) (rate(traefik_entrypoint_requests_total{entrypoint="websecure"}[5m])), 0.001)) and on(entrypoint) (sum by (entrypoint) (rate(traefik_entrypoint_requests_total{entrypoint="websecure"}[5m])) > 0.3) + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + range: true + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + settings: + mode: dropNN + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 15 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: traefik-overview + panelId: 6 + noDataState: OK + execErrState: Error + for: 10m + annotations: + __dashboardUid__: traefik-overview + __panelId__: "6" + description: Die 5xx-Quote auf {{ $labels.entrypoint }} liegt seit 10 Minuten ueber 15% (aktuell {{ printf "%.2f" $values.B.Value }}%). + summary: Traefik Websecure 5xx-Quote ist erhoeht + labels: + category: errors + cluster: microk8s + severity: warning + source: traefik + team: ops + isPaused: false + notification_settings: + receiver: n8n + - uid: traefik-router-5xx-ratio-critical + title: Traefik Router 5xx Ratio Critical + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (100 * sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal",code=~"5.."}[5m])) / clamp_min(sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal"}[5m])), 0.001)) and on(router) (sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal"}[5m])) > 0.1) + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + range: true + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + settings: + mode: dropNN + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 50 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: traefik-overview + panelId: 11 + noDataState: OK + execErrState: Error + for: 10m + annotations: + __dashboardUid__: traefik-overview + __panelId__: "11" + description: Der Router {{ $labels.router }} liegt seit 10 Minuten ueber 50% 5xx-Quote bei relevanter Last (aktuell {{ printf "%.2f" $values.B.Value }}%). + summary: Traefik-Router {{ $labels.router }} produziert viele 5xx + labels: + category: errors + cluster: microk8s + severity: critical + source: traefik + team: ops + isPaused: false + notification_settings: + receiver: n8n + - uid: traefik-router-p95-latency-high + title: Traefik Router P95 Latency High + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (1000 * histogram_quantile(0.95, sum by (router, le) (rate(traefik_router_request_duration_seconds_bucket{router!~"ping@internal|prometheus@internal"}[5m])))) and on(router) (sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal"}[5m])) > 0.05) + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + range: true + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + settings: + mode: dropNN + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 3000 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: traefik-overview + panelId: 44 + noDataState: OK + execErrState: Error + for: 10m + annotations: + __dashboardUid__: traefik-overview + __panelId__: "44" + description: Der Router {{ $labels.router }} hat seit 10 Minuten eine p95-Latenz ueber 3000 ms (aktuell {{ printf "%.0f" $values.B.Value }} ms). + summary: Traefik-Router {{ $labels.router }} ist langsam + labels: + category: latency + cluster: microk8s + severity: warning + source: traefik + team: ops + isPaused: false + notification_settings: + receiver: n8n + - uid: traefik-cert-expiry-soon + title: Traefik Certificate Expiry Soon + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: (max by (cn) (traefik_tls_certs_not_after) - time()) / 86400 + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + range: true + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + settings: + mode: dropNN + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 30 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: traefik-overview + panelId: 46 + noDataState: OK + execErrState: Error + for: 1h + annotations: + __dashboardUid__: traefik-overview + __panelId__: "46" + description: Das Zertifikat {{ $labels.cn }} laeuft in weniger als 30 Tagen ab (aktuell {{ printf "%.1f" $values.B.Value }} Tage Restlaufzeit). + summary: Traefik-Zertifikat {{ $labels.cn }} laeuft bald ab + labels: + category: certificate + cluster: microk8s + severity: warning + source: traefik + team: ops + isPaused: false + notification_settings: + receiver: n8n diff --git a/live-monitoring-deployments/00-namespaces.yaml b/live-monitoring-deployments/00-namespaces.yaml new file mode 100644 index 0000000..5f204f4 --- /dev/null +++ b/live-monitoring-deployments/00-namespaces.yaml @@ -0,0 +1,14 @@ +--- +"kind": "Namespace" +"apiVersion": "v1" +"metadata": + "name": "kube-mon" + "labels": + "kubernetes.io/metadata.name": "kube-mon" +--- +"kind": "Namespace" +"apiVersion": "v1" +"metadata": + "name": "loki" + "labels": + "kubernetes.io/metadata.name": "loki" diff --git a/live-monitoring-deployments/README.md b/live-monitoring-deployments/README.md new file mode 100644 index 0000000..691447e --- /dev/null +++ b/live-monitoring-deployments/README.md @@ -0,0 +1,31 @@ +# Live Monitoring Deployment Export + +Diese Manifeste wurden am 2026-04-25 14:31:07 UTC direkt aus den aktuell laufenden Instanzen im Cluster exportiert und von Laufzeitfeldern bereinigt. + +## Enthalten + +- `00-namespaces.yaml`: Namespaces `kube-mon` und `loki` +- `alertmanager/`: Alertmanager StatefulSet, Services, ServiceAccount und die aktuell genutzten Config-/TLS-Secrets +- `grafana/`: Grafana StatefulSet, Services, RBAC, Secret, Core-Config und alle aktuell gelabelten Grafana-Dashboard-/Datasource-ConfigMaps aus `kube-mon` +- `promtail/`: Promtail DaemonSet, ConfigMap, ServiceAccount und Cluster-RBAC aus `loki` +- `prometheus/`: Prometheus StatefulSet, Services, RBAC, Config-Secrets und das aktuell geladene Rulefile-ConfigMap +- `loki/`: Loki StatefulSet, Services, ServiceAccount, ConfigMap und PVC + +## Wichtige Hinweise + +- Der Export bildet die aktuell laufenden Instanzen moeglichst nah ab, aber nicht die urspruenglichen Helm-Releases oder Operator-CRDs. +- Grafana-Datasources werden unveraendert exportiert. Die aktuelle Prometheus-Datasource verweist zusaetzlich auf Alertmanager; der zugehoerige StatefulSet-/Service-/Secret-Stand ist in diesem Paket enthalten. +- Alertmanager- und Promtail-Konfigurationen werden inklusive der aktuell im Cluster hinterlegten Secrets bzw. Push-Ziele exportiert. Fuer andere Umgebungen muessen diese Werte ggf. angepasst oder vor einem Commit redigiert werden. +- Die Prometheus-Konfiguration ist als aktuell generiertes Secret exportiert. In einer anderen Umgebung koennen Scrape-Targets fehlen, wenn dort Cluster-Services, Endpoints oder Exporter anders heissen. +- Persistenz bleibt auf den aktuell genutzten StorageClasses (`nfs-csi`) und Groessen (`Grafana 20Gi`, `Prometheus 50Gi`, `Loki 20Gi`). +- Alertmanager ist wie im Live-Cluster als Operator-generiertes StatefulSet exportiert. Die zugehoerigen CRDs/CRs und der Prometheus-Operator selbst sind weiterhin nicht Teil dieses Pakets. +- NodePort-Services wurden mit den aktuell gesetzten Ports exportiert (`Grafana 30637`, `Prometheus 31595`, `Alertmanager 32626`, `Loki 31940/30320`). + +## Reihenfolge + +1. `00-namespaces.yaml` +2. `loki/01-core.yaml`, `loki/02-services.yaml`, `loki/03-statefulset.yaml` +3. `promtail/01-rbac.yaml`, `promtail/02-config.yaml`, `promtail/03-daemonset.yaml` +4. `alertmanager/01-core.yaml`, `alertmanager/02-services.yaml`, `alertmanager/03-statefulset.yaml` +5. `prometheus/01-rbac.yaml`, `prometheus/02-config.yaml`, `prometheus/03-services.yaml`, `prometheus/04-statefulset.yaml` +6. `grafana/01-rbac.yaml`, `grafana/02-core-config.yaml`, `grafana/03-services.yaml`, `grafana/04-statefulset.yaml`, `grafana/05-provisioning-configmaps.yaml` diff --git a/live-monitoring-deployments/alertmanager/01-core.yaml b/live-monitoring-deployments/alertmanager/01-core.yaml new file mode 100644 index 0000000..a6a71ac --- /dev/null +++ b/live-monitoring-deployments/alertmanager/01-core.yaml @@ -0,0 +1,88 @@ +--- +"kind": "ServiceAccount" +"apiVersion": "v1" +"metadata": + "name": "kube-prometheus-stack-1761-alertmanager" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-alertmanager" + "app.kubernetes.io/component": "alertmanager" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "kube-prometheus-stack-alertmanager" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"automountServiceAccountToken": true +--- +"kind": "Secret" +"apiVersion": "v1" +"metadata": + "name": "alertmanager-kube-prometheus-stack-1761-alertmanager" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-alertmanager" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "alertmanager.yaml": "Z2xvYmFsOgogIHJlc29sdmVfdGltZW91dDogNW0KICBzbXRwX2F1dGhfcGFzc3dvcmQ6IFJzMjAxMjE5ODgKICBzbXRwX2F1dGhfdXNlcm5hbWU6IGFsZXJ0aW5nQHJzaXRzZXJ2aWNlLmRlCiAgc210cF9mcm9tOiBhbGVydGluZ0Byc2l0c2VydmljZS5kZQogIHNtdHBfcmVxdWlyZV90bHM6IHRydWUKICBzbXRwX3NtYXJ0aG9zdDogbWFpbGNvdy5yc2l0c2VydmljZS5kZTo1ODcKaW5oaWJpdF9ydWxlczoKLSBlcXVhbDoKICAtIG5hbWVzcGFjZQogIC0gYWxlcnRuYW1lCiAgc291cmNlX21hdGNoZXJzOgogIC0gc2V2ZXJpdHkgPSBjcml0aWNhbAogIHRhcmdldF9tYXRjaGVyczoKICAtIHNldmVyaXR5ID1+IHdhcm5pbmd8aW5mbwotIGVxdWFsOgogIC0gbmFtZXNwYWNlCiAgLSBhbGVydG5hbWUKICBzb3VyY2VfbWF0Y2hlcnM6CiAgLSBzZXZlcml0eSA9IHdhcm5pbmcKICB0YXJnZXRfbWF0Y2hlcnM6CiAgLSBzZXZlcml0eSA9IGluZm8KLSBlcXVhbDoKICAtIG5hbWVzcGFjZQogIHNvdXJjZV9tYXRjaGVyczoKICAtIGFsZXJ0bmFtZSA9IEluZm9JbmhpYml0b3IKICB0YXJnZXRfbWF0Y2hlcnM6CiAgLSBzZXZlcml0eSA9IGluZm8KLSB0YXJnZXRfbWF0Y2hlcnM6CiAgLSBhbGVydG5hbWUgPSBJbmZvSW5oaWJpdG9yCnJlY2VpdmVyczoKLSBuYW1lOiAibnVsbCIKcm91dGU6CiAgZ3JvdXBfYnk6CiAgLSBhbGVydG5hbWUKICAtIGNsdXN0ZXIKICAtIHNlcnZpY2UKICBncm91cF9pbnRlcnZhbDogNW0KICBncm91cF93YWl0OiAzMHMKICByZWNlaXZlcjogbWFpbC1hbGVydHMKICByZWNlaXZlcnM6CiAgLSBlbWFpbF9jb25maWdzOgogICAgLSBzZW5kX3Jlc29sdmVkOiB0cnVlCiAgICAgIHRvOiBpbmZvQHJzaXRzZXJ2aWNlLmRlCiAgICBuYW1lOiBtYWlsLWFsZXJ0cwogIHJlcGVhdF9pbnRlcnZhbDogMTJoCiAgcm91dGVzOgogIC0gbWF0Y2hlcnM6CiAgICAtIGFsZXJ0bmFtZSA9ICJXYXRjaGRvZyIKICAgIHJlY2VpdmVyOiAibnVsbCIKdGVtcGxhdGVzOgotIC9ldGMvYWxlcnRtYW5hZ2VyL2NvbmZpZy8qLnRtcGw=" +"type": "Opaque" +--- +"kind": "Secret" +"apiVersion": "v1" +"metadata": + "name": "alertmanager-kube-prometheus-stack-1761-alertmanager-generated" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/managed-by": "prometheus-operator" + "managed-by": "prometheus-operator" +"data": + "alertmanager.yaml.gz": "H4sIAAAAAAAA/7ySy2r0MAyF934KMcsf5k8vdBOYB5gn6DJoXMUR+JLKcoaB0mcvTtppCg3tqlvpWN+xjpxPJ/StARDKyU/UKQdKRVt4CEZSUVqalngiaWEXi/c7A+AklbE7XWp7DxED5REtVXF9lZf6Nw8BAqodSGZJFaEn0ToBDrB7rM2n5D4ZZ2Rt4f4mXysclWRCP5us7kZCXVVv7wbDceATayfFVzN7UBRH2q3he8g0kbBe4PAKZ5TI0b1w7JMByKmIpU09WGFli94A0HNZlvh1Eauf/cyHX2LfXf4FdZ3LMfbpuKw0yRZ8A7c95+M85oCq4nonSmH0qEt0Dalt5ikBIzqSxqbYs2v+/dcwevMWAAD//yheIjPGAgAA" +"type": "Opaque" +--- +"kind": "Secret" +"apiVersion": "v1" +"metadata": + "name": "alertmanager-kube-prometheus-stack-1761-alertmanager-web-config" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/managed-by": "prometheus-operator" + "managed-by": "prometheus-operator" +"data": + "web-config.yaml": "" +"type": "Opaque" +--- +"kind": "Secret" +"apiVersion": "v1" +"metadata": + "name": "alertmanager-kube-prometheus-stack-1761-alertmanager-tls-assets-0" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/managed-by": "prometheus-operator" + "managed-by": "prometheus-operator" +"type": "Opaque" +--- +"kind": "Secret" +"apiVersion": "v1" +"metadata": + "name": "alertmanager-kube-prometheus-stack-1761-alertmanager-cluster-tls-config" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/managed-by": "prometheus-operator" + "managed-by": "prometheus-operator" +"data": + "cluster-tls-config.yaml": "" +"type": "Opaque" diff --git a/live-monitoring-deployments/alertmanager/02-services.yaml b/live-monitoring-deployments/alertmanager/02-services.yaml new file mode 100644 index 0000000..5ab6145 --- /dev/null +++ b/live-monitoring-deployments/alertmanager/02-services.yaml @@ -0,0 +1,91 @@ +--- +"kind": "Service" +"apiVersion": "v1" +"metadata": + "name": "kube-prometheus-stack-1761-alertmanager" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-alertmanager" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "self-monitor": "true" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"spec": + "ports": + - + "name": "http-web" + "protocol": "TCP" + "port": 9093 + "targetPort": 9093 + - + "name": "reloader-web" + "protocol": "TCP" + "appProtocol": "http" + "port": 8080 + "targetPort": "reloader-web" + "selector": + "alertmanager": "kube-prometheus-stack-1761-alertmanager" + "app.kubernetes.io/name": "alertmanager" + "type": "ClusterIP" + "sessionAffinity": "None" +--- +"kind": "Service" +"apiVersion": "v1" +"metadata": + "name": "alertmanager-operated" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/managed-by": "prometheus-operator" + "managed-by": "prometheus-operator" + "operated-alertmanager": "true" +"spec": + "ports": + - + "name": "http-web" + "protocol": "TCP" + "port": 9093 + "targetPort": "http-web" + - + "name": "tcp-mesh" + "protocol": "TCP" + "port": 9094 + "targetPort": "mesh-tcp" + - + "name": "udp-mesh" + "protocol": "UDP" + "port": 9094 + "targetPort": "mesh-udp" + "selector": + "app.kubernetes.io/name": "alertmanager" + "clusterIP": "None" + "type": "ClusterIP" + "sessionAffinity": "None" + "publishNotReadyAddresses": true +--- +"kind": "Service" +"apiVersion": "v1" +"metadata": + "name": "alertmanager-nodeport" + "namespace": "kube-mon" + "labels": + "k8slens-edit-resource-version": "v1" +"spec": + "ports": + - + "name": "http-web" + "protocol": "TCP" + "port": 9093 + "targetPort": "http-web" + "nodePort": 32626 + "selector": + "app.kubernetes.io/name": "alertmanager" + "type": "NodePort" + "sessionAffinity": "None" + "externalTrafficPolicy": "Cluster" diff --git a/live-monitoring-deployments/alertmanager/03-statefulset.yaml b/live-monitoring-deployments/alertmanager/03-statefulset.yaml new file mode 100644 index 0000000..6b1f671 --- /dev/null +++ b/live-monitoring-deployments/alertmanager/03-statefulset.yaml @@ -0,0 +1,317 @@ +--- +"kind": "StatefulSet" +"apiVersion": "apps/v1" +"metadata": + "name": "alertmanager-kube-prometheus-stack-1761-alertmanager" + "namespace": "kube-mon" + "labels": + "alertmanager": "kube-prometheus-stack-1761-alertmanager" + "app": "kube-prometheus-stack-alertmanager" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761-alertmanager" + "app.kubernetes.io/managed-by": "prometheus-operator" + "app.kubernetes.io/name": "alertmanager" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "heritage": "Helm" + "managed-by": "prometheus-operator" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" + "prometheus-operator-input-hash": "12770747348952997375" +"spec": + "replicas": 1 + "selector": + "matchLabels": + "alertmanager": "kube-prometheus-stack-1761-alertmanager" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761-alertmanager" + "app.kubernetes.io/managed-by": "prometheus-operator" + "app.kubernetes.io/name": "alertmanager" + "template": + "metadata": + "labels": + "alertmanager": "kube-prometheus-stack-1761-alertmanager" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761-alertmanager" + "app.kubernetes.io/managed-by": "prometheus-operator" + "app.kubernetes.io/name": "alertmanager" + "app.kubernetes.io/version": "0.28.1" + "annotations": + "kubectl.kubernetes.io/default-container": "alertmanager" + "spec": + "volumes": + - + "name": "config-volume" + "secret": + "secretName": "alertmanager-kube-prometheus-stack-1761-alertmanager-generated" + "defaultMode": 420 + - + "name": "tls-assets" + "projected": + "sources": + - + "secret": + "name": "alertmanager-kube-prometheus-stack-1761-alertmanager-tls-assets-0" + "defaultMode": 420 + - + "name": "config-out" + "emptyDir": + "medium": "Memory" + - + "name": "web-config" + "secret": + "secretName": "alertmanager-kube-prometheus-stack-1761-alertmanager-web-config" + "defaultMode": 420 + - + "name": "cluster-tls-config" + "secret": + "secretName": "alertmanager-kube-prometheus-stack-1761-alertmanager-cluster-tls-config" + "defaultMode": 420 + "initContainers": + - + "name": "init-config-reloader" + "image": "quay.io/prometheus-operator/prometheus-config-reloader:v0.86.1" + "command": + - "/bin/prometheus-config-reloader" + "args": + - "--watch-interval=0" + - "--listen-address=:8081" + - "--config-file=/etc/alertmanager/config/alertmanager.yaml.gz" + - "--config-envsubst-file=/etc/alertmanager/config_out/alertmanager.env.yaml" + - "--watched-dir=/etc/alertmanager/config" + "ports": + - + "name": "reloader-init" + "containerPort": 8081 + "protocol": "TCP" + "env": + - + "name": "POD_NAME" + "valueFrom": + "fieldRef": + "apiVersion": "v1" + "fieldPath": "metadata.name" + - + "name": "SHARD" + "value": "-1" + "resources": {} + "volumeMounts": + - + "name": "config-volume" + "readOnly": true + "mountPath": "/etc/alertmanager/config" + - + "name": "config-out" + "mountPath": "/etc/alertmanager/config_out" + - + "name": "web-config" + "readOnly": true + "mountPath": "/etc/alertmanager/web_config/web-config.yaml" + "subPath": "web-config.yaml" + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "FallbackToLogsOnError" + "imagePullPolicy": "IfNotPresent" + "securityContext": + "capabilities": + "drop": + - "ALL" + "readOnlyRootFilesystem": true + "allowPrivilegeEscalation": false + "containers": + - + "name": "alertmanager" + "image": "quay.io/prometheus/alertmanager:v0.28.1" + "args": + - "--config.file=/etc/alertmanager/config_out/alertmanager.env.yaml" + - "--storage.path=/alertmanager" + - "--data.retention=120h" + - "--cluster.listen-address=" + - "--web.listen-address=:9093" + - "--web.external-url=http://kube-prometheus-stack-1761-alertmanager.kube-mon:9093" + - "--web.route-prefix=/" + - "--cluster.label=kube-mon/kube-prometheus-stack-1761-alertmanager" + - "--cluster.peer=alertmanager-kube-prometheus-stack-1761-alertmanager-0.alertmanager-operated:9094" + - "--cluster.reconnect-timeout=5m" + - "--web.config.file=/etc/alertmanager/web_config/web-config.yaml" + "ports": + - + "name": "http-web" + "containerPort": 9093 + "protocol": "TCP" + - + "name": "mesh-tcp" + "containerPort": 9094 + "protocol": "TCP" + - + "name": "mesh-udp" + "containerPort": 9094 + "protocol": "UDP" + "env": + - + "name": "POD_IP" + "valueFrom": + "fieldRef": + "apiVersion": "v1" + "fieldPath": "status.podIP" + "resources": + "requests": + "memory": "200Mi" + "volumeMounts": + - + "name": "config-volume" + "mountPath": "/etc/alertmanager/config" + - + "name": "config-out" + "readOnly": true + "mountPath": "/etc/alertmanager/config_out" + - + "name": "tls-assets" + "readOnly": true + "mountPath": "/etc/alertmanager/certs" + - + "name": "alertmanager-kube-prometheus-stack-1761-alertmanager-db" + "mountPath": "/alertmanager" + "subPath": "alertmanager-db" + - + "name": "web-config" + "readOnly": true + "mountPath": "/etc/alertmanager/web_config/web-config.yaml" + "subPath": "web-config.yaml" + - + "name": "cluster-tls-config" + "readOnly": true + "mountPath": "/etc/alertmanager/cluster_tls_config/cluster-tls-config.yaml" + "subPath": "cluster-tls-config.yaml" + "livenessProbe": + "httpGet": + "path": "/-/healthy" + "port": "http-web" + "scheme": "HTTP" + "timeoutSeconds": 3 + "periodSeconds": 10 + "successThreshold": 1 + "failureThreshold": 10 + "readinessProbe": + "httpGet": + "path": "/-/ready" + "port": "http-web" + "scheme": "HTTP" + "initialDelaySeconds": 3 + "timeoutSeconds": 3 + "periodSeconds": 5 + "successThreshold": 1 + "failureThreshold": 10 + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "FallbackToLogsOnError" + "imagePullPolicy": "IfNotPresent" + "securityContext": + "capabilities": + "drop": + - "ALL" + "readOnlyRootFilesystem": true + "allowPrivilegeEscalation": false + - + "name": "config-reloader" + "image": "quay.io/prometheus-operator/prometheus-config-reloader:v0.86.1" + "command": + - "/bin/prometheus-config-reloader" + "args": + - "--listen-address=:8080" + - "--web-config-file=/etc/alertmanager/web_config/web-config.yaml" + - "--reload-url=http://127.0.0.1:9093/-/reload" + - "--config-file=/etc/alertmanager/config/alertmanager.yaml.gz" + - "--config-envsubst-file=/etc/alertmanager/config_out/alertmanager.env.yaml" + - "--watched-dir=/etc/alertmanager/config" + "ports": + - + "name": "reloader-web" + "containerPort": 8080 + "protocol": "TCP" + "env": + - + "name": "POD_NAME" + "valueFrom": + "fieldRef": + "apiVersion": "v1" + "fieldPath": "metadata.name" + - + "name": "SHARD" + "value": "-1" + "resources": {} + "volumeMounts": + - + "name": "config-volume" + "readOnly": true + "mountPath": "/etc/alertmanager/config" + - + "name": "config-out" + "mountPath": "/etc/alertmanager/config_out" + - + "name": "web-config" + "readOnly": true + "mountPath": "/etc/alertmanager/web_config/web-config.yaml" + "subPath": "web-config.yaml" + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "FallbackToLogsOnError" + "imagePullPolicy": "IfNotPresent" + "securityContext": + "capabilities": + "drop": + - "ALL" + "readOnlyRootFilesystem": true + "allowPrivilegeEscalation": false + "restartPolicy": "Always" + "terminationGracePeriodSeconds": 120 + "dnsPolicy": "ClusterFirst" + "serviceAccountName": "kube-prometheus-stack-1761-alertmanager" + "serviceAccount": "kube-prometheus-stack-1761-alertmanager" + "automountServiceAccountToken": true + "securityContext": + "runAsUser": 1000 + "runAsGroup": 2000 + "runAsNonRoot": true + "fsGroup": 2000 + "seccompProfile": + "type": "RuntimeDefault" + "affinity": + "podAntiAffinity": + "preferredDuringSchedulingIgnoredDuringExecution": + - + "weight": 100 + "podAffinityTerm": + "labelSelector": + "matchExpressions": + - + "key": "app.kubernetes.io/name" + "operator": "In" + "values": + - "alertmanager" + - + "key": "alertmanager" + "operator": "In" + "values": + - "kube-prometheus-stack-1761-alertmanager" + "topologyKey": "kubernetes.io/hostname" + "schedulerName": "default-scheduler" + "volumeClaimTemplates": + - + "kind": "PersistentVolumeClaim" + "apiVersion": "v1" + "metadata": + "name": "alertmanager-kube-prometheus-stack-1761-alertmanager-db" + "spec": + "accessModes": + - "ReadWriteOnce" + "resources": + "requests": + "storage": "50Gi" + "storageClassName": "nfs-csi" + "volumeMode": "Filesystem" + "serviceName": "alertmanager-operated" + "podManagementPolicy": "Parallel" + "updateStrategy": + "type": "RollingUpdate" + "revisionHistoryLimit": 10 + "persistentVolumeClaimRetentionPolicy": + "whenDeleted": "Retain" + "whenScaled": "Retain" diff --git a/live-monitoring-deployments/grafana/01-rbac.yaml b/live-monitoring-deployments/grafana/01-rbac.yaml new file mode 100644 index 0000000..60b4821 --- /dev/null +++ b/live-monitoring-deployments/grafana/01-rbac.yaml @@ -0,0 +1,104 @@ +--- +"kind": "ServiceAccount" +"apiVersion": "v1" +"metadata": + "name": "kube-prometheus-stack-1761819498-grafana" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"automountServiceAccountToken": true +--- +"kind": "Role" +"apiVersion": "rbac.authorization.k8s.io/v1" +"metadata": + "name": "kube-prometheus-stack-1761819498-grafana" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"rules": [] +--- +"kind": "RoleBinding" +"apiVersion": "rbac.authorization.k8s.io/v1" +"metadata": + "name": "kube-prometheus-stack-1761819498-grafana" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"subjects": + - + "kind": "ServiceAccount" + "name": "kube-prometheus-stack-1761819498-grafana" + "namespace": "kube-mon" +"roleRef": + "apiGroup": "rbac.authorization.k8s.io" + "kind": "Role" + "name": "kube-prometheus-stack-1761819498-grafana" +--- +"kind": "ClusterRole" +"apiVersion": "rbac.authorization.k8s.io/v1" +"metadata": + "name": "kube-prometheus-stack-1761819498-grafana-clusterrole" + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"rules": + - + "verbs": + - "get" + - "watch" + - "list" + "apiGroups": + - "" + "resources": + - "configmaps" + - "secrets" +--- +"kind": "ClusterRoleBinding" +"apiVersion": "rbac.authorization.k8s.io/v1" +"metadata": + "name": "kube-prometheus-stack-1761819498-grafana-clusterrolebinding" + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"subjects": + - + "kind": "ServiceAccount" + "name": "kube-prometheus-stack-1761819498-grafana" + "namespace": "kube-mon" +"roleRef": + "apiGroup": "rbac.authorization.k8s.io" + "kind": "ClusterRole" + "name": "kube-prometheus-stack-1761819498-grafana-clusterrole" diff --git a/live-monitoring-deployments/grafana/02-core-config.yaml b/live-monitoring-deployments/grafana/02-core-config.yaml new file mode 100644 index 0000000..9cb049f --- /dev/null +++ b/live-monitoring-deployments/grafana/02-core-config.yaml @@ -0,0 +1,83 @@ +--- +"kind": "ConfigMap" +"apiVersion": "v1" +"metadata": + "name": "kube-prometheus-stack-1761819498-grafana" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "grafana.ini": | + [analytics] + check_for_updates = true + [grafana_net] + url = https://grafana.net + [log] + mode = console + [paths] + data = /var/lib/grafana/ + logs = /var/log/grafana + plugins = /var/lib/grafana/plugins + provisioning = /etc/grafana/provisioning + [server] + domain = '' + +--- +"kind": "ConfigMap" +"apiVersion": "v1" +"metadata": + "name": "kube-prometheus-stack-1761819498-grafana-config-dashboards" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "provider.yaml": | + apiVersion: 1 + providers: + - name: 'sidecarProvider' + orgId: 1 + folder: '' + folderUid: '' + type: file + disableDeletion: true + allowUiUpdates: false + updateIntervalSeconds: 30 + options: + foldersFromFilesStructure: false + path: /tmp/dashboards +--- +"kind": "Secret" +"apiVersion": "v1" +"metadata": + "name": "kube-prometheus-stack-1761819498-grafana" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/component": "admin-secret" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "k8slens-edit-resource-version": "v1" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "admin-password": "UnMyMDEyMTk4OCE=" + "admin-user": "YWRtaW4=" + "ldap-toml": "" +"type": "Opaque" diff --git a/live-monitoring-deployments/grafana/03-services.yaml b/live-monitoring-deployments/grafana/03-services.yaml new file mode 100644 index 0000000..abfa5a1 --- /dev/null +++ b/live-monitoring-deployments/grafana/03-services.yaml @@ -0,0 +1,78 @@ +--- +"kind": "Service" +"apiVersion": "v1" +"metadata": + "name": "kube-prometheus-stack-1761819498-grafana" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "k8slens-edit-resource-version": "v1" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"spec": + "ports": + - + "name": "http-web" + "protocol": "TCP" + "port": 80 + "targetPort": "grafana" + "selector": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/name": "grafana" + "type": "ClusterIP" + "sessionAffinity": "None" +--- +"kind": "Service" +"apiVersion": "v1" +"metadata": + "name": "kube-prometheus-stack-1761819498-grafana-headless" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"spec": + "ports": + - + "name": "gossip-tcp" + "protocol": "TCP" + "port": 9094 + "targetPort": "gossip-tcp" + "selector": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/name": "grafana" + "type": "ClusterIP" + "sessionAffinity": "None" + "clusterIP": "None" +--- +"kind": "Service" +"apiVersion": "v1" +"metadata": + "name": "grafana-nodeport" + "namespace": "kube-mon" + "labels": + "k8slens-edit-resource-version": "v1" +"spec": + "ports": + - + "name": "http-web" + "protocol": "TCP" + "port": 80 + "targetPort": "grafana" + "nodePort": 30637 + "selector": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/name": "grafana" + "type": "NodePort" + "sessionAffinity": "None" + "externalTrafficPolicy": "Cluster" diff --git a/live-monitoring-deployments/grafana/04-statefulset.yaml b/live-monitoring-deployments/grafana/04-statefulset.yaml new file mode 100644 index 0000000..429bc52 --- /dev/null +++ b/live-monitoring-deployments/grafana/04-statefulset.yaml @@ -0,0 +1,333 @@ +--- +"kind": "StatefulSet" +"apiVersion": "apps/v1" +"metadata": + "name": "kube-prometheus-stack-1761819498-grafana" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "k8slens-edit-resource-version": "v1" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"spec": + "replicas": 1 + "selector": + "matchLabels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/name": "grafana" + "template": + "metadata": + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/name": "grafana" + "app.kubernetes.io/version": "12.2.1" + "helm.sh/chart": "grafana-10.1.4" + "annotations": + "checksum/config": "d4994ce00e7811465a2fa6543bf581120fee2eef2b029c1877d15c2a3886deae" + "checksum/dashboards-json-config": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" + "checksum/sc-dashboard-provider-config": "77f303eadac6bf5e0f42c223d042ba2827635e029336a34ae30a97433b486bdb" + "checksum/secret": "68ab9c5d48640ff3232bd4e6b4f93001b47e8cc44ae8bbd347e5af4fa2bafa55" + "kubectl.kubernetes.io/default-container": "grafana" + "spec": + "volumes": + - + "name": "config" + "configMap": + "name": "kube-prometheus-stack-1761819498-grafana" + "defaultMode": 420 + - + "name": "sc-dashboard-volume" + "emptyDir": {} + - + "name": "sc-dashboard-provider" + "configMap": + "name": "kube-prometheus-stack-1761819498-grafana-config-dashboards" + "defaultMode": 420 + - + "name": "sc-datasources-volume" + "emptyDir": {} + "initContainers": + - + "name": "init-chown-data" + "image": "docker.io/library/busybox:1.31.1" + "command": + - "chown" + - "-R" + - "472:472" + - "/var/lib/grafana" + "resources": {} + "volumeMounts": + - + "name": "storage" + "mountPath": "/var/lib/grafana" + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "File" + "imagePullPolicy": "IfNotPresent" + "securityContext": + "capabilities": + "add": + - "CHOWN" + "drop": + - "ALL" + "runAsUser": 0 + "runAsNonRoot": false + "readOnlyRootFilesystem": false + "seccompProfile": + "type": "RuntimeDefault" + "containers": + - + "name": "grafana-sc-dashboard" + "image": "quay.io/kiwigrid/k8s-sidecar:1.30.10" + "env": + - + "name": "METHOD" + "value": "WATCH" + - + "name": "LABEL" + "value": "grafana_dashboard" + - + "name": "LABEL_VALUE" + "value": "1" + - + "name": "FOLDER" + "value": "/tmp/dashboards" + - + "name": "RESOURCE" + "value": "both" + - + "name": "NAMESPACE" + "value": "ALL" + - + "name": "SKIP_TLS_VERIFY" + "value": "true" + - + "name": "REQ_USERNAME" + "valueFrom": + "secretKeyRef": + "name": "kube-prometheus-stack-1761819498-grafana" + "key": "admin-user" + - + "name": "REQ_PASSWORD" + "valueFrom": + "secretKeyRef": + "name": "kube-prometheus-stack-1761819498-grafana" + "key": "admin-password" + - + "name": "REQ_URL" + "value": "http://localhost:3000/api/admin/provisioning/dashboards/reload" + - + "name": "REQ_METHOD" + "value": "POST" + "resources": {} + "volumeMounts": + - + "name": "sc-dashboard-volume" + "mountPath": "/tmp/dashboards" + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "File" + "imagePullPolicy": "IfNotPresent" + "securityContext": + "capabilities": + "drop": + - "ALL" + "allowPrivilegeEscalation": false + "seccompProfile": + "type": "RuntimeDefault" + - + "name": "grafana-sc-datasources" + "image": "quay.io/kiwigrid/k8s-sidecar:1.30.10" + "env": + - + "name": "METHOD" + "value": "WATCH" + - + "name": "LABEL" + "value": "grafana_datasource" + - + "name": "LABEL_VALUE" + "value": "1" + - + "name": "FOLDER" + "value": "/etc/grafana/provisioning/datasources" + - + "name": "RESOURCE" + "value": "both" + - + "name": "SKIP_TLS_VERIFY" + "value": "true" + - + "name": "REQ_USERNAME" + "valueFrom": + "secretKeyRef": + "name": "kube-prometheus-stack-1761819498-grafana" + "key": "admin-user" + - + "name": "REQ_PASSWORD" + "valueFrom": + "secretKeyRef": + "name": "kube-prometheus-stack-1761819498-grafana" + "key": "admin-password" + - + "name": "REQ_URL" + "value": "http://localhost:3000/api/admin/provisioning/datasources/reload" + - + "name": "REQ_METHOD" + "value": "POST" + "resources": {} + "volumeMounts": + - + "name": "sc-datasources-volume" + "mountPath": "/etc/grafana/provisioning/datasources" + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "File" + "imagePullPolicy": "IfNotPresent" + "securityContext": + "capabilities": + "drop": + - "ALL" + "allowPrivilegeEscalation": false + "seccompProfile": + "type": "RuntimeDefault" + - + "name": "grafana" + "image": "docker.io/grafana/grafana:13.1.0-24485707904" + "ports": + - + "name": "grafana" + "containerPort": 3000 + "protocol": "TCP" + - + "name": "gossip-tcp" + "containerPort": 9094 + "protocol": "TCP" + - + "name": "gossip-udp" + "containerPort": 9094 + "protocol": "UDP" + - + "name": "profiling" + "containerPort": 6060 + "protocol": "TCP" + "env": + - + "name": "POD_IP" + "valueFrom": + "fieldRef": + "apiVersion": "v1" + "fieldPath": "status.podIP" + - + "name": "GF_SECURITY_ADMIN_USER" + "valueFrom": + "secretKeyRef": + "name": "kube-prometheus-stack-1761819498-grafana" + "key": "admin-user" + - + "name": "GF_SECURITY_ADMIN_PASSWORD" + "valueFrom": + "secretKeyRef": + "name": "kube-prometheus-stack-1761819498-grafana" + "key": "admin-password" + - + "name": "GF_PATHS_DATA" + "value": "/var/lib/grafana/" + - + "name": "GF_PATHS_LOGS" + "value": "/var/log/grafana" + - + "name": "GF_PATHS_PLUGINS" + "value": "/var/lib/grafana/plugins" + - + "name": "GF_PATHS_PROVISIONING" + "value": "/etc/grafana/provisioning" + "resources": {} + "volumeMounts": + - + "name": "config" + "mountPath": "/etc/grafana/grafana.ini" + "subPath": "grafana.ini" + - + "name": "storage" + "mountPath": "/var/lib/grafana" + - + "name": "sc-dashboard-volume" + "mountPath": "/tmp/dashboards" + - + "name": "sc-dashboard-provider" + "mountPath": "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + "subPath": "provider.yaml" + - + "name": "sc-datasources-volume" + "mountPath": "/etc/grafana/provisioning/datasources" + "livenessProbe": + "httpGet": + "path": "/api/health" + "port": 3000 + "scheme": "HTTP" + "initialDelaySeconds": 60 + "timeoutSeconds": 30 + "periodSeconds": 10 + "successThreshold": 1 + "failureThreshold": 10 + "readinessProbe": + "httpGet": + "path": "/api/health" + "port": 3000 + "scheme": "HTTP" + "timeoutSeconds": 1 + "periodSeconds": 10 + "successThreshold": 1 + "failureThreshold": 3 + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "File" + "imagePullPolicy": "IfNotPresent" + "securityContext": + "capabilities": + "drop": + - "ALL" + "allowPrivilegeEscalation": false + "seccompProfile": + "type": "RuntimeDefault" + "restartPolicy": "Always" + "terminationGracePeriodSeconds": 30 + "dnsPolicy": "ClusterFirst" + "serviceAccountName": "kube-prometheus-stack-1761819498-grafana" + "serviceAccount": "kube-prometheus-stack-1761819498-grafana" + "automountServiceAccountToken": true + "shareProcessNamespace": false + "securityContext": + "runAsUser": 472 + "runAsGroup": 472 + "runAsNonRoot": true + "fsGroup": 472 + "schedulerName": "default-scheduler" + "enableServiceLinks": true + "volumeClaimTemplates": + - + "kind": "PersistentVolumeClaim" + "apiVersion": "v1" + "metadata": + "name": "storage" + "spec": + "accessModes": + - "ReadWriteOnce" + "resources": + "requests": + "storage": "20Gi" + "storageClassName": "nfs-csi" + "volumeMode": "Filesystem" + "serviceName": "kube-prometheus-stack-1761819498-grafana-headless" + "podManagementPolicy": "OrderedReady" + "updateStrategy": + "type": "RollingUpdate" + "rollingUpdate": + "partition": 0 + "maxUnavailable": 1 + "revisionHistoryLimit": 10 + "persistentVolumeClaimRetentionPolicy": + "whenDeleted": "Retain" + "whenScaled": "Retain" diff --git a/live-monitoring-deployments/grafana/05-provisioning-configmaps.yaml b/live-monitoring-deployments/grafana/05-provisioning-configmaps.yaml new file mode 100644 index 0000000..1335a96 --- /dev/null +++ b/live-monitoring-deployments/grafana/05-provisioning-configmaps.yaml @@ -0,0 +1,8158 @@ +--- +"metadata": + "name": "alertmanager-kube-mon-dashboard" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "grafana_dashboard": "1" +"data": + "alertmanager-kube-mon.json": | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Alertmanager Up", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "max(up{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"})" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { "text": "Down", "color": "red" }, + "1": { "text": "Up", "color": "green" } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name", + "colorMode": "background" + } + }, + { + "id": 2, + "type": "stat", + "title": "Active Alerts", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "sum(alertmanager_alerts{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\",state=\"active\"})" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 10 } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name", + "colorMode": "background" + } + }, + { + "id": 3, + "type": "stat", + "title": "Suppressed Alerts", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "sum(alertmanager_alerts{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\",state=\"suppressed\"})" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "orange", "value": 5 }, + { "color": "red", "value": 20 } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name", + "colorMode": "background" + } + }, + { + "id": 4, + "type": "stat", + "title": "Active Silences", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "sum(alertmanager_silences{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\",state=\"active\"})" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 10 } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name", + "colorMode": "background" + } + }, + { + "id": 5, + "type": "stat", + "title": "Config Reload", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "max(alertmanager_config_last_reload_successful{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"})" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { "text": "Failed", "color": "red" }, + "1": { "text": "OK", "color": "green" } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name", + "colorMode": "background" + } + }, + { + "id": 6, + "type": "stat", + "title": "Cluster Mode", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "max(alertmanager_cluster_enabled{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"})" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { "text": "Single Node", "color": "yellow" }, + "1": { "text": "HA Enabled", "color": "green" } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "yellow", "value": null }, + { "color": "green", "value": 1 } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name", + "colorMode": "background" + } + }, + { + "id": 7, + "type": "stat", + "title": "Inhibition Rules", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "max(alertmanager_inhibition_rules{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"})" + } + ], + "fieldConfig": { "defaults": {}, "overrides": [] }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name" + } + }, + { + "id": 8, + "type": "stat", + "title": "Aggregation Groups", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "sum(alertmanager_dispatcher_aggregation_groups{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"})" + } + ], + "fieldConfig": { "defaults": {}, "overrides": [] }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name" + } + }, + { + "id": 9, + "type": "stat", + "title": "Configured Receivers", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "max(alertmanager_receivers{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"})" + } + ], + "fieldConfig": { "defaults": {}, "overrides": [] }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name" + } + }, + { + "id": 10, + "type": "stat", + "title": "Last Successful Config Reload", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "max(alertmanager_config_last_reload_success_timestamp_seconds{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"})" + } + ], + "fieldConfig": { + "defaults": { "unit": "dateTimeAsIso" }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name" + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Alerts by State", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "sum by (state) (alertmanager_alerts{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"})", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 20, + "lineWidth": 2, + "stacking": { "group": "A", "mode": "normal" } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Silences by State", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "sum by (state) (alertmanager_silences{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"})", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 20, + "lineWidth": 2, + "stacking": { "group": "A", "mode": "normal" } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Alerts Received by Status", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "sum by (status) (rate(alertmanager_alerts_received_total{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"}[$__rate_interval]))", + "legendFormat": "{{status}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { "fillOpacity": 14, "lineWidth": 2, "stacking": { "group": "A", "mode": "normal" } }, + "min": 0, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Notification Pipeline", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "sum(rate(alertmanager_notification_requests_total{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"}[$__rate_interval]))", + "legendFormat": "Requests" + }, + { + "refId": "B", + "editorMode": "code", + "expr": "sum(rate(alertmanager_notifications_total{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"}[$__rate_interval]))", + "legendFormat": "Sent" + }, + { + "refId": "C", + "editorMode": "code", + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"}[$__rate_interval]))", + "legendFormat": "Failed" + }, + { + "refId": "D", + "editorMode": "code", + "expr": "sum(rate(alertmanager_notifications_suppressed_total{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"}[$__rate_interval]))", + "legendFormat": "Suppressed" + } + ], + "fieldConfig": { + "defaults": { + "custom": { "fillOpacity": 10, "lineWidth": 2 }, + "min": 0, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Notification Latency", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (le) (rate(alertmanager_notification_latency_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(alertmanager_notification_latency_seconds_bucket{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"}[$__rate_interval])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "custom": { "fillOpacity": 10, "lineWidth": 2 }, + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + }, + { + "id": 16, + "type": "bargauge", + "title": "Failed Notifications by Reason (24h)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "sum by (reason) (increase(alertmanager_notifications_failed_total{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"}[24h]))", + "legendFormat": "{{reason}}" + } + ], + "fieldConfig": { + "defaults": { + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showUnfilled": true + } + }, + { + "id": 17, + "type": "timeseries", + "title": "State Store Size", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 32 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "editorMode": "code", + "expr": "sum(alertmanager_silences_snapshot_size_bytes{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"})", + "legendFormat": "Silences snapshot" + }, + { + "refId": "B", + "editorMode": "code", + "expr": "sum(alertmanager_nflog_snapshot_size_bytes{namespace=\"$namespace\",job=~\"$job\",pod=~\"$pod\",instance=~\"$instance\",container=\"alertmanager\"})", + "legendFormat": "NFLog snapshot" + } + ], + "fieldConfig": { + "defaults": { + "custom": { "fillOpacity": 10, "lineWidth": 2 }, + "min": 0, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": ["alertmanager", "kube-mon", "prometheus"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "kube-mon", + "value": "kube-mon" + }, + "hide": 0, + "label": "Namespace", + "name": "namespace", + "query": "kube-mon", + "type": "constant" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(up{namespace=\"$namespace\",container=\"alertmanager\"}, job)", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(up{namespace=\"$namespace\",container=\"alertmanager\"}, job)", + "refId": "job" + }, + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(up{namespace=\"$namespace\",job=~\"$job\",container=\"alertmanager\"}, pod)", + "hide": 0, + "includeAll": true, + "label": "Pod", + "multi": true, + "name": "pod", + "options": [], + "query": { + "query": "label_values(up{namespace=\"$namespace\",job=~\"$job\",container=\"alertmanager\"}, pod)", + "refId": "pod" + }, + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(up{namespace=\"$namespace\",job=~\"$job\",container=\"alertmanager\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(up{namespace=\"$namespace\",job=~\"$job\",container=\"alertmanager\"}, instance)", + "refId": "instance" + }, + "refresh": 2, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Alertmanager / kube-mon", + "uid": "alertmanager-kube-mon", + "version": 1, + "weekStart": "" + } + +--- +"metadata": + "name": "enshrouded-player-dashboard" + "namespace": "kube-mon" + "labels": + "grafana_dashboard": "1" +"data": + "enshrouded-player-dashboard.json": | + { + "id": null, + "uid": "enshrouded-players", + "title": "Gaming - Enshrouded Players", + "tags": ["gaming", "enshrouded", "loki"], + "timezone": "browser", + "schemaVersion": 41, + "version": 5, + "refresh": "10s", + "time": { + "from": "now-24h", + "to": "now" + }, + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "loki", + "value": "dffja3ca1t6o0f" + }, + "name": "datasource", + "type": "datasource", + "query": "loki", + "refresh": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Current Players (Join-Leave)", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "datasource": "$datasource", + "targets": [ + { + "refId": "A", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |= \"logged in with Permissions:\"[$__range])) - sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |= \"Remove Player\"[$__range]))" + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "orange" + }, + { + "value": 4, + "color": "red" + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Login / Logout Events", + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 0 + }, + "datasource": "$datasource", + "targets": [ + { + "refId": "A", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |= \"logged in with Permissions:\"[$__interval]))", + "legendFormat": "Logins", + "queryType": "range" + }, + { + "refId": "B", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |= \"Remove Player\"[$__interval]))", + "legendFormat": "Logouts", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + } + }, + { + "id": 3, + "type": "logs", + "title": "Enshrouded Player Log Events", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 + }, + "datasource": "$datasource", + "targets": [ + { + "refId": "A", + "expr": "{namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |~ \"logged in with Permissions:|Remove Player|Remote player added|Player removed\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending" + } + }, + { + "id": 4, + "type": "timeseries", + "title": "Server Activities (per interval)", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "datasource": "$datasource", + "targets": [ + { + "refId": "A", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |= \"Start Saving\"[$__interval]))", + "legendFormat": "Save Start", + "queryType": "range" + }, + { + "refId": "B", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |= \"Saved\"[$__interval]))", + "legendFormat": "Save Done", + "queryType": "range" + }, + { + "refId": "C", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |~ \"Added Water Dispenser|Removed Water Dispenser|Added Water Drain|Removed Water Drain\"[$__interval]))", + "legendFormat": "World Changes", + "queryType": "range" + }, + { + "refId": "D", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |= \"Bad Performance\"[$__interval]))", + "legendFormat": "Bad Performance", + "queryType": "range" + }, + { + "refId": "E", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |~ \"Remote player added|Player removed\"[$__interval]))", + "legendFormat": "Player Session Events", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + } + }, + { + "id": 5, + "type": "logs", + "title": "Enshrouded Activity Log", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 26 + }, + "datasource": "$datasource", + "targets": [ + { + "refId": "A", + "expr": "{namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |~ \"Start Saving|Saved|Save Serialization took|Bad Performance|Added Water Dispenser|Removed Water Dispenser|Added Water Drain|Removed Water Drain|Remote player added|Player removed|logged in with Permissions:|Remove Player\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending" + } + }, + { + "id": 6, + "type": "stat", + "title": "Saves (last 60m)", + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 36 + }, + "datasource": "$datasource", + "targets": [ + { + "refId": "A", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |= \"Saved\"[60m]))" + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + } + }, + { + "id": 7, + "type": "stat", + "title": "World Changes (last 60m)", + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 36 + }, + "datasource": "$datasource", + "targets": [ + { + "refId": "A", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |~ \"Added Water Dispenser|Removed Water Dispenser|Added Water Drain|Removed Water Drain\"[60m]))" + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + } + }, + { + "id": 8, + "type": "stat", + "title": "Bad Performance (last 60m)", + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 36 + }, + "datasource": "$datasource", + "targets": [ + { + "refId": "A", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |= \"Bad Performance\"[60m]))" + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + } + }, + { + "id": 9, + "type": "stat", + "title": "Player Events (last 60m)", + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 36 + }, + "datasource": "$datasource", + "targets": [ + { + "refId": "A", + "expr": "sum(count_over_time({namespace=\"gaming\",app=\"enshrouded-k8s\",container=\"enshrouded-k8s\"} |~ \"logged in with Permissions:|Remove Player|Remote player added|Player removed\"[60m]))" + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + } + } + ] + } +--- +"metadata": + "name": "haproxy-grafana-dashboard" + "namespace": "kube-mon" + "labels": + "grafana_dashboard": "1" + "release": "kube-prometheus-stack-1761819498" +"data": + "haproxy-overview.json": | + { + "title": "HAProxy TCP Pass-Through Overview", + "uid": "haproxy-tcp-overview", + "schemaVersion": 41, + "version": 1, + "refresh": "30s", + "time": { "from": "now-6h", "to": "now" }, + "timezone": "browser", + "tags": ["haproxy", "tcp", "prometheus", "kubernetes"], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "templating": { + "list": [ + { + "name": "ds", + "type": "datasource", + "query": "prometheus", + "label": "Prometheus", + "refresh": 1, + "current": { "selected": true, "text": "Prometheus", "value": "Prometheus" } + }, + { + "name": "frontend", + "type": "query", + "datasource": "$ds", + "label": "Frontend (TCP)", + "includeAll": true, + "multi": true, + "allValue": ".*", + "refresh": 1, + "query": { + "query": "label_values(haproxy_frontend_current_sessions{proxy!~\"prometheus|stats\"}, proxy)", + "refId": "FrontendVar" + }, + "current": { "selected": true, "text": "All", "value": "$__all" } + }, + { + "name": "backend", + "type": "query", + "datasource": "$ds", + "label": "Backend", + "includeAll": true, + "multi": true, + "allValue": ".*", + "refresh": 1, + "query": { + "query": "label_values(haproxy_backend_current_sessions{proxy=~\"traefik_.*\"}, proxy)", + "refId": "BackendVar" + }, + "current": { "selected": true, "text": "All", "value": "$__all" } + }, + { + "name": "server", + "type": "query", + "datasource": "$ds", + "label": "Server", + "includeAll": true, + "multi": true, + "allValue": ".*", + "refresh": 1, + "query": { + "query": "label_values(haproxy_server_current_sessions{proxy=~\"$backend\"}, server)", + "refId": "ServerVar" + }, + "current": { "selected": true, "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Active TCP Sessions", + "datasource": "$ds", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "targets": [{ "refId": "A", "expr": "sum(haproxy_frontend_current_sessions{proxy=~\"$frontend\"})" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 2, + "type": "stat", + "title": "New TCP Connections/s", + "datasource": "$ds", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, + "targets": [{ "refId": "A", "expr": "sum(rate(haproxy_frontend_connections_total{proxy=~\"$frontend\"}[$__rate_interval]))" }], + "fieldConfig": { "defaults": { "unit": "cps" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 3, + "type": "stat", + "title": "Backend Queue", + "datasource": "$ds", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 }, + "targets": [{ "refId": "A", "expr": "sum(haproxy_backend_current_queue{proxy=~\"$backend\"})" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 4, + "type": "stat", + "title": "Servers UP", + "datasource": "$ds", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 }, + "targets": [{ "refId": "A", "expr": "sum(haproxy_server_status{proxy=~\"$backend\",server=~\"$server\",state=\"UP\"})" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 5, + "type": "stat", + "title": "Servers DOWN", + "datasource": "$ds", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 }, + "targets": [{ "refId": "A", "expr": "sum(haproxy_server_status{proxy=~\"$backend\",server=~\"$server\",state=\"DOWN\"})" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 6, + "type": "stat", + "title": "Process Connections", + "datasource": "$ds", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 }, + "targets": [{ "refId": "A", "expr": "max(haproxy_process_current_connections)" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 7, + "type": "timeseries", + "title": "Frontend Current Sessions by Proxy", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 4 }, + "targets": [{ "refId": "A", "expr": "sum by (proxy) (haproxy_frontend_current_sessions{proxy=~\"$frontend\"})", "legendFormat": "{{proxy}}" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } + }, + { + "id": 8, + "type": "timeseries", + "title": "Frontend New Connections/s by Proxy", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 4 }, + "targets": [{ "refId": "A", "expr": "sum by (proxy) (rate(haproxy_frontend_connections_total{proxy=~\"$frontend\"}[$__rate_interval]))", "legendFormat": "{{proxy}}" }], + "fieldConfig": { "defaults": { "unit": "cps" }, "overrides": [] } + }, + { + "id": 9, + "type": "timeseries", + "title": "Frontend Throughput", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 4 }, + "targets": [ + { "refId": "A", "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_in_total{proxy=~\"$frontend\"}[$__rate_interval]))", "legendFormat": "in {{proxy}}" }, + { "refId": "B", "expr": "sum by (proxy) (rate(haproxy_frontend_bytes_out_total{proxy=~\"$frontend\"}[$__rate_interval]))", "legendFormat": "out {{proxy}}" } + ], + "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] } + }, + { + "id": 10, + "type": "timeseries", + "title": "Backend Current Sessions", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 12 }, + "targets": [{ "refId": "A", "expr": "sum by (proxy) (haproxy_backend_current_sessions{proxy=~\"$backend\"})", "legendFormat": "{{proxy}}" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } + }, + { + "id": 11, + "type": "timeseries", + "title": "Backend Sessions/s", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 12 }, + "targets": [{ "refId": "A", "expr": "sum by (proxy) (rate(haproxy_backend_sessions_total{proxy=~\"$backend\"}[$__rate_interval]))", "legendFormat": "{{proxy}}" }], + "fieldConfig": { "defaults": { "unit": "cps" }, "overrides": [] } + }, + { + "id": 12, + "type": "timeseries", + "title": "Backend Queue by Proxy", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 12 }, + "targets": [{ "refId": "A", "expr": "sum by (proxy) (haproxy_backend_current_queue{proxy=~\"$backend\"})", "legendFormat": "{{proxy}}" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } + }, + { + "id": 13, + "type": "bargauge", + "title": "Top Servers by Current Sessions", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 20 }, + "targets": [{ "refId": "A", "expr": "topk(20, haproxy_server_current_sessions{proxy=~\"$backend\",server=~\"$server\"})", "legendFormat": "{{proxy}}/{{server}}" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 14, + "type": "bargauge", + "title": "Top Servers by Loadbalanced/s", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 20 }, + "targets": [{ "refId": "A", "expr": "topk(20, rate(haproxy_server_loadbalanced_total{proxy=~\"$backend\",server=~\"$server\"}[$__rate_interval]))", "legendFormat": "{{proxy}}/{{server}}" }], + "fieldConfig": { "defaults": { "unit": "cps" }, "overrides": [] }, + "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 15, + "type": "timeseries", + "title": "Server State Count", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 20 }, + "targets": [{ "refId": "A", "expr": "sum by (state) (haproxy_server_status{proxy=~\"$backend\",server=~\"$server\"})", "legendFormat": "{{state}}" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } + }, + { + "id": 16, + "type": "timeseries", + "title": "Server Errors/s", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 28 }, + "targets": [ + { "refId": "A", "expr": "topk(20, rate(haproxy_server_connection_errors_total{proxy=~\"$backend\",server=~\"$server\"}[$__rate_interval]))", "legendFormat": "conn err {{proxy}}/{{server}}" }, + { "refId": "B", "expr": "topk(20, rate(haproxy_server_check_failures_total{proxy=~\"$backend\",server=~\"$server\"}[$__rate_interval]))", "legendFormat": "check fail {{proxy}}/{{server}}" } + ], + "fieldConfig": { "defaults": { "unit": "cps" }, "overrides": [] } + }, + { + "id": 17, + "type": "timeseries", + "title": "Server Downtime Growth", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 28 }, + "targets": [{ "refId": "A", "expr": "sum by (proxy,server) (rate(haproxy_server_downtime_seconds_total{proxy=~\"$backend\",server=~\"$server\"}[$__rate_interval]))", "legendFormat": "{{proxy}}/{{server}}" }], + "fieldConfig": { "defaults": { "unit": "sps" }, "overrides": [] } + }, + { + "id": 18, + "type": "table", + "title": "Server Status Matrix (Instant)", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 28 }, + "targets": [{ "refId": "A", "expr": "max by (proxy,server,state) (haproxy_server_status{proxy=~\"$backend\",server=~\"$server\",state=~\"UP|DOWN|MAINT|DRAIN|NOLB\"})", "instant": true, "format": "table" }], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } }, + { "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "Value": "active" } } } + ] + }, + { + "id": 19, + "type": "timeseries", + "title": "Process Connection/Session Rates", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, + "targets": [ + { "refId": "A", "expr": "sum(rate(haproxy_process_connections_total[$__rate_interval]))", "legendFormat": "new connections/s" }, + { "refId": "B", "expr": "max(haproxy_process_current_connection_rate)", "legendFormat": "current connection rate" }, + { "refId": "C", "expr": "max(haproxy_process_current_session_rate)", "legendFormat": "current session rate" } + ], + "fieldConfig": { "defaults": { "unit": "cps" }, "overrides": [] } + }, + { + "id": 20, + "type": "timeseries", + "title": "Process Reliability Signals", + "datasource": "$ds", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, + "targets": [ + { "refId": "A", "expr": "max(haproxy_process_idle_time_percent)", "legendFormat": "idle %" }, + { "refId": "B", "expr": "max(haproxy_process_total_warnings)", "legendFormat": "warnings" }, + { "refId": "C", "expr": "sum(rate(haproxy_process_failed_resolutions[$__rate_interval]))", "legendFormat": "failed DNS/s" }, + { "refId": "D", "expr": "sum(rate(haproxy_process_dropped_logs_total[$__rate_interval]))", "legendFormat": "dropped logs/s" } + ], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } + }, + { + "id": 21, + "type": "stat", + "title": "Process Uptime", + "datasource": "$ds", + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 44 }, + "targets": [{ "refId": "A", "expr": "max(haproxy_process_uptime_seconds)" }], + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 22, + "type": "stat", + "title": "Process Idle %", + "datasource": "$ds", + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 44 }, + "targets": [{ "refId": "A", "expr": "max(haproxy_process_idle_time_percent)" }], + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + }, + { + "id": 23, + "type": "stat", + "title": "Process Warnings", + "datasource": "$ds", + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 44 }, + "targets": [{ "refId": "A", "expr": "max(haproxy_process_total_warnings)" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } } + } + ] + } + +--- +"metadata": + "name": "kube-prometheus-stack-1761-alertmanager-overview" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "alertmanager-overview.json": "{\"graphTooltip\":1,\"panels\":[{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"panels\":[],\"title\":\"Alerts\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"description\":\"current set of alerts stored in the Alertmanager\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"none\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":1},\"id\":2,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(alertmanager_alerts{namespace=~\\\"$namespace\\\",service=~\\\"$service\\\"}) by (namespace,service,instance)\",\"intervalFactor\":2,\"legendFormat\":\"{{instance}}\"}],\"title\":\"Alerts\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"description\":\"rate of successful and invalid alerts received by the Alertmanager\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":1},\"id\":3,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(rate(alertmanager_alerts_received_total{namespace=~\\\"$namespace\\\",service=~\\\"$service\\\"}[$__rate_interval])) by (namespace,service,instance)\",\"intervalFactor\":2,\"legendFormat\":\"{{instance}} Received\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(rate(alertmanager_alerts_invalid_total{namespace=~\\\"$namespace\\\",service=~\\\"$service\\\"}[$__rate_interval])) by (namespace,service,instance)\",\"intervalFactor\":2,\"legendFormat\":\"{{instance}} Invalid\"}],\"title\":\"Alerts receive rate\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":8},\"id\":4,\"panels\":[],\"title\":\"Notifications\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"description\":\"rate of successful and invalid notifications sent by the Alertmanager\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":9},\"id\":5,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"repeat\":\"integration\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(rate(alertmanager_notifications_total{namespace=~\\\"$namespace\\\",service=~\\\"$service\\\", integration=\\\"$integration\\\"}[$__rate_interval])) by (integration,namespace,service,instance)\",\"intervalFactor\":2,\"legendFormat\":\"{{instance}} Total\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(rate(alertmanager_notifications_failed_total{namespace=~\\\"$namespace\\\",service=~\\\"$service\\\", integration=\\\"$integration\\\"}[$__rate_interval])) by (integration,namespace,service,instance)\",\"intervalFactor\":2,\"legendFormat\":\"{{instance}} Failed\"}],\"title\":\"$integration: Notifications Send Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"description\":\"latency of notifications sent by the Alertmanager\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":9},\"id\":6,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"repeat\":\"integration\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.99,\\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\\\"$namespace\\\",service=~\\\"$service\\\", integration=\\\"$integration\\\"}[$__rate_interval])) by (le,namespace,service,instance)\\n)\\n\",\"intervalFactor\":2,\"legendFormat\":\"{{instance}} 99th Percentile\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.50,\\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\\\"$namespace\\\",service=~\\\"$service\\\", integration=\\\"$integration\\\"}[$__rate_interval])) by (le,namespace,service,instance)\\n)\\n\",\"intervalFactor\":2,\"legendFormat\":\"{{instance}} Median\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\\\"$namespace\\\",service=~\\\"$service\\\", integration=\\\"$integration\\\"}[$__rate_interval])) by (namespace,service,instance)\\n/\\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\\\"$namespace\\\",service=~\\\"$service\\\", integration=\\\"$integration\\\"}[$__rate_interval])) by (namespace,service,instance)\\n\",\"intervalFactor\":2,\"legendFormat\":\"{{instance}} Average\"}],\"title\":\"$integration: Notification Duration\",\"type\":\"timeseries\"}],\"schemaVersion\":39,\"tags\":[\"alertmanager-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":false,\"text\":\"Prometheus\",\"value\":\"Prometheus\"},\"hide\":0,\"label\":\"Data Source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"type\":\"datasource\"},{\"current\":{\"selected\":false,\"text\":\"\",\"value\":\"\"},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"includeAll\":false,\"label\":\"namespace\",\"name\":\"namespace\",\"query\":\"label_values(alertmanager_alerts, namespace)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"},{\"current\":{\"selected\":false,\"text\":\"\",\"value\":\"\"},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"includeAll\":false,\"label\":\"service\",\"name\":\"service\",\"query\":\"label_values(alertmanager_alerts, service)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"},{\"current\":{\"selected\":false,\"text\":\"$__all\",\"value\":\"$__all\"},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"includeAll\":true,\"name\":\"integration\",\"query\":\"label_values(alertmanager_notifications_total{integration=~\\\".*\\\"}, integration)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timepicker\":{\"refresh_intervals\":[\"30s\"]},\"timezone\": \"utc\",\"title\":\"Alertmanager / Overview\",\"uid\":\"alertmanager-overview\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-apiserver" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "apiserver.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"description\":\"The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.\",\"gridPos\":{\"h\":2,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"options\":{\"content\":\"The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.\"},\"pluginVersion\":\"v11.4.0\",\"title\":\"Notice\",\"type\":\"text\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"description\":\"How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?\",\"fieldConfig\":{\"defaults\":{\"decimals\":3,\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":2},\"id\":2,\"interval\":\"1m\",\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"apiserver_request:availability30d{verb=\\\"all\\\", cluster=\\\"$cluster\\\"}\"}],\"title\":\"Availability (30d) > 99.000%\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"description\":\"How much error budget is left looking at our 0.990% availability guarantees?\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100},\"decimals\":3,\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":16,\"x\":8,\"y\":2},\"id\":3,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"100 * (apiserver_request:availability30d{verb=\\\"all\\\", cluster=\\\"$cluster\\\"} - 0.990000)\",\"legendFormat\":\"errorbudget\"}],\"title\":\"ErrorBudget (30d) > 99.000%\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"description\":\"How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?\",\"fieldConfig\":{\"defaults\":{\"decimals\":3,\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":0,\"y\":9},\"id\":4,\"interval\":\"1m\",\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"apiserver_request:availability30d{verb=\\\"read\\\", cluster=\\\"$cluster\\\"}\"}],\"title\":\"Read Availability (30d)\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"description\":\"How many read requests (LIST,GET) per second do the apiservers get by code?\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"reqps\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/2../i\"},\"properties\":[{\"id\":\"color\",\"value\":\"#56A64B\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/3../i\"},\"properties\":[{\"id\":\"color\",\"value\":\"#F2CC0C\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/4../i\"},\"properties\":[{\"id\":\"color\",\"value\":\"#3274D9\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/5../i\"},\"properties\":[{\"id\":\"color\",\"value\":\"#E02F44\"}]}]},\"gridPos\":{\"h\":7,\"w\":6,\"x\":6,\"y\":9},\"id\":5,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\\\"read\\\", cluster=\\\"$cluster\\\"})\",\"legendFormat\":\"{{ code }}\"}],\"title\":\"Read SLI - Requests\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"description\":\"How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?\",\"fieldConfig\":{\"defaults\":{\"min\":0,\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":12,\"y\":9},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\\\"read\\\",code=~\\\"5..\\\", cluster=\\\"$cluster\\\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\\\"read\\\", cluster=\\\"$cluster\\\"})\",\"legendFormat\":\"{{ resource }}\"}],\"title\":\"Read SLI - Errors\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"description\":\"How many seconds is the 99th percentile for reading (LIST|GET) a given resource?\",\"fieldConfig\":{\"defaults\":{\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":18,\"y\":9},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile{verb=\\\"read\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"{{ resource }}\"}],\"title\":\"Read SLI - Duration\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"description\":\"How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?\",\"fieldConfig\":{\"defaults\":{\"decimals\":3,\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":0,\"y\":16},\"id\":8,\"interval\":\"1m\",\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"apiserver_request:availability30d{verb=\\\"write\\\", cluster=\\\"$cluster\\\"}\"}],\"title\":\"Write Availability (30d)\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"description\":\"How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"reqps\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/2../i\"},\"properties\":[{\"id\":\"color\",\"value\":\"#56A64B\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/3../i\"},\"properties\":[{\"id\":\"color\",\"value\":\"#F2CC0C\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/4../i\"},\"properties\":[{\"id\":\"color\",\"value\":\"#3274D9\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/5../i\"},\"properties\":[{\"id\":\"color\",\"value\":\"#E02F44\"}]}]},\"gridPos\":{\"h\":7,\"w\":6,\"x\":6,\"y\":16},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\\\"write\\\", cluster=\\\"$cluster\\\"})\",\"legendFormat\":\"{{ code }}\"}],\"title\":\"Write SLI - Requests\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"description\":\"How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?\",\"fieldConfig\":{\"defaults\":{\"min\":0,\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":12,\"y\":16},\"id\":10,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\\\"write\\\",code=~\\\"5..\\\", cluster=\\\"$cluster\\\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\\\"write\\\", cluster=\\\"$cluster\\\"})\",\"legendFormat\":\"{{ resource }}\"}],\"title\":\"Write SLI - Errors\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"description\":\"How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?\",\"fieldConfig\":{\"defaults\":{\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":18,\"y\":16},\"id\":11,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile{verb=\\\"write\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"{{ resource }}\"}],\"title\":\"Write SLI - Duration\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"min\":0,\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":23},\"id\":12,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":false},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(workqueue_adds_total{job=\\\"apiserver\\\", instance=~\\\"$instance\\\", cluster=\\\"$cluster\\\"}[$__rate_interval])) by (instance, name)\",\"legendFormat\":\"{{instance}} {{name}}\"}],\"title\":\"Work Queue Add Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"min\":0,\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":23},\"id\":13,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":false},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(workqueue_depth{job=\\\"apiserver\\\", instance=~\\\"$instance\\\", cluster=\\\"$cluster\\\"}[$__rate_interval])) by (instance, name)\",\"legendFormat\":\"{{instance}} {{name}}\"}],\"title\":\"Work Queue Depth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"min\":0,\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":30},\"id\":14,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\\\"apiserver\\\", instance=~\\\"$instance\\\", cluster=\\\"$cluster\\\"}[$__rate_interval])) by (instance, name, le))\",\"legendFormat\":\"{{instance}} {{name}}\"}],\"title\":\"Work Queue Latency\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":37},\"id\":15,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"process_resident_memory_bytes{job=\\\"apiserver\\\",instance=~\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Memory\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"min\":0,\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":8,\"y\":37},\"id\":16,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"rate(process_cpu_seconds_total{job=\\\"apiserver\\\",instance=~\\\"$instance\\\", cluster=\\\"$cluster\\\"}[$__rate_interval])\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"CPU usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":16,\"y\":37},\"id\":17,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"go_goroutines{job=\\\"apiserver\\\",instance=~\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Goroutines\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"apiserver\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"name\":\"instance\",\"query\":\"label_values(up{job=\\\"apiserver\\\", cluster=\\\"$cluster\\\"}, instance)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / API server\",\"uid\":\"09ec8aa1e996d6ffcd6817bbaff4db1b\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-cluster-total" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "cluster-total.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Current Rate of Bytes Received\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":0},\"id\":2,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Current Rate of Bytes Transmitted\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Bytes/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"binBps\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Packets/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"pps\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Namespace\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down\",\"url\":\"/d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?${datasource:queryparam}&var-cluster=${cluster}&var-namespace=${__data.fields.Namespace}\"}]}]}]},\"gridPos\":{\"h\":9,\"w\":24,\"x\":0,\"y\":9},\"id\":3,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"avg by (namespace) (\\n rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"avg by (namespace) (\\n rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_receive_packets_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_transmit_packets_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_receive_packets_dropped_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_transmit_packets_dropped_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true}],\"title\":\"Current Status\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"namespace\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true,\"Time 7\":true,\"Time 8\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Time 7\":6,\"Time 8\":7,\"Value #A\":9,\"Value #B\":10,\"Value #C\":11,\"Value #D\":12,\"Value #E\":13,\"Value #F\":14,\"Value #G\":15,\"Value #H\":16,\"namespace\":8},\"renameByName\":{\"Value #A\":\"Rx Bytes\",\"Value #B\":\"Tx Bytes\",\"Value #C\":\"Rx Bytes (Avg)\",\"Value #D\":\"Tx Bytes (Avg)\",\"Value #E\":\"Rx Packets\",\"Value #F\":\"Tx Packets\",\"Value #G\":\"Rx Packets Dropped\",\"Value #H\":\"Tx Packets Dropped\",\"namespace\":\"Namespace\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":18},\"id\":4,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"avg by (namespace) (\\n rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Rate of Bytes Received\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":18},\"id\":5,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"avg by (namespace) (\\n rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Rate of Bytes Transmitted\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":27},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Receive Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":27},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Transmit Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":36},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_receive_packets_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":36},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_transmit_packets_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":45},\"id\":10,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_receive_packets_dropped_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":45},\"id\":11,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (namespace) (\\n rate(container_network_transmit_packets_dropped_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":54},\"id\":12,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (instance) (\\n rate(node_netstat_Tcp_RetransSegs{cluster=\\\"$cluster\\\"}[$__rate_interval]) / rate(node_netstat_Tcp_OutSegs{cluster=\\\"$cluster\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of TCP Retransmits out of all sent segments\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":54},\"id\":13,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (instance) (\\n rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\\\"$cluster\\\"}[$__rate_interval]) / rate(node_netstat_Tcp_RetransSegs{cluster=\\\"$cluster\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of TCP SYN Retransmits out of all retransmits\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Networking / Cluster\",\"uid\":\"ff635a025bcfea7bc3dd4f508990a3e9\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-controller-manager" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "controller-manager.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"none\"}},\"gridPos\":{\"h\":7,\"w\":4,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(up{cluster=\\\"$cluster\\\", job=\\\"kube-controller-manager\\\"})\",\"instant\":true}],\"title\":\"Up\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":20,\"x\":4,\"y\":0},\"id\":2,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(workqueue_adds_total{cluster=\\\"$cluster\\\", job=\\\"kube-controller-manager\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (cluster, instance, name)\",\"legendFormat\":\"{{cluster}} {{instance}} {{name}}\"}],\"title\":\"Work Queue Add Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":7},\"id\":3,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(workqueue_depth{cluster=\\\"$cluster\\\", job=\\\"kube-controller-manager\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (cluster, instance, name)\",\"legendFormat\":\"{{cluster}} {{instance}} {{name}}\"}],\"title\":\"Work Queue Depth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":14},\"id\":4,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-controller-manager\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (cluster, instance, name, le))\",\"legendFormat\":\"{{cluster}} {{instance}} {{name}}\"}],\"title\":\"Work Queue Latency\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":21},\"id\":5,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{job=\\\"kube-controller-manager\\\", instance=~\\\"$instance\\\",code=~\\\"2..\\\"}[$__rate_interval]))\",\"legendFormat\":\"2xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{job=\\\"kube-controller-manager\\\", instance=~\\\"$instance\\\",code=~\\\"3..\\\"}[$__rate_interval]))\",\"legendFormat\":\"3xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{job=\\\"kube-controller-manager\\\", instance=~\\\"$instance\\\",code=~\\\"4..\\\"}[$__rate_interval]))\",\"legendFormat\":\"4xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{job=\\\"kube-controller-manager\\\", instance=~\\\"$instance\\\",code=~\\\"5..\\\"}[$__rate_interval]))\",\"legendFormat\":\"5xx\"}],\"title\":\"Kube API Request Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":16,\"x\":8,\"y\":21},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-controller-manager\\\", instance=~\\\"$instance\\\", verb=\\\"POST\\\"}[$__rate_interval])) by (verb, le))\",\"legendFormat\":\"{{verb}}\"}],\"title\":\"Post Request Latency 99th Quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":28},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-controller-manager\\\", instance=~\\\"$instance\\\", verb=\\\"GET\\\"}[$__rate_interval])) by (verb, le))\",\"legendFormat\":\"{{verb}}\"}],\"title\":\"Get Request Latency 99th Quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":35},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"process_resident_memory_bytes{cluster=\\\"$cluster\\\", job=\\\"kube-controller-manager\\\",instance=~\\\"$instance\\\"}\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Memory\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":8,\"y\":35},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"rate(process_cpu_seconds_total{cluster=\\\"$cluster\\\", job=\\\"kube-controller-manager\\\",instance=~\\\"$instance\\\"}[$__rate_interval])\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"CPU usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":16,\"y\":35},\"id\":10,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"go_goroutines{cluster=\\\"$cluster\\\", job=\\\"kube-controller-manager\\\",instance=~\\\"$instance\\\"}\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Goroutines\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kube-controller-manager\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"label\":\"instance\",\"name\":\"instance\",\"query\":\"label_values(up{cluster=\\\"$cluster\\\", job=\\\"kube-controller-manager\\\"}, instance)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Controller Manager\",\"uid\":\"72e0e05bef5099e5f049b05fdc429ed4\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-etcd" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "etcd.json": "{\"description\":\"etcd sample Grafana dashboard with Prometheus\",\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"gridPos\":{\"h\":7,\"w\":6,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\",\"graphMode\":\"none\",\"reduceOptions\":{\"calcs\":[\"lastNotNull\"]}},\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(etcd_server_has_leader{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"})\",\"legendFormat\":\"{{cluster}} - {{namespace}}\\n\"}],\"title\":\"Up\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":10,\"x\":6,\"y\":0},\"id\":2,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(rate(grpc_server_started_total{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\",grpc_type=\\\"unary\\\"}[$__rate_interval]))\",\"legendFormat\":\"RPC rate\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(rate(grpc_server_handled_total{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\",grpc_type=\\\"unary\\\",grpc_code=~\\\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\\\"}[$__rate_interval]))\",\"legendFormat\":\"RPC failed rate\"}],\"title\":\"RPC rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"}}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":16,\"y\":0},\"id\":3,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(grpc_server_started_total{job=~\\\".*etcd.*\\\",job=\\\"$cluster\\\",grpc_service=\\\"etcdserverpb.Watch\\\",grpc_type=\\\"bidi_stream\\\"}) - sum(grpc_server_handled_total{job=\\\"$cluster\\\",grpc_service=\\\"etcdserverpb.Watch\\\",grpc_type=\\\"bidi_stream\\\"})\",\"legendFormat\":\"Watch streams\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(grpc_server_started_total{job=~\\\".*etcd.*\\\",job=\\\"$cluster\\\",grpc_service=\\\"etcdserverpb.Lease\\\",grpc_type=\\\"bidi_stream\\\"}) - sum(grpc_server_handled_total{job=\\\"$cluster\\\",grpc_service=\\\"etcdserverpb.Lease\\\",grpc_type=\\\"bidi_stream\\\"})\",\"legendFormat\":\"Lease streams\"}],\"title\":\"Active streams\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"},\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":25},\"id\":4,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"etcd_mvcc_db_total_size_in_bytes{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"}\",\"legendFormat\":\"{{instance}} DB size\"}],\"title\":\"DB size\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":8,\"y\":25},\"id\":5,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"}[$__rate_interval])) by (instance, le))\",\"legendFormat\":\"{{instance}} WAL fsync\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"}[$__rate_interval])) by (instance, le))\",\"legendFormat\":\"{{instance}} DB fsync\"}],\"title\":\"Disk sync duration\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"},\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":16,\"y\":25},\"id\":6,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"process_resident_memory_bytes{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"}\",\"legendFormat\":\"{{instance}} resident memory\"}],\"title\":\"Memory\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":0,\"y\":50},\"id\":7,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(etcd_network_client_grpc_received_bytes_total{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"}[$__rate_interval])\",\"legendFormat\":\"{{instance}} client traffic in\"}],\"title\":\"Client traffic in\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":6,\"y\":50},\"id\":8,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(etcd_network_client_grpc_sent_bytes_total{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"}[$__rate_interval])\",\"legendFormat\":\"{{instance}} client traffic out\"}],\"title\":\"Client traffic out\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":12,\"y\":50},\"id\":9,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(rate(etcd_network_peer_received_bytes_total{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"}[$__rate_interval])) by (instance)\",\"legendFormat\":\"{{instance}} peer traffic in\"}],\"title\":\"Peer traffic in\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":18,\"y\":50},\"id\":10,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(rate(etcd_network_peer_sent_bytes_total{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"}[$__rate_interval])) by (instance)\",\"legendFormat\":\"{{instance}} peer traffic out\"}],\"title\":\"Peer traffic out\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"}}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":75},\"id\":11,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"changes(etcd_server_leader_changes_seen_total{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"}[1d])\",\"legendFormat\":\"{{instance}} total leader elections per day\"}],\"title\":\"Raft proposals\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"}}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":8,\"y\":75},\"id\":12,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"changes(etcd_server_leader_changes_seen_total{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"}[1d])\",\"legendFormat\":\"{{instance}} total leader elections per day\"}],\"title\":\"Total leader elections per day\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"lineWidth\":2,\"showPoints\":\"never\"},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":16,\"y\":75},\"id\":13,\"interval\":\"1m\",\"pluginVersion\":\"v10.0.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~\\\".*etcd.*\\\", job=\\\"$cluster\\\"}[$__rate_interval])))\",\"legendFormat\":\"{{instance}} peer round trip time\"}],\"title\":\"Peer round trip time\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":36,\"tags\":[\"etcd-mixin\"],\"templating\":{\"list\":[{\"label\":\"Data Source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(etcd_server_has_leader{job=~\\\".*etcd.*\\\"}, job)\",\"refresh\":2,\"type\":\"query\",\"allValue\":\".*\",\"hide\":2}]},\"time\":{\"from\":\"now-15m\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"etcd\",\"uid\":\"c2f4e12cdf69feb95caa41a5a1b423d9\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-grafana-datasource" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_datasource": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "datasource.yaml": | + apiVersion: 1 + datasources: + - name: "Prometheus" + type: prometheus + uid: prometheus + url: http://kube-prometheus-stack-1761-prometheus.kube-mon:9090/ + access: proxy + isDefault: true + jsonData: + httpMethod: POST + timeInterval: 30s + - name: "Alertmanager" + type: alertmanager + uid: alertmanager + url: http://kube-prometheus-stack-1761-alertmanager.kube-mon:9093/ + access: proxy + jsonData: + handleGrafanaManagedAlerts: false + implementation: prometheus +--- +"metadata": + "name": "kube-prometheus-stack-1761-grafana-overview" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "grafana-overview.json": "{\"annotations\":{\"list\":[{\"builtIn\":1,\"datasource\":\"-- Grafana --\",\"enable\":true,\"hide\":true,\"iconColor\":\"rgba(0, 211, 255, 1)\",\"name\":\"Annotations & Alerts\",\"target\":{\"limit\":100,\"matchAny\":false,\"tags\":[],\"type\":\"dashboard\"},\"type\":\"dashboard\"}]},\"editable\":true,\"gnetId\":null,\"graphTooltip\":0,\"id\":3085,\"iteration\":1631554945276,\"links\":[],\"panels\":[{\"datasource\":\"$datasource\",\"fieldConfig\":{\"defaults\":{\"mappings\":[],\"noValue\":\"0\",\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]}},\"overrides\":[]},\"gridPos\":{\"h\":5,\"w\":6,\"x\":0,\"y\":0},\"id\":6,\"options\":{\"colorMode\":\"value\",\"graphMode\":\"area\",\"justifyMode\":\"auto\",\"orientation\":\"auto\",\"reduceOptions\":{\"calcs\":[\"mean\"],\"fields\":\"\",\"values\":false},\"text\":{},\"textMode\":\"auto\"},\"pluginVersion\":\"8.1.3\",\"targets\":[{\"expr\":\"grafana_alerting_result_total{job=~\\\"$job\\\", instance=~\\\"$instance\\\", state=\\\"alerting\\\"}\",\"instant\":true,\"interval\":\"1m\",\"legendFormat\":\"\",\"refId\":\"A\"}],\"timeFrom\":null,\"timeShift\":null,\"title\":\"Firing Alerts\",\"type\":\"stat\"},{\"datasource\":\"$datasource\",\"fieldConfig\":{\"defaults\":{\"mappings\":[],\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]}},\"overrides\":[]},\"gridPos\":{\"h\":5,\"w\":6,\"x\":6,\"y\":0},\"id\":8,\"options\":{\"colorMode\":\"value\",\"graphMode\":\"area\",\"justifyMode\":\"auto\",\"orientation\":\"auto\",\"reduceOptions\":{\"calcs\":[\"mean\"],\"fields\":\"\",\"values\":false},\"text\":{},\"textMode\":\"auto\"},\"pluginVersion\":\"8.1.3\",\"targets\":[{\"expr\":\"sum(grafana_stat_totals_dashboard{job=~\\\"$job\\\", instance=~\\\"$instance\\\"})\",\"interval\":\"1m\",\"legendFormat\":\"\",\"refId\":\"A\"}],\"timeFrom\":null,\"timeShift\":null,\"title\":\"Dashboards\",\"type\":\"stat\"},{\"datasource\":\"$datasource\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"align\":null,\"displayMode\":\"auto\"},\"mappings\":[],\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]}},\"overrides\":[]},\"gridPos\":{\"h\":5,\"w\":12,\"x\":12,\"y\":0},\"id\":10,\"options\":{\"showHeader\":true},\"pluginVersion\":\"8.1.3\",\"targets\":[{\"expr\":\"grafana_build_info{job=~\\\"$job\\\", instance=~\\\"$instance\\\"}\",\"instant\":true,\"interval\":\"1m\",\"legendFormat\":\"\",\"refId\":\"A\"}],\"timeFrom\":null,\"timeShift\":null,\"title\":\"Build Info\",\"transformations\":[{\"id\":\"labelsToFields\",\"options\":{}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Value\":true,\"branch\":true,\"container\":true,\"goversion\":true,\"namespace\":true,\"pod\":true,\"revision\":true},\"indexByName\":{\"Time\":7,\"Value\":11,\"branch\":4,\"container\":8,\"edition\":2,\"goversion\":6,\"instance\":1,\"job\":0,\"namespace\":9,\"pod\":10,\"revision\":5,\"version\":3},\"renameByName\":{}}}],\"type\":\"table\"},{\"aliasColors\":{},\"bars\":false,\"dashLength\":10,\"dashes\":false,\"datasource\":\"$datasource\",\"fieldConfig\":{\"defaults\":{\"links\":[]},\"overrides\":[]},\"fill\":1,\"fillGradient\":0,\"gridPos\":{\"h\":8,\"w\":12,\"x\":0,\"y\":5},\"hiddenSeries\":false,\"id\":2,\"legend\":{\"avg\":false,\"current\":false,\"max\":false,\"min\":false,\"show\":true,\"total\":false,\"values\":false},\"lines\":true,\"linewidth\":1,\"nullPointMode\":\"null\",\"options\":{\"alertThreshold\":true},\"percentage\":false,\"pluginVersion\":\"8.1.3\",\"pointradius\":2,\"points\":false,\"renderer\":\"flot\",\"seriesOverrides\":[],\"spaceLength\":10,\"stack\":true,\"steppedLine\":false,\"targets\":[{\"expr\":\"sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\\\"$job\\\", instance=~\\\"$instance\\\"}[1m])) \",\"interval\":\"1m\",\"legendFormat\":\"{{status_code}}\",\"refId\":\"A\"}],\"thresholds\":[],\"timeFrom\":null,\"timeRegions\":[],\"timeShift\":null,\"title\":\"RPS\",\"tooltip\":{\"shared\":true,\"sort\":0,\"value_type\":\"individual\"},\"type\":\"graph\",\"xaxis\":{\"buckets\":null,\"mode\":\"time\",\"name\":null,\"show\":true,\"values\":[]},\"yaxes\":[{\"$$hashKey\":\"object:157\",\"format\":\"reqps\",\"label\":null,\"logBase\":1,\"max\":null,\"min\":null,\"show\":true},{\"$$hashKey\":\"object:158\",\"format\":\"short\",\"label\":null,\"logBase\":1,\"max\":null,\"min\":null,\"show\":false}],\"yaxis\":{\"align\":false,\"alignLevel\":null}},{\"aliasColors\":{},\"bars\":false,\"dashLength\":10,\"dashes\":false,\"datasource\":\"$datasource\",\"fieldConfig\":{\"defaults\":{\"links\":[]},\"overrides\":[]},\"fill\":1,\"fillGradient\":0,\"gridPos\":{\"h\":8,\"w\":12,\"x\":12,\"y\":5},\"hiddenSeries\":false,\"id\":4,\"legend\":{\"avg\":false,\"current\":false,\"max\":false,\"min\":false,\"show\":true,\"total\":false,\"values\":false},\"lines\":true,\"linewidth\":1,\"nullPointMode\":\"null\",\"options\":{\"alertThreshold\":true},\"percentage\":false,\"pluginVersion\":\"8.1.3\",\"pointradius\":2,\"points\":false,\"renderer\":\"flot\",\"seriesOverrides\":[],\"spaceLength\":10,\"stack\":false,\"steppedLine\":false,\"targets\":[{\"exemplar\":true,\"expr\":\"histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\\\"$instance\\\", job=~\\\"$job\\\"}[$__rate_interval])) by (le)) * 1\",\"interval\":\"1m\",\"legendFormat\":\"99th Percentile\",\"refId\":\"A\"},{\"exemplar\":true,\"expr\":\"histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\\\"$instance\\\", job=~\\\"$job\\\"}[$__rate_interval])) by (le)) * 1\",\"interval\":\"1m\",\"legendFormat\":\"50th Percentile\",\"refId\":\"B\"},{\"exemplar\":true,\"expr\":\"sum(irate(grafana_http_request_duration_seconds_sum{instance=~\\\"$instance\\\", job=~\\\"$job\\\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\\\"$instance\\\", job=~\\\"$job\\\"}[$__rate_interval]))\",\"interval\":\"1m\",\"legendFormat\":\"Average\",\"refId\":\"C\"}],\"thresholds\":[],\"timeFrom\":null,\"timeRegions\":[],\"timeShift\":null,\"title\":\"Request Latency\",\"tooltip\":{\"shared\":true,\"sort\":0,\"value_type\":\"individual\"},\"type\":\"graph\",\"xaxis\":{\"buckets\":null,\"mode\":\"time\",\"name\":null,\"show\":true,\"values\":[]},\"yaxes\":[{\"$$hashKey\":\"object:210\",\"format\":\"ms\",\"label\":null,\"logBase\":1,\"max\":null,\"min\":null,\"show\":true},{\"$$hashKey\":\"object:211\",\"format\":\"short\",\"label\":null,\"logBase\":1,\"max\":null,\"min\":null,\"show\":true}],\"yaxis\":{\"align\":false,\"alignLevel\":null}}],\"schemaVersion\":30,\"tags\":[],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"dev-cortex\",\"value\":\"dev-cortex\"},\"description\":null,\"error\":null,\"hide\":0,\"includeAll\":false,\"label\":null,\"multi\":false,\"name\":\"datasource\",\"options\":[],\"query\":\"prometheus\",\"queryValue\":\"\",\"refresh\":1,\"regex\":\"\",\"skipUrlSync\":false,\"type\":\"datasource\"},{\"allValue\":\".*\",\"current\":{\"selected\":false,\"text\":[\"default/grafana\"],\"value\":[\"default/grafana\"]},\"datasource\":\"$datasource\",\"definition\":\"label_values(grafana_build_info, job)\",\"description\":null,\"error\":null,\"hide\":0,\"includeAll\":true,\"label\":null,\"multi\":true,\"name\":\"job\",\"options\":[],\"query\":{\"query\":\"label_values(grafana_build_info, job)\",\"refId\":\"Billing Admin-job-Variable-Query\"},\"refresh\":1,\"regex\":\"\",\"skipUrlSync\":false,\"sort\":0,\"tagValuesQuery\":\"\",\"tagsQuery\":\"\",\"type\":\"query\",\"useTags\":false},{\"allValue\":\".*\",\"current\":{\"selected\":false,\"text\":\"All\",\"value\":\"$__all\"},\"datasource\":\"$datasource\",\"definition\":\"label_values(grafana_build_info, instance)\",\"description\":null,\"error\":null,\"hide\":0,\"includeAll\":true,\"label\":null,\"multi\":true,\"name\":\"instance\",\"options\":[],\"query\":{\"query\":\"label_values(grafana_build_info, instance)\",\"refId\":\"Billing Admin-instance-Variable-Query\"},\"refresh\":1,\"regex\":\"\",\"skipUrlSync\":false,\"sort\":0,\"tagValuesQuery\":\"\",\"tagsQuery\":\"\",\"type\":\"query\",\"useTags\":false}]},\"time\":{\"from\":\"now-6h\",\"to\":\"now\"},\"timepicker\":{\"refresh_intervals\":[\"10s\",\"30s\",\"1m\",\"5m\",\"15m\",\"30m\",\"1h\",\"2h\",\"1d\"]},\"timezone\": \"utc\",\"title\":\"Grafana Overview\",\"uid\":\"6be0s85Mk\",\"version\":2}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-k8s-coredns" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "k8s-coredns.json": "{\"annotations\":{\"list\":[{\"builtIn\":1,\"datasource\":{\"type\":\"datasource\",\"uid\":\"grafana\"},\"enable\":true,\"hide\":true,\"iconColor\":\"rgba(0, 211, 255, 1)\",\"name\":\"Annotations & Alerts\",\"type\":\"dashboard\"}]},\"description\":\"A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+. Based on the CoreDNS dashboard by buhay.\",\"editable\":true,\"fiscalYearStartMonth\":0,\"gnetId\":12539,\"graphTooltip\":0,\"id\":7,\"links\":[{\"icon\":\"external link\",\"tags\":[],\"targetBlank\":true,\"title\":\"CoreDNS.io\",\"type\":\"link\",\"url\":\"https://coredns.io\"}],\"liveNow\":false,\"panels\":[{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"normal\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"pps\",\"unitScale\":true},\"overrides\":[]},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":0},\"id\":2,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"sum(rate(coredns_dns_request_count_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (proto) or\\nsum(rate(coredns_dns_requests_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (proto)\",\"format\":\"time_series\",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}\",\"refId\":\"A\",\"step\":60}],\"title\":\"Requests (total)\",\"type\":\"timeseries\"},{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"normal\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"pps\",\"unitScale\":true},\"overrides\":[]},\"gridPos\":{\"h\":7,\"w\":8,\"x\":8,\"y\":0},\"id\":4,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"sum(rate(coredns_dns_request_type_count_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (type) or \\nsum(rate(coredns_dns_requests_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (type)\",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"{{ type }}\",\"refId\":\"A\",\"step\":60}],\"title\":\"Requests (by qtype)\",\"type\":\"timeseries\"},{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"normal\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"pps\",\"unitScale\":true},\"overrides\":[]},\"gridPos\":{\"h\":7,\"w\":8,\"x\":16,\"y\":0},\"id\":6,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"sum(rate(coredns_dns_request_count_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (zone) or\\nsum(rate(coredns_dns_requests_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (zone)\",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"{{ zone }}\",\"refId\":\"A\",\"step\":60}],\"title\":\"Requests (by zone)\",\"type\":\"timeseries\"},{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"none\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"pps\",\"unitScale\":true},\"overrides\":[]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":7},\"id\":8,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"sum(rate(coredns_dns_request_do_count_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) or\\nsum(rate(coredns_dns_do_requests_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m]))\",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"DO\",\"refId\":\"A\",\"step\":40},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"sum(rate(coredns_dns_request_count_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) or\\nsum(rate(coredns_dns_requests_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m]))\",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"total\",\"refId\":\"B\",\"step\":40}],\"title\":\"Requests (DO bit)\",\"type\":\"timeseries\"},{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"none\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"bytes\",\"unitScale\":true},\"overrides\":[{\"matcher\":{\"id\":\"byName\",\"options\":\"tcp:90\"},\"properties\":[{\"id\":\"unit\",\"value\":\"short\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"tcp:99 \"},\"properties\":[{\"id\":\"unit\",\"value\":\"short\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"tcp:50\"},\"properties\":[{\"id\":\"unit\",\"value\":\"short\"}]}]},\"gridPos\":{\"h\":7,\"w\":6,\"x\":12,\"y\":7},\"id\":10,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"none\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.99, (sum(rate(coredns_dns_request_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_request_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (le,proto)))\",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:99 \",\"refId\":\"A\",\"step\":60},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.90, (sum(rate(coredns_dns_request_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_request_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (le,proto)))\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:90\",\"refId\":\"B\",\"step\":60},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.50, (sum(rate(coredns_dns_request_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_request_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (le,proto)))\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:50\",\"refId\":\"C\",\"step\":60}],\"title\":\"Requests (size, udp)\",\"type\":\"timeseries\"},{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"none\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"bytes\",\"unitScale\":true},\"overrides\":[]},\"gridPos\":{\"h\":7,\"w\":6,\"x\":18,\"y\":7},\"id\":12,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"none\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.99, (sum(rate(coredns_dns_request_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_request_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (le,proto)))\",\"format\":\"time_series\",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:99 \",\"refId\":\"A\",\"step\":60},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.90, (sum(rate(coredns_dns_request_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_request_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (le,proto)))\",\"format\":\"time_series\",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:90\",\"refId\":\"B\",\"step\":60},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.50, (sum(rate(coredns_dns_request_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_request_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (le,proto)))\",\"format\":\"time_series\",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:50\",\"refId\":\"C\",\"step\":60}],\"title\":\"Requests (size,tcp)\",\"type\":\"timeseries\"},{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"normal\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"pps\",\"unitScale\":true},\"overrides\":[]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":14},\"id\":14,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"sum(rate(coredns_dns_response_rcode_count_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (rcode) or\\nsum(rate(coredns_dns_responses_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (rcode)\",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"{{ rcode }}\",\"refId\":\"A\",\"step\":40}],\"title\":\"Responses (by rcode)\",\"type\":\"timeseries\"},{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"none\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"s\",\"unitScale\":true},\"overrides\":[]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":14},\"id\":32,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"none\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.99, (sum(rate(coredns_dns_request_duration_seconds{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (job)) or (sum(rate(coredns_dns_request_duration_seconds_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (le, job)))\",\"format\":\"time_series\",\"intervalFactor\":2,\"legendFormat\":\"99%\",\"refId\":\"A\",\"step\":40},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.90, (sum(rate(coredns_dns_request_duration_seconds{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by ()) or (sum(rate(coredns_dns_request_duration_seconds_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (le)))\",\"format\":\"time_series\",\"intervalFactor\":2,\"legendFormat\":\"90%\",\"refId\":\"B\",\"step\":40},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.50, (sum(rate(coredns_dns_request_duration_seconds{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by ()) or (sum(rate(coredns_dns_request_duration_seconds_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (le)))\",\"format\":\"time_series\",\"intervalFactor\":2,\"legendFormat\":\"50%\",\"refId\":\"C\",\"step\":40}],\"title\":\"Responses (duration)\",\"type\":\"timeseries\"},{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"none\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"bytes\",\"unitScale\":true},\"overrides\":[{\"matcher\":{\"id\":\"byName\",\"options\":\"tcp:50%\"},\"properties\":[{\"id\":\"unit\",\"value\":\"short\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"tcp:90%\"},\"properties\":[{\"id\":\"unit\",\"value\":\"short\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"tcp:99%\"},\"properties\":[{\"id\":\"unit\",\"value\":\"short\"}]}]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":21},\"id\":18,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"none\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.99, (sum(rate(coredns_dns_response_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_response_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (le,proto))) \",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:99%\",\"refId\":\"A\",\"step\":40},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.90, (sum(rate(coredns_dns_response_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_response_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (le,proto))) \",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:90%\",\"refId\":\"B\",\"step\":40},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.50, (sum(rate(coredns_dns_response_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_response_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"udp\\\"}[5m])) by (le,proto))) \",\"hide\":false,\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:50%\",\"metric\":\"\",\"refId\":\"C\",\"step\":40}],\"title\":\"Responses (size, udp)\",\"type\":\"timeseries\"},{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"none\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"bytes\",\"unitScale\":true},\"overrides\":[]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":21},\"id\":20,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"none\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.99, (sum(rate(coredns_dns_response_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_response_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (le,proto))) \",\"format\":\"time_series\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:99%\",\"refId\":\"A\",\"step\":40},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.90, (sum(rate(coredns_dns_response_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_response_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (le,proto))) \",\"format\":\"time_series\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:90%\",\"refId\":\"B\",\"step\":40},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"histogram_quantile(0.50, (sum(rate(coredns_dns_response_size_bytes{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (proto)) or (sum(rate(coredns_dns_response_size_bytes_bucket{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\",proto=\\\"tcp\\\"}[5m])) by (le,proto))) \",\"format\":\"time_series\",\"intervalFactor\":2,\"legendFormat\":\"{{ proto }}:50%\",\"metric\":\"\",\"refId\":\"C\",\"step\":40}],\"title\":\"Responses (size, tcp)\",\"type\":\"timeseries\"},{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"normal\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"decbytes\",\"unitScale\":true},\"overrides\":[]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":28},\"id\":22,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"sum(coredns_cache_size{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}) by (type) or\\nsum(coredns_cache_entries{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}) by (type)\",\"interval\":\"1m\",\"intervalFactor\":2,\"legendFormat\":\"{{ type }}\",\"refId\":\"A\",\"step\":40}],\"title\":\"Cache (size)\",\"type\":\"timeseries\"},{\"datasource\":{\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"palette-classic\"},\"custom\":{\"axisBorderShow\":false,\"axisCenteredZero\":false,\"axisColorMode\":\"text\",\"axisLabel\":\"\",\"axisPlacement\":\"auto\",\"barAlignment\":0,\"drawStyle\":\"line\",\"fillOpacity\":10,\"gradientMode\":\"none\",\"hideFrom\":{\"legend\":false,\"tooltip\":false,\"viz\":false},\"insertNulls\":false,\"lineInterpolation\":\"linear\",\"lineWidth\":2,\"pointSize\":5,\"scaleDistribution\":{\"type\":\"linear\"},\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"group\":\"A\",\"mode\":\"normal\"},\"thresholdsStyle\":{\"mode\":\"off\"}},\"links\":[],\"mappings\":[],\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":80}]},\"unit\":\"pps\",\"unitScale\":true},\"overrides\":[]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":28},\"id\":24,\"links\":[],\"options\":{\"legend\":{\"calcs\":[],\"displayMode\":\"list\",\"placement\":\"bottom\",\"showLegend\":true},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"10.3.3\",\"targets\":[{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"sum(rate(coredns_cache_hits_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (type)\",\"hide\":false,\"intervalFactor\":2,\"legendFormat\":\"hits:{{ type }}\",\"refId\":\"A\",\"step\":40},{\"datasource\":{\"uid\":\"$datasource\"},\"expr\":\"sum(rate(coredns_cache_misses_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\",instance=~\\\"$instance\\\"}[5m])) by (type)\",\"hide\":false,\"intervalFactor\":2,\"legendFormat\":\"misses\",\"refId\":\"B\",\"step\":40}],\"title\":\"Cache (hitrate)\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"dns\",\"coredns\"],\"templating\":{\"list\":[{\"current\":{},\"hide\":0,\"includeAll\":false,\"multi\":false,\"name\":\"datasource\",\"options\":[],\"query\":\"prometheus\",\"queryValue\":\"\",\"refresh\":1,\"regex\":\"\",\"skipUrlSync\":false,\"type\":\"datasource\"},{\"allValue\":\".*\",\"current\":{\"selected\":false,\"text\":\"All\",\"value\":\"$__all\"},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"definition\":\"label_values(coredns_dns_requests_total, cluster)\",\"hide\":2,\"includeAll\":true,\"label\":\"Cluster\",\"multi\":false,\"name\":\"cluster\",\"options\":[],\"query\":\"label_values(coredns_dns_requests_total, cluster)\",\"refresh\":2,\"regex\":\"\",\"skipUrlSync\":false,\"sort\":1,\"tagValuesQuery\":\"\",\"tagsQuery\":\"\",\"type\":\"query\",\"useTags\":false},{\"allValue\":\".*\",\"current\":{\"selected\":false,\"text\":\"All\",\"value\":\"$__all\"},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"definition\":\"label_values(coredns_dns_requests_total{cluster=~\\\"$cluster\\\"},job)\",\"hide\":0,\"includeAll\":true,\"label\":\"Job\",\"multi\":false,\"name\":\"job\",\"options\":[],\"query\":{\"qryType\":1,\"query\":\"label_values(coredns_dns_requests_total{cluster=~\\\"$cluster\\\"},job)\",\"refId\":\"PrometheusVariableQueryEditor-VariableQuery\"},\"refresh\":2,\"regex\":\"\",\"skipUrlSync\":false,\"sort\":1,\"type\":\"query\"},{\"allValue\":\".*\",\"current\":{\"selected\":false,\"text\":\"All\",\"value\":\"$__all\"},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"definition\":\"label_values(coredns_dns_requests_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\"}, instance)\",\"hide\":0,\"includeAll\":true,\"label\":\"Instance\",\"multi\":false,\"name\":\"instance\",\"options\":[],\"query\":\"label_values(coredns_dns_requests_total{job=~\\\"$job\\\",cluster=~\\\"$cluster\\\"}, instance)\",\"refresh\":2,\"regex\":\"\",\"skipUrlSync\":false,\"sort\":3,\"tagValuesQuery\":\"\",\"tagsQuery\":\"\",\"type\":\"query\",\"useTags\":false}]},\"time\":{\"from\":\"now-3h\",\"to\":\"now\"},\"timepicker\":{\"refresh_intervals\":[\"10s\",\"30s\",\"1m\",\"5m\",\"15m\",\"30m\",\"1h\",\"2h\",\"1d\"]},\"timezone\": \"utc\",\"title\":\"CoreDNS\",\"uid\":\"vkQ0UHxik\",\"version\":3,\"weekStart\":\"\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-k8s-resources-cluster" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "k8s-resources-cluster.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"cluster:node_cpu:ratio_rate5m{cluster=\\\"$cluster\\\"}\",\"instant\":true}],\"title\":\"CPU Utilisation\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":4,\"y\":0},\"id\":2,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\\\"$cluster\\\"}) / sum(kube_node_status_allocatable{job=\\\"kube-state-metrics\\\",resource=\\\"cpu\\\",cluster=\\\"$cluster\\\"})\",\"instant\":true}],\"title\":\"CPU Requests Commitment\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":8,\"y\":0},\"id\":3,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\\\"$cluster\\\"}) / sum(kube_node_status_allocatable{job=\\\"kube-state-metrics\\\",resource=\\\"cpu\\\",cluster=\\\"$cluster\\\"})\",\"instant\":true}],\"title\":\"CPU Limits Commitment\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":12,\"y\":0},\"id\":4,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\\\"$cluster\\\"}) / sum(node_memory_MemTotal_bytes{job=\\\"node-exporter\\\",cluster=\\\"$cluster\\\"})\",\"instant\":true}],\"title\":\"Memory Utilisation\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":16,\"y\":0},\"id\":5,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\\\"$cluster\\\"}) / sum(kube_node_status_allocatable{job=\\\"kube-state-metrics\\\",resource=\\\"memory\\\",cluster=\\\"$cluster\\\"})\",\"instant\":true}],\"title\":\"Memory Requests Commitment\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":20,\"y\":0},\"id\":6,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\\\"$cluster\\\"}) / sum(kube_node_status_allocatable{job=\\\"kube-state-metrics\\\",resource=\\\"memory\\\",cluster=\\\"$cluster\\\"})\",\"instant\":true}],\"title\":\"Memory Limits Commitment\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true}}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":6},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\"}) by (namespace)\",\"legendFormat\":\"__auto\"}],\"title\":\"CPU Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Namespace\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?${datasource:queryparam}&var-cluster=$cluster&var-namespace=${__data.fields.Namespace}\"}]}]}]},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":12},\"id\":8,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_pod_owner{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\"}) by (workload, namespace)) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\\\"$cluster\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\\\"$cluster\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\\\"$cluster\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\\\"$cluster\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true}],\"title\":\"CPU Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"namespace\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true,\"Time 7\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Time 7\":6,\"Value #A\":8,\"Value #B\":9,\"Value #C\":10,\"Value #D\":11,\"Value #E\":12,\"Value #F\":13,\"Value #G\":14,\"namespace\":7},\"renameByName\":{\"Value #A\":\"Pods\",\"Value #B\":\"Workloads\",\"Value #C\":\"CPU Usage\",\"Value #D\":\"CPU Requests\",\"Value #E\":\"CPU Requests %\",\"Value #F\":\"CPU Limits\",\"Value #G\":\"CPU Limits %\",\"namespace\":\"Namespace\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"bytes\"}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":18},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_rss{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", container!=\\\"\\\"}) by (namespace)\",\"legendFormat\":\"__auto\"}],\"title\":\"Memory\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Memory Usage\"},\"properties\":[{\"id\":\"unit\",\"value\":\"bytes\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Memory Requests\"},\"properties\":[{\"id\":\"unit\",\"value\":\"bytes\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Memory Limits\"},\"properties\":[{\"id\":\"unit\",\"value\":\"bytes\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Namespace\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?${datasource:queryparam}&var-cluster=$cluster&var-namespace=${__data.fields.Namespace}\"}]}]}]},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":24},\"id\":10,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_pod_owner{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\"}) by (workload, namespace)) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_rss{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", container!=\\\"\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\\\"$cluster\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_rss{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", container!=\\\"\\\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\\\"$cluster\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\\\"$cluster\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_rss{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", container!=\\\"\\\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\\\"$cluster\\\"}) by (namespace)\",\"format\":\"table\",\"instant\":true}],\"title\":\"Memory Requests by Namespace\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"namespace\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true,\"Time 7\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Time 7\":6,\"Value #A\":8,\"Value #B\":9,\"Value #C\":10,\"Value #D\":11,\"Value #E\":12,\"Value #F\":13,\"Value #G\":14,\"namespace\":7},\"renameByName\":{\"Value #A\":\"Pods\",\"Value #B\":\"Workloads\",\"Value #C\":\"Memory Usage\",\"Value #D\":\"Memory Requests\",\"Value #E\":\"Memory Requests %\",\"Value #F\":\"Memory Limits\",\"Value #G\":\"Memory Limits %\",\"namespace\":\"Namespace\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Bandwidth/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"Bps\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Packets/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"pps\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Namespace\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?${datasource:queryparam}&var-cluster=$cluster&var-namespace=${__data.fields.Namespace}\"}]}]}]},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":30},\"id\":11,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"format\":\"table\",\"instant\":true}],\"title\":\"Current Network Usage\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"namespace\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Value #A\":7,\"Value #B\":8,\"Value #C\":9,\"Value #D\":10,\"Value #E\":11,\"Value #F\":12,\"namespace\":6},\"renameByName\":{\"Value #A\":\"Current Receive Bandwidth\",\"Value #B\":\"Current Transmit Bandwidth\",\"Value #C\":\"Rate of Received Packets\",\"Value #D\":\"Rate of Transmitted Packets\",\"Value #E\":\"Rate of Received Packets Dropped\",\"Value #F\":\"Rate of Transmitted Packets Dropped\",\"namespace\":\"Namespace\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":36},\"id\":12,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"legendFormat\":\"__auto\"}],\"title\":\"Receive Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":42},\"id\":13,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"legendFormat\":\"__auto\"}],\"title\":\"Transmit Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":48},\"id\":14,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"avg(irate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Container Bandwidth by Namespace: Received\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":54},\"id\":15,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"avg(irate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Container Bandwidth by Namespace: Transmitted\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":60},\"id\":16,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(irate(container_network_receive_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":66},\"id\":17,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(irate(container_network_transmit_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":72},\"id\":18,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(irate(container_network_receive_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":78},\"id\":19,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(irate(container_network_transmit_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=~\\\".+\\\"}[$__rate_interval])) by (namespace)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"iops\"}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":84},\"id\":20,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"ceil(sum by(namespace) (rate(container_fs_reads_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval])))\",\"legendFormat\":\"__auto\"}],\"title\":\"IOPS(Reads+Writes)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":90},\"id\":21,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(namespace) (rate(container_fs_reads_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval]))\",\"legendFormat\":\"__auto\"}],\"title\":\"ThroughPut(Read+Write)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/IOPS/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"iops\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Throughput/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"Bps\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Namespace\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?${datasource:queryparam}&var-cluster=$cluster&var-namespace=${__data.fields.Namespace}\"}]}]}]},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":96},\"id\":22,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(namespace) (rate(container_fs_reads_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(namespace) (rate(container_fs_writes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(namespace) (rate(container_fs_reads_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(namespace) (rate(container_fs_reads_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(namespace) (rate(container_fs_writes_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(namespace) (rate(container_fs_reads_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace!=\\\"\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true}],\"title\":\"Current Storage IO\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"namespace\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Value #A\":7,\"Value #B\":8,\"Value #C\":9,\"Value #D\":10,\"Value #E\":11,\"Value #F\":12,\"namespace\":6},\"renameByName\":{\"Value #A\":\"IOPS(Reads)\",\"Value #B\":\"IOPS(Writes)\",\"Value #C\":\"IOPS(Reads + Writes)\",\"Value #D\":\"Throughput(Read)\",\"Value #E\":\"Throughput(Write)\",\"Value #F\":\"Throughput(Read + Write)\",\"namespace\":\"Namespace\"}}}],\"type\":\"table\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Compute Resources / Cluster\",\"uid\":\"efa86fd1d0c121a26444b636a3f509a8\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-k8s-resources-multicluster" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "k8s-resources-multicluster.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"none\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:node_cpu:ratio_rate5m) / count(cluster:node_cpu:ratio_rate5m)\",\"instant\":true}],\"title\":\"CPU Utilisation\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":4,\"y\":0},\"id\":2,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", resource=\\\"cpu\\\"}) / sum(kube_node_status_allocatable{job=\\\"kube-state-metrics\\\", resource=\\\"cpu\\\"})\",\"instant\":true}],\"title\":\"CPU Requests Commitment\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":8,\"y\":0},\"id\":3,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", resource=\\\"cpu\\\"}) / sum(kube_node_status_allocatable{job=\\\"kube-state-metrics\\\", resource=\\\"cpu\\\"})\",\"instant\":true}],\"title\":\"CPU Limits Commitment\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":12,\"y\":0},\"id\":4,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\\\"node-exporter\\\"})\",\"instant\":true}],\"title\":\"Memory Utilisation\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":16,\"y\":0},\"id\":5,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", resource=\\\"memory\\\"}) / sum(kube_node_status_allocatable{job=\\\"kube-state-metrics\\\", resource=\\\"memory\\\"})\",\"instant\":true}],\"title\":\"Memory Requests Commitment\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":4,\"x\":20,\"y\":0},\"id\":6,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", resource=\\\"memory\\\"}) / sum(kube_node_status_allocatable{job=\\\"kube-state-metrics\\\", resource=\\\"memory\\\"})\",\"instant\":true}],\"title\":\"Memory Limits Commitment\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"}}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":1},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m) by (cluster)\",\"legendFormat\":\"__auto\"}],\"title\":\"CPU Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Cluster\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down\",\"url\":\"/d/efa86fd1d0c121a26444b636a3f509a8/kubernetes-compute-resources-cluster?${datasource:queryparam}&var-cluster=${__data.fields.Cluster}\"}]}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":2},\"id\":8,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m) by (cluster)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", resource=\\\"cpu\\\"}) by (cluster)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m) by (cluster) / sum(kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", resource=\\\"cpu\\\"}) by (cluster)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", resource=\\\"cpu\\\"}) by (cluster)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m) by (cluster) / sum(kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", resource=\\\"cpu\\\"}) by (cluster)\",\"format\":\"table\",\"instant\":true}],\"title\":\"CPU Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"cluster\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Value #A\":6,\"Value #B\":7,\"Value #C\":8,\"Value #D\":9,\"Value #E\":10,\"cluster\":5},\"renameByName\":{\"Value #A\":\"CPU Usage\",\"Value #B\":\"CPU Requests\",\"Value #C\":\"CPU Requests %\",\"Value #D\":\"CPU Limits\",\"Value #E\":\"CPU Limits %\",\"cluster\":\"Cluster\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":3},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_rss{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\"}) by (cluster)\",\"legendFormat\":\"__auto\"}],\"title\":\"Memory Usage (w/o cache)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"bytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Cluster\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down\",\"url\":\"/d/efa86fd1d0c121a26444b636a3f509a8/kubernetes-compute-resources-cluster?${datasource:queryparam}&var-cluster=${__data.fields.Cluster}\"}]}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":4},\"id\":10,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_rss{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\"}) by (cluster)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", resource=\\\"memory\\\"}) by (cluster)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_rss{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", resource=\\\"memory\\\"}) by (cluster)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", resource=\\\"memory\\\"}) by (cluster)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_rss{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", resource=\\\"memory\\\"}) by (cluster)\",\"format\":\"table\",\"instant\":true}],\"title\":\"Memory Requests by Cluster\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"cluster\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Value #A\":6,\"Value #B\":7,\"Value #C\":8,\"Value #D\":9,\"Value #E\":10,\"cluster\":5},\"renameByName\":{\"Value #A\":\"Memory Usage\",\"Value #B\":\"Memory Requests\",\"Value #C\":\"Memory Requests %\",\"Value #D\":\"Memory Limits\",\"Value #E\":\"Memory Limits %\",\"cluster\":\"Cluster\"}}}],\"type\":\"table\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Compute Resources / Multi-Cluster\",\"uid\":\"b59e6c9f2fcbe2e16d77fc492374cc4f\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-k8s-resources-namespace" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "k8s-resources-namespace.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":6,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) / sum(kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"cpu\\\"})\",\"instant\":true}],\"title\":\"CPU Utilisation (from requests)\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":6,\"x\":6,\"y\":0},\"id\":2,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) / sum(kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"cpu\\\"})\",\"instant\":true}],\"title\":\"CPU Utilisation (from limits)\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":6,\"x\":12,\"y\":0},\"id\":3,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\",container!=\\\"\\\", image!=\\\"\\\"}) / sum(kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"memory\\\"})\",\"instant\":true}],\"title\":\"Memory Utilisation (from requests)\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":3,\"w\":6,\"x\":18,\"y\":0},\"id\":4,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\",container!=\\\"\\\", image!=\\\"\\\"}) / sum(kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"memory\\\"})\",\"instant\":true}],\"title\":\"Memory Utilisation (from limits)\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true}},\"overrides\":[{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"B\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"red\",\"mode\":\"fixed\"}}]},{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"C\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"orange\",\"mode\":\"fixed\"}}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":7},\"id\":5,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod)\",\"legendFormat\":\"__auto\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"scalar(max(kube_resourcequota{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", type=\\\"hard\\\",resource=\\\"requests.cpu\\\"}))\",\"legendFormat\":\"quota - requests\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"scalar(max(kube_resourcequota{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", type=\\\"hard\\\",resource=\\\"limits.cpu\\\"}))\",\"legendFormat\":\"quota - limits\"}],\"title\":\"CPU Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Pod\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}\"}]}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":14},\"id\":6,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true}],\"title\":\"CPU Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"pod\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Value #A\":6,\"Value #B\":7,\"Value #C\":8,\"Value #D\":9,\"Value #E\":10,\"pod\":5},\"renameByName\":{\"Value #A\":\"CPU Usage\",\"Value #B\":\"CPU Requests\",\"Value #C\":\"CPU Requests %\",\"Value #D\":\"CPU Limits\",\"Value #E\":\"CPU Limits %\",\"pod\":\"Pod\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"bytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"B\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"red\",\"mode\":\"fixed\"}}]},{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"C\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"orange\",\"mode\":\"fixed\"}}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":21},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", container!=\\\"\\\", image!=\\\"\\\"}) by (pod)\",\"legendFormat\":\"__auto\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"scalar(max(kube_resourcequota{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", type=\\\"hard\\\",resource=\\\"requests.memory\\\"}))\",\"legendFormat\":\"quota - requests\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"scalar(max(kube_resourcequota{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", type=\\\"hard\\\",resource=\\\"limits.memory\\\"}))\",\"legendFormat\":\"quota - limits\"}],\"title\":\"Memory Usage (w/o cache)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"bytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Pod\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}\"}]}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":28},\"id\":8,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\",container!=\\\"\\\", image!=\\\"\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\",container!=\\\"\\\", image!=\\\"\\\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\",container!=\\\"\\\", image!=\\\"\\\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_rss{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\",container!=\\\"\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_cache{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\",container!=\\\"\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_swap{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\",container!=\\\"\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true}],\"title\":\"Memory Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"pod\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true,\"Time 7\":true,\"Time 8\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Time 7\":6,\"Time 8\":7,\"Value #A\":9,\"Value #B\":10,\"Value #C\":11,\"Value #D\":12,\"Value #E\":13,\"Value #F\":14,\"Value #G\":15,\"Value #H\":16,\"pod\":8},\"renameByName\":{\"Value #A\":\"Memory Usage\",\"Value #B\":\"Memory Requests\",\"Value #C\":\"Memory Requests %\",\"Value #D\":\"Memory Limits\",\"Value #E\":\"Memory Limits %\",\"Value #F\":\"Memory Usage (RSS)\",\"Value #G\":\"Memory Usage (Cache)\",\"Value #H\":\"Memory Usage (Swap)\",\"pod\":\"Pod\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Bandwidth/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"Bps\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Packets/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"pps\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Pod\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}\"}]}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":35},\"id\":9,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"format\":\"table\",\"instant\":true}],\"title\":\"Current Network Usage\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"pod\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Value #A\":7,\"Value #B\":8,\"Value #C\":9,\"Value #D\":10,\"Value #E\":11,\"Value #F\":12,\"pod\":6},\"renameByName\":{\"Value #A\":\"Current Receive Bandwidth\",\"Value #B\":\"Current Transmit Bandwidth\",\"Value #C\":\"Rate of Received Packets\",\"Value #D\":\"Rate of Transmitted Packets\",\"Value #E\":\"Rate of Received Packets Dropped\",\"Value #F\":\"Rate of Transmitted Packets Dropped\",\"pod\":\"Pod\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":42},\"id\":10,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Receive Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":42},\"id\":11,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Transmit Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":49},\"id\":12,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(irate(container_network_receive_packets_total{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":49},\"id\":13,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(irate(container_network_transmit_packets_total{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":56},\"id\":14,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(irate(container_network_receive_packets_dropped_total{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":56},\"id\":15,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(irate(container_network_transmit_packets_dropped_total{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"iops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":63},\"id\":16,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"ceil(sum by(pod) (rate(container_fs_reads_total{container!=\\\"\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\\\"\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])))\",\"legendFormat\":\"__auto\"}],\"title\":\"IOPS(Reads+Writes)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":63},\"id\":17,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(pod) (rate(container_fs_reads_bytes_total{container!=\\\"\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\\\"\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval]))\",\"legendFormat\":\"__auto\"}],\"title\":\"ThroughPut(Read+Write)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/IOPS/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"iops\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Throughput/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"Bps\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Pod\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}\"}]}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":70},\"id\":18,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(pod) (rate(container_fs_reads_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(pod) (rate(container_fs_writes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(pod) (rate(container_fs_reads_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(pod) (rate(container_fs_reads_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(pod) (rate(container_fs_writes_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(pod) (rate(container_fs_reads_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true}],\"title\":\"Current Storage IO\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"pod\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Value #A\":7,\"Value #B\":8,\"Value #C\":9,\"Value #D\":10,\"Value #E\":11,\"Value #F\":12,\"pod\":6},\"renameByName\":{\"Value #A\":\"IOPS(Reads)\",\"Value #B\":\"IOPS(Writes)\",\"Value #C\":\"IOPS(Reads + Writes)\",\"Value #D\":\"Throughput(Read)\",\"Value #E\":\"Throughput(Write)\",\"Value #F\":\"Throughput(Read + Write)\",\"pod\":\"Pod\"}}}],\"type\":\"table\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kube-state-metrics\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"namespace\",\"name\":\"namespace\",\"query\":\"label_values(kube_namespace_status_phase{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\"}, namespace)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Compute Resources / Namespace (Pods)\",\"uid\":\"85a562078cdf77779eaa1add43ccec1e\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-k8s-resources-node" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "k8s-resources-node.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"mode\":\"normal\"}}},\"overrides\":[{\"matcher\":{\"id\":\"byName\",\"options\":\"max capacity\"},\"properties\":[{\"id\":\"color\",\"value\":{\"fixedColor\":\"red\",\"mode\":\"fixed\"}},{\"id\":\"custom.stacking\",\"value\":{\"mode\":\"none\"}},{\"id\":\"custom.hideFrom\",\"value\":{\"legend\":false,\"tooltip\":true,\"viz\":false}},{\"id\":\"custom.lineStyle\",\"value\":{\"dash\":[10,10],\"fill\":\"dash\"}}]}]},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_node_status_capacity{cluster=\\\"$cluster\\\", job=\\\"kube-state-metrics\\\", node=~\\\"$node\\\", resource=\\\"cpu\\\"})\",\"legendFormat\":\"max capacity\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod)\",\"legendFormat\":\"{{pod}}\"}],\"title\":\"CPU Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Pod\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}\"}]}]}]},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":6},\"id\":2,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true}],\"title\":\"CPU Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"pod\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true},\"renameByName\":{\"Value #A\":\"CPU Usage\",\"Value #B\":\"CPU Requests\",\"Value #C\":\"CPU Requests %\",\"Value #D\":\"CPU Limits\",\"Value #E\":\"CPU Limits %\",\"pod\":\"Pod\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"bytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byName\",\"options\":\"max capacity\"},\"properties\":[{\"id\":\"color\",\"value\":{\"fixedColor\":\"red\",\"mode\":\"fixed\"}},{\"id\":\"custom.stacking\",\"value\":{\"mode\":\"none\"}},{\"id\":\"custom.hideFrom\",\"value\":{\"legend\":false,\"tooltip\":true,\"viz\":false}},{\"id\":\"custom.lineStyle\",\"value\":{\"dash\":[10,10],\"fill\":\"dash\"}}]}]},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":12},\"id\":3,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_node_status_capacity{cluster=\\\"$cluster\\\", job=\\\"kube-state-metrics\\\", node=~\\\"$node\\\", resource=\\\"memory\\\"})\",\"legendFormat\":\"max capacity\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\\\"$cluster\\\", node=~\\\"$node\\\", container!=\\\"\\\"}) by (pod)\",\"legendFormat\":\"{{pod}}\"}],\"title\":\"Memory Usage (w/cache)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true,\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"bytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byName\",\"options\":\"max capacity\"},\"properties\":[{\"id\":\"color\",\"value\":{\"fixedColor\":\"red\",\"mode\":\"fixed\"}},{\"id\":\"custom.stacking\",\"value\":{\"mode\":\"none\"}},{\"id\":\"custom.hideFrom\",\"value\":{\"legend\":false,\"tooltip\":true,\"viz\":false}},{\"id\":\"custom.lineStyle\",\"value\":{\"dash\":[10,10],\"fill\":\"dash\"}}]}]},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":18},\"id\":4,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kube_node_status_capacity{cluster=\\\"$cluster\\\", job=\\\"kube-state-metrics\\\", node=~\\\"$node\\\", resource=\\\"memory\\\"})\",\"legendFormat\":\"max capacity\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_memory_rss{cluster=\\\"$cluster\\\", node=~\\\"$node\\\", container!=\\\"\\\"}) by (pod)\",\"legendFormat\":\"{{pod}}\"}],\"title\":\"Memory Usage (w/o cache)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"bytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Pod\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}\"}]}]}]},\"gridPos\":{\"h\":6,\"w\":24,\"x\":0,\"y\":24},\"id\":5,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\\\"$cluster\\\", node=~\\\"$node\\\",container!=\\\"\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\\\"$cluster\\\", node=~\\\"$node\\\",container!=\\\"\\\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\\\"$cluster\\\", node=~\\\"$node\\\",container!=\\\"\\\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", node=~\\\"$node\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_memory_rss{cluster=\\\"$cluster\\\", node=~\\\"$node\\\",container!=\\\"\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_memory_cache{cluster=\\\"$cluster\\\", node=~\\\"$node\\\",container!=\\\"\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_memory_swap{cluster=\\\"$cluster\\\", node=~\\\"$node\\\",container!=\\\"\\\"}) by (pod)\",\"format\":\"table\",\"instant\":true}],\"title\":\"Memory Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"pod\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true,\"Time 7\":true,\"Time 8\":true},\"renameByName\":{\"Value #A\":\"Memory Usage\",\"Value #B\":\"Memory Requests\",\"Value #C\":\"Memory Requests %\",\"Value #D\":\"Memory Limits\",\"Value #E\":\"Memory Limits %\",\"Value #F\":\"Memory Usage (RSS)\",\"Value #G\":\"Memory Usage (Cache)\",\"Value #H\":\"Memory Usage (Swap)\",\"pod\":\"Pod\"}}}],\"type\":\"table\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kube-state-metrics\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"node\",\"multi\":true,\"name\":\"node\",\"query\":\"label_values(kube_node_info{cluster=\\\"$cluster\\\"}, node)\",\"refresh\":2,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Compute Resources / Node (Pods)\",\"uid\":\"200ac8fdbfbb74b39aff88118e4d1c2c\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-k8s-resources-pod" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "k8s-resources-pod.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true}},\"overrides\":[{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"B\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"red\",\"mode\":\"fixed\"}}]},{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"C\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"orange\",\"mode\":\"fixed\"}}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", cluster=\\\"$cluster\\\", container!=\\\"\\\"}) by (container)\",\"legendFormat\":\"__auto\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", resource=\\\"cpu\\\"}\\n)\\n\",\"legendFormat\":\"requests\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", resource=\\\"cpu\\\"}\\n)\\n\",\"legendFormat\":\"limits\"}],\"title\":\"CPU Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"axisColorMode\":\"thresholds\",\"axisSoftMax\":1,\"axisSoftMin\":0,\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true,\"thresholdsStyle\":{\"mode\":\"dashed+area\"}},\"unit\":\"percentunit\"},\"overrides\":[{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"A\"},\"properties\":[{\"id\":\"thresholds\",\"value\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":null},{\"color\":\"red\",\"value\":0.25}]}},{\"id\":\"color\",\"value\":{\"mode\":\"thresholds\",\"seriesBy\":\"lastNotNull\"}}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":7},\"id\":2,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(increase(container_cpu_cfs_throttled_periods_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\"}[$__rate_interval])) by (container)\",\"legendFormat\":\"__auto\"}],\"title\":\"CPU Throttling\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":14},\"id\":3,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\"}) by (container)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\"}) by (container)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\"}) by (container)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\"}) by (container)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\"}) by (container)\",\"format\":\"table\",\"instant\":true}],\"title\":\"CPU Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"container\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Value #A\":6,\"Value #B\":7,\"Value #C\":8,\"Value #D\":9,\"Value #E\":10,\"container\":5},\"renameByName\":{\"Value #A\":\"CPU Usage\",\"Value #B\":\"CPU Requests\",\"Value #C\":\"CPU Requests %\",\"Value #D\":\"CPU Limits\",\"Value #E\":\"CPU Limits %\",\"container\":\"Container\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"bytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"B\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"red\",\"mode\":\"fixed\"}}]},{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"C\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"orange\",\"mode\":\"fixed\"}}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":21},\"id\":4,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\", image!=\\\"\\\"}) by (container)\",\"legendFormat\":\"__auto\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", resource=\\\"memory\\\"}\\n)\\n\",\"legendFormat\":\"requests\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", resource=\\\"memory\\\"}\\n)\\n\",\"legendFormat\":\"limits\"}],\"title\":\"Memory Usage (WSS)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"bytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":28},\"id\":5,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\", image!=\\\"\\\"}) by (container)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}) by (container)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", image!=\\\"\\\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}) by (container)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}) by (container)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container!=\\\"\\\", image!=\\\"\\\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}) by (container)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_rss{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container != \\\"\\\", container != \\\"POD\\\"}) by (container)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_cache{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container != \\\"\\\", container != \\\"POD\\\"}) by (container)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(container_memory_swap{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\", container != \\\"\\\", container != \\\"POD\\\"}) by (container)\",\"format\":\"table\",\"instant\":true}],\"title\":\"Memory Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"container\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true,\"Time 7\":true,\"Time 8\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Time 7\":6,\"Time 8\":7,\"Value #A\":9,\"Value #B\":10,\"Value #C\":11,\"Value #D\":12,\"Value #E\":13,\"Value #F\":14,\"Value #G\":15,\"Value #H\":16,\"container\":8},\"renameByName\":{\"Value #A\":\"Memory Usage\",\"Value #B\":\"Memory Requests\",\"Value #C\":\"Memory Requests %\",\"Value #D\":\"Memory Limits\",\"Value #E\":\"Memory Limits %\",\"Value #F\":\"Memory Usage (RSS)\",\"Value #G\":\"Memory Usage (Cache)\",\"Value #H\":\"Memory Usage (Swap)\",\"container\":\"Container\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":35},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(irate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Receive Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":35},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Transmit Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":42},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":42},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":49},\"id\":10,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":49},\"id\":11,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"iops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":56},\"id\":12,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"ceil(sum by(pod) (rate(container_fs_reads_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])))\",\"legendFormat\":\"Reads\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"ceil(sum by(pod) (rate(container_fs_writes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])))\",\"legendFormat\":\"Writes\"}],\"title\":\"IOPS (Pod)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":56},\"id\":13,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(pod) (rate(container_fs_reads_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval]))\",\"legendFormat\":\"Reads\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(pod) (rate(container_fs_writes_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval]))\",\"legendFormat\":\"Writes\"}],\"title\":\"ThroughPut (Pod)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"iops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":63},\"id\":14,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"ceil(sum by(container) (rate(container_fs_reads_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval])))\",\"legendFormat\":\"__auto\"}],\"title\":\"IOPS (Containers)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":63},\"id\":15,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(container) (rate(container_fs_reads_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval]))\",\"legendFormat\":\"__auto\"}],\"title\":\"ThroughPut (Containers)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/IOPS/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"iops\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Throughput/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"Bps\"}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":70},\"id\":16,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(container) (rate(container_fs_reads_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(container) (rate(container_fs_writes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\",device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(container) (rate(container_fs_reads_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(container) (rate(container_fs_reads_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(container) (rate(container_fs_writes_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by(container) (rate(container_fs_reads_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\", container!=\\\"\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}[$__rate_interval]))\",\"format\":\"table\",\"instant\":true}],\"title\":\"Current Storage IO\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"container\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Value #A\":7,\"Value #B\":8,\"Value #C\":9,\"Value #D\":10,\"Value #E\":11,\"Value #F\":12,\"container\":6},\"renameByName\":{\"Value #A\":\"IOPS(Reads)\",\"Value #B\":\"IOPS(Writes)\",\"Value #C\":\"IOPS(Reads + Writes)\",\"Value #D\":\"Throughput(Read)\",\"Value #E\":\"Throughput(Write)\",\"Value #F\":\"Throughput(Read + Write)\",\"container\":\"Container\"}}}],\"type\":\"table\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kube-state-metrics\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"namespace\",\"name\":\"namespace\",\"query\":\"label_values(kube_namespace_status_phase{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\"}, namespace)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"pod\",\"name\":\"pod\",\"query\":\"label_values(kube_pod_info{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}, pod)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Compute Resources / Pod\",\"uid\":\"6581e46e4e5c7ba40a07646395ef7b23\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-k8s-resources-workload" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "k8s-resources-workload.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true}}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"CPU Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Pod\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}\"}]}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":7},\"id\":2,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"cpu\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n/sum(\\n kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"cpu\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"cpu\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n/sum(\\n kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"cpu\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"format\":\"table\",\"instant\":true}],\"title\":\"CPU Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"pod\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Value #A\":6,\"Value #B\":7,\"Value #C\":8,\"Value #D\":9,\"Value #E\":10,\"pod\":5},\"renameByName\":{\"Value #A\":\"CPU Usage\",\"Value #B\":\"CPU Requests\",\"Value #C\":\"CPU Requests %\",\"Value #D\":\"CPU Limits\",\"Value #E\":\"CPU Limits %\",\"pod\":\"Pod\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":14},\"id\":3,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n container_memory_working_set_bytes{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", container!=\\\"\\\", image!=\\\"\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Memory Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"bytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Pod\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}\"}]}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":21},\"id\":4,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n container_memory_working_set_bytes{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", container!=\\\"\\\", image!=\\\"\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"memory\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n container_memory_working_set_bytes{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", container!=\\\"\\\", image!=\\\"\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n/sum(\\n kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"memory\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"memory\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n container_memory_working_set_bytes{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", container!=\\\"\\\", image!=\\\"\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n/sum(\\n kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"memory\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=\\\"$workload\\\", workload_type=~\\\"$type\\\"}\\n) by (pod)\\n\",\"format\":\"table\",\"instant\":true}],\"title\":\"Memory Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"pod\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Value #A\":9,\"Value #B\":10,\"Value #C\":11,\"Value #D\":12,\"Value #E\":13,\"pod\":8},\"renameByName\":{\"Value #A\":\"Memory Usage\",\"Value #B\":\"Memory Requests\",\"Value #C\":\"Memory Requests %\",\"Value #D\":\"Memory Limits\",\"Value #E\":\"Memory Limits %\",\"pod\":\"Pod\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Bandwidth/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"Bps\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Packets/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"pps\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Pod\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to pods\",\"url\":\"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}\"}]}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":28},\"id\":5,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"format\":\"table\",\"instant\":true}],\"title\":\"Current Network Usage\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"pod\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Value #A\":7,\"Value #B\":8,\"Value #C\":9,\"Value #D\":10,\"Value #E\":11,\"Value #F\":12,\"pod\":6},\"renameByName\":{\"Value #A\":\"Current Receive Bandwidth\",\"Value #B\":\"Current Transmit Bandwidth\",\"Value #C\":\"Rate of Received Packets\",\"Value #D\":\"Rate of Transmitted Packets\",\"Value #E\":\"Rate of Received Packets Dropped\",\"Value #F\":\"Rate of Transmitted Packets Dropped\",\"pod\":\"Pod\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":35},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Receive Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":35},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Transmit Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":42},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(avg(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Container Bandwidth by Pod: Received\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":42},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(avg(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Container Bandwidth by Pod: Transmitted\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":49},\"id\":10,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":49},\"id\":11,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":56},\"id\":12,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":56},\"id\":13,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets Dropped\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kube-state-metrics\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"namespace\",\"name\":\"namespace\",\"query\":\"label_values(kube_namespace_status_phase{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\"}, namespace)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"label\":\"workload_type\",\"name\":\"type\",\"query\":\"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}, workload_type)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"workload\",\"name\":\"workload\",\"query\":\"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}, workload)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Compute Resources / Workload\",\"uid\":\"a164a7f0339f99e89cea5cb47e9be617\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-k8s-resources-workloads-namespace" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "k8s-resources-workloads-namespace.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true}},\"overrides\":[{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"B\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"red\",\"mode\":\"fixed\"}}]},{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"C\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"orange\",\"mode\":\"fixed\"}}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"legendFormat\":\"{{workload}} - {{workload_type}}\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"scalar(max(kube_resourcequota{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", type=\\\"hard\\\",resource=~\\\"requests.cpu|cpu\\\"}))\",\"legendFormat\":\"quota - requests\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"scalar(max(kube_resourcequota{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", type=\\\"hard\\\",resource=~\\\"limits.cpu\\\"}))\",\"legendFormat\":\"quota - limits\"}],\"title\":\"CPU Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Workload\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to workloads\",\"url\":\"/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-type=${__data.fields.Type}&var-workload=${__data.fields.Workload}\"}]}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Running Pods\"},\"properties\":[{\"id\":\"unit\",\"value\":\"none\"}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":7},\"id\":2,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}) by (workload, workload_type)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"cpu\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n/sum(\\n kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"cpu\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"cpu\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n/sum(\\n kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"cpu\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"format\":\"table\",\"instant\":true}],\"title\":\"CPU Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"workload\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true,\"workload_type 2\":true,\"workload_type 3\":true,\"workload_type 4\":true,\"workload_type 5\":true,\"workload_type 6\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Value #A\":8,\"Value #B\":9,\"Value #C\":10,\"Value #D\":11,\"Value #E\":12,\"Value #F\":13,\"workload\":6,\"workload_type 1\":7,\"workload_type 2\":14,\"workload_type 3\":15,\"workload_type 4\":16,\"workload_type 5\":17,\"workload_type 6\":18},\"renameByName\":{\"Value #A\":\"Running Pods\",\"Value #B\":\"CPU Usage\",\"Value #C\":\"CPU Requests\",\"Value #D\":\"CPU Requests %\",\"Value #E\":\"CPU Limits\",\"Value #F\":\"CPU Limits %\",\"workload\":\"Workload\",\"workload_type 1\":\"Type\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"bytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"B\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"red\",\"mode\":\"fixed\"}}]},{\"matcher\":{\"id\":\"byFrameRefID\",\"options\":\"C\"},\"properties\":[{\"id\":\"custom.lineStyle\",\"value\":{\"fill\":\"dash\"}},{\"id\":\"custom.lineWidth\",\"value\":2},{\"id\":\"color\",\"value\":{\"fixedColor\":\"orange\",\"mode\":\"fixed\"}}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":14},\"id\":3,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", container!=\\\"\\\", image!=\\\"\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"legendFormat\":\"{{workload}} - {{workload_type}}\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"scalar(max(kube_resourcequota{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", type=\\\"hard\\\",resource=~\\\"requests.memory|memory\\\"}))\",\"legendFormat\":\"quota - requests\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"scalar(max(kube_resourcequota{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", type=\\\"hard\\\",resource=~\\\"limits.memory\\\"}))\",\"legendFormat\":\"quota - limits\"}],\"title\":\"Memory Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"bytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/%/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Workload\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to workloads\",\"url\":\"/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-type=${__data.fields.Type}&var-workload=${__data.fields.Workload}\"}]}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Running Pods\"},\"properties\":[{\"id\":\"unit\",\"value\":\"none\"}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":21},\"id\":4,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}) by (workload, workload_type)\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", container!=\\\"\\\", image!=\\\"\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"memory\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", container!=\\\"\\\", image!=\\\"\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n/sum(\\n kube_pod_container_resource_requests{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"memory\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"memory\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(\\n container_memory_working_set_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", container!=\\\"\\\", image!=\\\"\\\"}\\n * on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n/sum(\\n kube_pod_container_resource_limits{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", resource=\\\"memory\\\"}\\n* on(namespace,pod)\\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}\\n) by (workload, workload_type)\\n\",\"format\":\"table\",\"instant\":true}],\"title\":\"Memory Quota\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"workload\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true,\"workload_type 2\":true,\"workload_type 3\":true,\"workload_type 4\":true,\"workload_type 5\":true,\"workload_type 6\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Value #A\":8,\"Value #B\":9,\"Value #C\":10,\"Value #D\":11,\"Value #E\":12,\"Value #F\":13,\"workload\":6,\"workload_type 1\":7,\"workload_type 2\":14,\"workload_type 3\":15,\"workload_type 4\":16,\"workload_type 5\":17,\"workload_type 6\":18},\"renameByName\":{\"Value #A\":\"Running Pods\",\"Value #B\":\"Memory Usage\",\"Value #C\":\"Memory Requests\",\"Value #D\":\"Memory Requests %\",\"Value #E\":\"Memory Limits\",\"Value #F\":\"Memory Limits %\",\"workload\":\"Workload\",\"workload_type 1\":\"Type\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Bandwidth/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"Bps\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Packets/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"pps\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Workload\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down to workloads\",\"url\":\"/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-type=${__data.fields.Type}&var-workload=${__data.fields.Workload}\"}]}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":28},\"id\":5,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"format\":\"table\",\"instant\":true}],\"title\":\"Current Network Usage\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"workload\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Value #A\":7,\"Value #B\":8,\"Value #C\":9,\"Value #D\":10,\"Value #E\":11,\"Value #F\":12,\"workload\":6},\"renameByName\":{\"Value #A\":\"Current Receive Bandwidth\",\"Value #B\":\"Current Transmit Bandwidth\",\"Value #C\":\"Rate of Received Packets\",\"Value #D\":\"Rate of Transmitted Packets\",\"Value #E\":\"Rate of Received Packets Dropped\",\"Value #F\":\"Rate of Transmitted Packets Dropped\",\"workload\":\"Workload\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":35},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Receive Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":35},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Transmit Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":42},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(avg(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Container Bandwidth by Workload: Received\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":42},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(avg(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Container Bandwidth by Workload: Transmitted\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":49},\"id\":10,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":49},\"id\":11,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":56},\"id\":12,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_receive_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":56},\"id\":13,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(sum(rate(container_network_transmit_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets Dropped\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kube-state-metrics\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"namespace\",\"name\":\"namespace\",\"query\":\"label_values(kube_namespace_status_phase{job=\\\"kube-state-metrics\\\", cluster=\\\"$cluster\\\"}, namespace)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"label\":\"workload_type\",\"name\":\"type\",\"query\":\"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\".+\\\"}, workload_type)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Compute Resources / Namespace (Workloads)\",\"uid\":\"a87fb0d919ec0ea5f6543124e16c42a5\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-kubelet" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "kubelet.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"none\"}},\"gridPos\":{\"h\":7,\"w\":4,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kubelet_node_name{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\"})\",\"instant\":true}],\"title\":\"Running Kubelets\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"none\"}},\"gridPos\":{\"h\":7,\"w\":4,\"x\":4,\"y\":0},\"id\":2,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kubelet_running_pods{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\"})\",\"instant\":true}],\"title\":\"Running Pods\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"none\"}},\"gridPos\":{\"h\":7,\"w\":4,\"x\":8,\"y\":0},\"id\":3,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(kubelet_running_containers{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\"})\",\"instant\":true}],\"title\":\"Running Containers\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"none\"}},\"gridPos\":{\"h\":7,\"w\":4,\"x\":12,\"y\":0},\"id\":4,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(volume_manager_total_volumes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\", state=\\\"actual_state_of_world\\\"})\",\"instant\":true}],\"title\":\"Actual Volume Count\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"none\"}},\"gridPos\":{\"h\":7,\"w\":4,\"x\":16,\"y\":0},\"id\":5,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(volume_manager_total_volumes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\",state=\\\"desired_state_of_world\\\"})\",\"instant\":true}],\"title\":\"Desired Volume Count\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"none\"}},\"gridPos\":{\"h\":7,\"w\":4,\"x\":20,\"y\":0},\"id\":6,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(kubelet_node_config_error{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\"}[$__rate_interval]))\",\"instant\":true}],\"title\":\"Config Error Count\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":7},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(kubelet_runtime_operations_total{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (operation_type, instance)\",\"legendFormat\":\"{{instance}} {{operation_type}}\"}],\"title\":\"Operation Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":7},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(kubelet_runtime_operations_errors_total{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, operation_type)\",\"legendFormat\":\"{{instance}} {{operation_type}}\"}],\"title\":\"Operation Error Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":14},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, operation_type, le))\",\"legendFormat\":\"{{instance}} {{operation_type}}\"}],\"title\":\"Operation Duration 99th quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":21},\"id\":10,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance)\",\"legendFormat\":\"{{instance}} pod\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance)\",\"legendFormat\":\"{{instance}} worker\"}],\"title\":\"Pod Start Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":21},\"id\":11,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, le))\",\"legendFormat\":\"{{instance}} pod\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, le))\",\"legendFormat\":\"{{instance}} worker\"}],\"title\":\"Pod Start Duration\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":28},\"id\":12,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(storage_operation_duration_seconds_count{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)\",\"legendFormat\":\"{{instance}} {{operation_name}} {{volume_plugin}}\"}],\"title\":\"Storage Operation Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":28},\"id\":13,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(storage_operation_errors_total{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)\",\"legendFormat\":\"{{instance}} {{operation_name}} {{volume_plugin}}\"}],\"title\":\"Storage Operation Error Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":35},\"id\":14,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))\",\"legendFormat\":\"{{instance}} {{operation_name}} {{volume_plugin}}\"}],\"title\":\"Storage Operation Duration 99th quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":42},\"id\":15,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, operation_type)\",\"legendFormat\":\"{{operation_type}}\"}],\"title\":\"Cgroup manager operation rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":42},\"id\":16,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, operation_type, le))\",\"legendFormat\":\"{{instance}} {{operation_type}}\"}],\"title\":\"Cgroup manager 99th quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":49},\"id\":17,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance)\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"PLEG relist rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":49},\"id\":18,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, le))\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"PLEG relist interval\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":56},\"id\":19,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, le))\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"PLEG relist duration\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":63},\"id\":20,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\",code=~\\\"2..\\\"}[$__rate_interval]))\",\"legendFormat\":\"2xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\",code=~\\\"3..\\\"}[$__rate_interval]))\",\"legendFormat\":\"3xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\",code=~\\\"4..\\\"}[$__rate_interval]))\",\"legendFormat\":\"4xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\",code=~\\\"5..\\\"}[$__rate_interval]))\",\"legendFormat\":\"5xx\"}],\"title\":\"RPC rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":70},\"id\":21,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, verb, le))\",\"legendFormat\":\"{{instance}} {{verb}}\"}],\"title\":\"Request duration 99th quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":77},\"id\":22,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"process_resident_memory_bytes{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Memory\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":8,\"y\":77},\"id\":23,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"rate(process_cpu_seconds_total{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}[$__rate_interval])\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"CPU usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":16,\"y\":77},\"id\":24,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"go_goroutines{cluster=\\\"$cluster\\\",job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",instance=~\\\"$instance\\\"}\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Goroutines\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"label\":\"instance\",\"name\":\"instance\",\"query\":\"label_values(up{job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\",cluster=\\\"$cluster\\\"}, instance)\",\"refresh\":2,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Kubelet\",\"uid\":\"3138fa155d5915769fbded898ac09fd9\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-namespace-by-pod" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "namespace-by-pod.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"displayName\":\"$namespace\",\"max\":10000000000,\"min\":0,\"thresholds\":{\"steps\":[{\"color\":\"dark-green\",\"index\":0,\"value\":null},{\"color\":\"dark-yellow\",\"index\":1,\"value\":5000000000},{\"color\":\"dark-red\",\"index\":2,\"value\":7000000000}]},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum (\\n rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Current Rate of Bytes Received\",\"type\":\"gauge\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"displayName\":\"$namespace\",\"max\":10000000000,\"min\":0,\"thresholds\":{\"steps\":[{\"color\":\"dark-green\",\"index\":0,\"value\":null},{\"color\":\"dark-yellow\",\"index\":1,\"value\":5000000000},{\"color\":\"dark-red\",\"index\":2,\"value\":7000000000}]},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":0},\"id\":2,\"interval\":\"1m\",\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum (\\n rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Current Rate of Bytes Transmitted\",\"type\":\"gauge\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Bandwidth/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"Bps\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Packets/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"pps\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Pod\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down\",\"url\":\"/d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?${datasource:queryparam}&var-cluster=${cluster}&var-namespace=${namespace}&var-pod=${__data.fields.Pod}\"}]}]}]},\"gridPos\":{\"h\":9,\"w\":24,\"x\":0,\"y\":9},\"id\":3,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_receive_packets_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_transmit_packets_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_receive_packets_dropped_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_transmit_packets_dropped_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"format\":\"table\",\"instant\":true}],\"title\":\"Current Network Usage\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"pod\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Value #A\":7,\"Value #B\":8,\"Value #C\":9,\"Value #D\":10,\"Value #E\":11,\"Value #F\":12,\"pod\":6},\"renameByName\":{\"Value #A\":\"Current Receive Bandwidth\",\"Value #B\":\"Current Transmit Bandwidth\",\"Value #C\":\"Rate of Received Packets\",\"Value #D\":\"Rate of Transmitted Packets\",\"Value #E\":\"Rate of Received Packets Dropped\",\"Value #F\":\"Rate of Transmitted Packets Dropped\",\"pod\":\"Pod\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":18},\"id\":4,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Receive Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":18},\"id\":5,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Transmit Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":27},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_receive_packets_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":27},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_transmit_packets_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":36},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_receive_packets_dropped_total{cluster=\\\"$cluster\\\",namespace!=\\\"\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":36},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum by (pod) (\\n rate(container_network_transmit_packets_dropped_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n * on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n)\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets Dropped\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"allValue\":\".+\",\"current\":{\"selected\":false,\"text\":\"kube-system\",\"value\":\"kube-system\"},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"label\":\"namespace\",\"name\":\"namespace\",\"query\":\"label_values(container_network_receive_packets_total{cluster=\\\"$cluster\\\"}, namespace)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Networking / Namespace (Pods)\",\"uid\":\"8b7a8b326d7a6f1f04244066368c67af\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-namespace-by-workload" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "namespace-by-workload.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"fixedColor\":\"green\",\"mode\":\"fixed\"},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"displayMode\":\"basic\",\"showUnfilled\":false},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n* on (cluster,namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Current Rate of Bytes Received\",\"type\":\"bargauge\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"fixedColor\":\"green\",\"mode\":\"fixed\"},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":0},\"id\":2,\"interval\":\"1m\",\"options\":{\"displayMode\":\"basic\",\"showUnfilled\":false},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n* on (cluster,namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Current Rate of Bytes Transmitted\",\"type\":\"bargauge\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Bytes/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"binBps\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Packets/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"pps\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Workload\"},\"properties\":[{\"id\":\"links\",\"value\":[{\"title\":\"Drill down\",\"url\":\"/d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?${datasource:queryparam}&var-cluster=${cluster}&var-namespace=${namespace}&var-type=${__data.fields.Type}&var-workload=${__data.fields.Workload}\"}]}]}]},\"gridPos\":{\"h\":9,\"w\":24,\"x\":0,\"y\":9},\"id\":3,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod) kube_pod_info{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\",host_network=\\\"false\\\"}\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload, workload_type))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod) kube_pod_info{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\",host_network=\\\"false\\\"}\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload, workload_type))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(avg(rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod) kube_pod_info{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\",host_network=\\\"false\\\"}\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload, workload_type))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(avg(rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod) kube_pod_info{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\",host_network=\\\"false\\\"}\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload, workload_type))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_receive_packets_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod) kube_pod_info{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\",host_network=\\\"false\\\"}\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload, workload_type))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_transmit_packets_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod) kube_pod_info{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\",host_network=\\\"false\\\"}\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload, workload_type))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_receive_packets_dropped_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod) kube_pod_info{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\",host_network=\\\"false\\\"}\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload, workload_type))\\n\",\"format\":\"table\",\"instant\":true},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_transmit_packets_dropped_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod) kube_pod_info{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\",host_network=\\\"false\\\"}\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload, workload_type))\\n\",\"format\":\"table\",\"instant\":true}],\"title\":\"Current Status\",\"transformations\":[{\"id\":\"joinByField\",\"options\":{\"byField\":\"workload\",\"mode\":\"outer\"}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{\"Time\":true,\"Time 1\":true,\"Time 2\":true,\"Time 3\":true,\"Time 4\":true,\"Time 5\":true,\"Time 6\":true,\"Time 7\":true,\"Time 8\":true,\"workload_type 2\":true,\"workload_type 3\":true,\"workload_type 4\":true,\"workload_type 5\":true,\"workload_type 6\":true,\"workload_type 7\":true,\"workload_type 8\":true},\"indexByName\":{\"Time 1\":0,\"Time 2\":1,\"Time 3\":2,\"Time 4\":3,\"Time 5\":4,\"Time 6\":5,\"Time 7\":6,\"Time 8\":7,\"Value #A\":10,\"Value #B\":11,\"Value #C\":12,\"Value #D\":13,\"Value #E\":14,\"Value #F\":15,\"Value #G\":16,\"Value #H\":17,\"workload\":8,\"workload_type 1\":9,\"workload_type 2\":18,\"workload_type 3\":19,\"workload_type 4\":20,\"workload_type 5\":21,\"workload_type 6\":22,\"workload_type 7\":23,\"workload_type 8\":24},\"renameByName\":{\"Value #A\":\"Rx Bytes\",\"Value #B\":\"Tx Bytes\",\"Value #C\":\"Rx Bytes (Avg)\",\"Value #D\":\"Tx Bytes (Avg)\",\"Value #E\":\"Rx Packets\",\"Value #F\":\"Tx Packets\",\"Value #G\":\"Rx Packets Dropped\",\"Value #H\":\"Tx Packets Dropped\",\"workload\":\"Workload\",\"workload_type 1\":\"Type\"}}}],\"type\":\"table\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":18},\"id\":4,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n* on (cluster,namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Receive Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":18},\"id\":5,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n* on (cluster,namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Transmit Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":27},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(avg(rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n* on (cluster,namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Container Bandwidth by Workload: Received\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":27},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(avg(rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n* on (cluster,namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Container Bandwidth by Workload: Transmitted\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":36},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_receive_packets_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n* on (cluster,namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":36},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_transmit_packets_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n* on (cluster,namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":45},\"id\":10,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_receive_packets_dropped_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n* on (cluster,namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":45},\"id\":11,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_transmit_packets_dropped_total{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\"}[$__rate_interval])\\n* on (cluster,namespace,pod) group_left ()\\n topk by (cluster,namespace,pod) (\\n 1,\\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\\\"false\\\"})\\n )\\n* on (cluster,namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=\\\"$namespace\\\", workload=~\\\".+\\\", workload_type=~\\\"$type\\\"}) by (workload))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets Dropped\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"current\":{\"selected\":false,\"text\":\"kube-system\",\"value\":\"kube-system\"},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"namespace\",\"name\":\"namespace\",\"query\":\"label_values(container_network_receive_packets_total{cluster=\\\"$cluster\\\"}, namespace)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"label\":\"workload_type\",\"name\":\"type\",\"query\":\"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=\\\"$namespace\\\", workload=~\\\".+\\\"}, workload_type)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Networking / Namespace (Workload)\",\"uid\":\"bbb2a765a623ae38130206c7d94a160f\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-node-cluster-rsrc-use" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "node-cluster-rsrc-use.json": "{\"graphTooltip\":1,\"panels\":[{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"panels\":[],\"title\":\"CPU\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":1},\"id\":2,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"((\\n instance:node_cpu_utilisation:rate5m{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}\\n *\\n instance:node_num_cpu:sum{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}\\n) != 0 )\\n/ scalar(sum(instance:node_num_cpu:sum{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}))\\n\",\"legendFormat\":\"{{ instance }}\"}],\"title\":\"CPU Utilisation\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":1},\"id\":3,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"(\\n instance:node_load1_per_cpu:ratio{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}\\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}))\\n) != 0\\n\",\"legendFormat\":\"{{ instance }}\"}],\"title\":\"CPU Saturation (Load1 per CPU)\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":8},\"id\":4,\"panels\":[],\"title\":\"Memory\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":9},\"id\":5,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"(\\n instance:node_memory_utilisation:ratio{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}\\n / scalar(count(instance:node_memory_utilisation:ratio{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}))\\n) != 0\\n\",\"legendFormat\":\"{{ instance }}\"}],\"title\":\"Memory Utilisation\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"rds\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":9},\"id\":6,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_vmstat_pgmajfault:rate5m{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"{{ instance }}\"}],\"title\":\"Memory Saturation (Major Page Faults)\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":16},\"id\":7,\"panels\":[],\"title\":\"Network\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"Bps\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Transmit/\"},\"properties\":[{\"id\":\"custom.transform\",\"value\":\"negative-Y\"}]}]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":17},\"id\":8,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_network_receive_bytes_excluding_lo:rate5m{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"{{ instance }} Receive\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"{{ instance }} Transmit\"}],\"title\":\"Network Utilisation (Bytes Receive/Transmit)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"Bps\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Transmit/\"},\"properties\":[{\"id\":\"custom.transform\",\"value\":\"negative-Y\"}]}]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":17},\"id\":9,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_network_receive_drop_excluding_lo:rate5m{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"{{ instance }} Receive\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_network_transmit_drop_excluding_lo:rate5m{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"{{ instance }} Transmit\"}],\"title\":\"Network Saturation (Drops Receive/Transmit)\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":24},\"id\":10,\"panels\":[],\"title\":\"Disk IO\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":25},\"id\":11,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance_device:node_disk_io_time_seconds:rate5m{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}\\n/ scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}))\\n\",\"legendFormat\":\"{{ instance }} {{device}}\"}],\"title\":\"Disk IO Utilisation\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":25},\"id\":12,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}\\n/ scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}))\\n\",\"legendFormat\":\"{{ instance }} {{device}}\"}],\"title\":\"Disk IO Saturation\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":34},\"id\":13,\"panels\":[],\"title\":\"Disk Space\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":35},\"id\":14,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum without (device) (\\n max without (fstype, mountpoint) ((\\n node_filesystem_size_bytes{job=\\\"node-exporter\\\", fstype!=\\\"\\\", mountpoint!=\\\"\\\", cluster=\\\"$cluster\\\"}\\n -\\n node_filesystem_avail_bytes{job=\\\"node-exporter\\\", fstype!=\\\"\\\", mountpoint!=\\\"\\\", cluster=\\\"$cluster\\\"}\\n ) != 0)\\n)\\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\\\"node-exporter\\\", fstype!=\\\"\\\", mountpoint!=\\\"\\\", cluster=\\\"$cluster\\\"})))\\n\",\"legendFormat\":\"{{ instance }}\"}],\"title\":\"Disk Space Utilisation\",\"type\":\"timeseries\"}],\"refresh\":\"30s\",\"schemaVersion\":39,\"tags\":[\"node-exporter-mixin\"],\"templating\":{\"list\":[{\"name\":\"datasource\",\"query\":\"prometheus\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"includeAll\":false,\"name\":\"cluster\",\"query\":\"label_values(node_time_seconds, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Node Exporter / USE Method / Cluster\",\"uid\":\"3e97d1d02672cdd0861f4c97c64f89b2\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-node-rsrc-use" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "node-rsrc-use.json": "{\"graphTooltip\":1,\"panels\":[{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"panels\":[],\"title\":\"CPU\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":1},\"id\":2,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_cpu_utilisation:rate5m{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"Utilisation\"}],\"title\":\"CPU Utilisation\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":1},\"id\":3,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_load1_per_cpu:ratio{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"Saturation\"}],\"title\":\"CPU Saturation (Load1 per CPU)\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":8},\"id\":4,\"panels\":[],\"title\":\"Memory\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":9},\"id\":5,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_memory_utilisation:ratio{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"Utilisation\"}],\"title\":\"Memory Utilisation\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"rds\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":9},\"id\":6,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_vmstat_pgmajfault:rate5m{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"Major page Faults\"}],\"title\":\"Memory Saturation (Major Page Faults)\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":16},\"id\":7,\"panels\":[],\"title\":\"Network\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"Bps\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Transmit/\"},\"properties\":[{\"id\":\"custom.transform\",\"value\":\"negative-Y\"}]}]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":17},\"id\":8,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_network_receive_bytes_excluding_lo:rate5m{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"Receive\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"Transmit\"}],\"title\":\"Network Utilisation (Bytes Receive/Transmit)\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"Bps\"},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/Transmit/\"},\"properties\":[{\"id\":\"custom.transform\",\"value\":\"negative-Y\"}]}]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":17},\"id\":9,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_network_receive_drop_excluding_lo:rate5m{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"Receive\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance:node_network_transmit_drop_excluding_lo:rate5m{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"Transmit\"}],\"title\":\"Network Saturation (Drops Receive/Transmit)\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":24},\"id\":10,\"panels\":[],\"title\":\"Disk IO\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":25},\"id\":11,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance_device:node_disk_io_time_seconds:rate5m{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"{{device}}\"}],\"title\":\"Disk IO Utilisation\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":25},\"id\":12,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} != 0\",\"legendFormat\":\"{{device}}\"}],\"title\":\"Disk IO Saturation\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":34},\"id\":13,\"panels\":[],\"title\":\"Disk Space\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":35},\"id\":14,\"options\":{\"legend\":{\"showLegend\":false},\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sort_desc(1 -\\n (\\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\\\"node-exporter\\\", fstype!=\\\"\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"})\\n /\\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\\\"node-exporter\\\", fstype!=\\\"\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"})\\n ) != 0\\n)\\n\",\"legendFormat\":\"{{device}}\"}],\"title\":\"Disk Space Utilisation\",\"type\":\"timeseries\"}],\"refresh\":\"30s\",\"schemaVersion\":39,\"tags\":[\"node-exporter-mixin\"],\"templating\":{\"list\":[{\"name\":\"datasource\",\"query\":\"prometheus\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"includeAll\":false,\"name\":\"cluster\",\"query\":\"label_values(node_time_seconds, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"name\":\"instance\",\"query\":\"label_values(node_exporter_build_info{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\"}, instance)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Node Exporter / USE Method / Node\",\"uid\":\"fac67cfbe174d3ef53eb473d73d9212f\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-nodes" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "nodes.json": "{\"graphTooltip\":1,\"panels\":[{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"panels\":[],\"title\":\"CPU\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"max\":1,\"min\":0,\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":1},\"id\":2,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"(\\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\\\"node-exporter\\\", mode=~\\\"idle|iowait|steal\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}[$__rate_interval])))\\n/ ignoring(cpu) group_left\\n count without (cpu, mode) (node_cpu_seconds_total{job=\\\"node-exporter\\\", mode=\\\"idle\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"})\\n)\\n\",\"intervalFactor\":5,\"legendFormat\":\"{{cpu}}\"}],\"title\":\"CPU Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"showPoints\":\"never\"},\"min\":0,\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":1},\"id\":3,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_load1{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"1m load average\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_load5{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"5m load average\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_load15{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"15m load average\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"count(node_cpu_seconds_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", mode=\\\"idle\\\"})\",\"legendFormat\":\"logical cores\"}],\"title\":\"Load Average\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":8},\"id\":4,\"title\":\"Memory\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"min\":0,\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":18,\"x\":0,\"y\":9},\"id\":5,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"(\\n node_memory_MemTotal_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\\n-\\n node_memory_MemFree_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\\n-\\n node_memory_Buffers_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\\n-\\n node_memory_Cached_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\\n)\\n\",\"legendFormat\":\"memory used\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_memory_Buffers_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"memory buffers\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_memory_Cached_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"memory cached\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_memory_MemFree_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"memory free\"}],\"title\":\"Memory Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"max\":100,\"min\":0,\"thresholds\":{\"steps\":[{\"color\":\"rgba(50, 172, 45, 0.97)\"},{\"color\":\"rgba(237, 129, 40, 0.89)\",\"value\":80},{\"color\":\"rgba(245, 54, 54, 0.9)\",\"value\":90}]},\"unit\":\"percent\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":18,\"y\":9},\"id\":6,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"100 -\\n(\\n avg(node_memory_MemAvailable_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}) /\\n avg(node_memory_MemTotal_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"})\\n* 100\\n)\\n\"}],\"title\":\"Memory Usage\",\"type\":\"gauge\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":18},\"id\":7,\"panels\":[],\"title\":\"Disk\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"showPoints\":\"never\"},\"min\":0},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/ read| written/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"Bps\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/ io time/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]}]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":19},\"id\":8,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_disk_read_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\"}[$__rate_interval])\",\"intervalFactor\":1,\"legendFormat\":\"{{device}} read\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_disk_written_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\"}[$__rate_interval])\",\"intervalFactor\":1,\"legendFormat\":\"{{device}} written\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_disk_io_time_seconds_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\"}[$__rate_interval])\",\"intervalFactor\":1,\"legendFormat\":\"{{device}} io time\"}],\"title\":\"Disk I/O\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"thresholds\":{\"steps\":[{\"color\":\"green\"},{\"color\":\"yellow\",\"value\":0.8},{\"color\":\"red\",\"value\":0.9}]},\"unit\":\"decbytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byName\",\"options\":\"Mounted on\"},\"properties\":[{\"id\":\"custom.width\",\"value\":260}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Size\"},\"properties\":[{\"id\":\"custom.width\",\"value\":93}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Used\"},\"properties\":[{\"id\":\"custom.width\",\"value\":72}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Available\"},\"properties\":[{\"id\":\"custom.width\",\"value\":88}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Used, %\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"},{\"id\":\"custom.cellOptions\",\"value\":{\"type\":\"gauge\"}},{\"id\":\"max\",\"value\":1},{\"id\":\"min\",\"value\":0}]}]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":19},\"id\":9,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"max by (mountpoint) (node_filesystem_size_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", fstype!=\\\"\\\", mountpoint!=\\\"\\\"})\\n\",\"format\":\"table\",\"instant\":true,\"legendFormat\":\"\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"max by (mountpoint) (node_filesystem_avail_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", fstype!=\\\"\\\", mountpoint!=\\\"\\\"})\\n\",\"format\":\"table\",\"instant\":true,\"legendFormat\":\"\"}],\"title\":\"Disk Space Usage\",\"transformations\":[{\"id\":\"groupBy\",\"options\":{\"fields\":{\"Value #A\":{\"aggregations\":[\"lastNotNull\"],\"operation\":\"aggregate\"},\"Value #B\":{\"aggregations\":[\"lastNotNull\"],\"operation\":\"aggregate\"},\"mountpoint\":{\"aggregations\":[],\"operation\":\"groupby\"}}}},{\"id\":\"merge\"},{\"id\":\"calculateField\",\"options\":{\"alias\":\"Used\",\"binary\":{\"left\":\"Value #A (lastNotNull)\",\"operator\":\"-\",\"reducer\":\"sum\",\"right\":\"Value #B (lastNotNull)\"},\"mode\":\"binary\",\"reduce\":{\"reducer\":\"sum\"}}},{\"id\":\"calculateField\",\"options\":{\"alias\":\"Used, %\",\"binary\":{\"left\":\"Used\",\"operator\":\"/\",\"reducer\":\"sum\",\"right\":\"Value #A (lastNotNull)\"},\"mode\":\"binary\",\"reduce\":{\"reducer\":\"sum\"}}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{},\"indexByName\":{},\"renameByName\":{\"Value #A (lastNotNull)\":\"Size\",\"Value #B (lastNotNull)\":\"Available\",\"mountpoint\":\"Mounted on\"}}},{\"id\":\"sortBy\",\"options\":{\"fields\":{},\"sort\":[{\"field\":\"Mounted on\"}]}}],\"type\":\"table\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":26},\"id\":10,\"panels\":[],\"title\":\"Network\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"description\":\"Network received (bits/s)\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"showPoints\":\"never\"},\"min\":0,\"unit\":\"bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":27},\"id\":11,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_network_receive_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device!=\\\"lo\\\"}[$__rate_interval]) * 8\",\"intervalFactor\":1,\"legendFormat\":\"{{device}}\"}],\"title\":\"Network Received\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"description\":\"Network transmitted (bits/s)\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0},\"min\":0,\"unit\":\"bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":27},\"id\":12,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_network_transmit_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device!=\\\"lo\\\"}[$__rate_interval]) * 8\",\"intervalFactor\":1,\"legendFormat\":\"{{device}}\"}],\"title\":\"Network Transmitted\",\"type\":\"timeseries\"}],\"refresh\":\"30s\",\"schemaVersion\":39,\"tags\":[\"node-exporter-mixin\"],\"templating\":{\"list\":[{\"name\":\"datasource\",\"query\":\"prometheus\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"Cluster\",\"name\":\"cluster\",\"query\":\"label_values(node_uname_info{job=\\\"node-exporter\\\", sysname!=\\\"Darwin\\\"}, cluster)\",\"refresh\":2,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"label\":\"Instance\",\"name\":\"instance\",\"query\":\"label_values(node_uname_info{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\", sysname!=\\\"Darwin\\\"}, instance)\",\"refresh\":2,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Node Exporter / Nodes\",\"uid\":\"7d57716318ee0dddbac5a7f451fb7753\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-nodes-aix" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "nodes-aix.json": "{\"graphTooltip\":1,\"panels\":[{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"panels\":[],\"title\":\"CPU\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"max\":1,\"min\":0,\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":1},\"id\":2,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"(\\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\\\"node-exporter\\\", mode=~\\\"idle|iowait|steal\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}[$__rate_interval])))\\n/ ignoring(cpu) group_left\\n count without (cpu, mode) (node_cpu_seconds_total{job=\\\"node-exporter\\\", mode=\\\"idle\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"})\\n)\\n\",\"intervalFactor\":5,\"legendFormat\":\"{{cpu}}\"}],\"title\":\"CPU Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"showPoints\":\"never\"},\"min\":0,\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":1},\"id\":3,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_load1{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"1m load average\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_load5{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"5m load average\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_load15{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"15m load average\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"count(node_cpu_seconds_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", mode=\\\"idle\\\"})\",\"legendFormat\":\"logical cores\"}],\"title\":\"Load Average\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":8},\"id\":4,\"title\":\"Memory\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"none\"}},\"min\":0,\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":18,\"x\":0,\"y\":9},\"id\":5,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_memory_total_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"Physical Memory\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"(\\n node_memory_total_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} -\\n node_memory_available_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\\n)\\n\",\"legendFormat\":\"Memory Used\"}],\"title\":\"Memory Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"max\":100,\"min\":0,\"thresholds\":{\"steps\":[{\"color\":\"rgba(50, 172, 45, 0.97)\"},{\"color\":\"rgba(237, 129, 40, 0.89)\",\"value\":80},{\"color\":\"rgba(245, 54, 54, 0.9)\",\"value\":90}]},\"unit\":\"percent\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":18,\"y\":9},\"id\":6,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"100 -\\n(\\n avg(node_memory_available_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}) /\\n avg(node_memory_total_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"})\\n * 100\\n)\\n\"}],\"title\":\"Memory Usage\",\"type\":\"gauge\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":18},\"id\":7,\"panels\":[],\"title\":\"Disk\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"showPoints\":\"never\"},\"min\":0},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/ read| written/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"Bps\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/ io time/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]}]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":19},\"id\":8,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_disk_read_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\"}[$__rate_interval])\",\"intervalFactor\":1,\"legendFormat\":\"{{device}} read\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_disk_written_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\"}[$__rate_interval])\",\"intervalFactor\":1,\"legendFormat\":\"{{device}} written\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_disk_io_time_seconds_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\"}[$__rate_interval])\",\"intervalFactor\":1,\"legendFormat\":\"{{device}} io time\"}],\"title\":\"Disk I/O\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"thresholds\":{\"steps\":[{\"color\":\"green\"},{\"color\":\"yellow\",\"value\":0.8},{\"color\":\"red\",\"value\":0.9}]},\"unit\":\"decbytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byName\",\"options\":\"Mounted on\"},\"properties\":[{\"id\":\"custom.width\",\"value\":260}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Size\"},\"properties\":[{\"id\":\"custom.width\",\"value\":93}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Used\"},\"properties\":[{\"id\":\"custom.width\",\"value\":72}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Available\"},\"properties\":[{\"id\":\"custom.width\",\"value\":88}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Used, %\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"},{\"id\":\"custom.cellOptions\",\"value\":{\"type\":\"gauge\"}},{\"id\":\"max\",\"value\":1},{\"id\":\"min\",\"value\":0}]}]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":19},\"id\":9,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"max by (mountpoint) (node_filesystem_size_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", fstype!=\\\"\\\", mountpoint!=\\\"\\\"})\\n\",\"format\":\"table\",\"instant\":true,\"legendFormat\":\"\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"max by (mountpoint) (node_filesystem_avail_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", fstype!=\\\"\\\", mountpoint!=\\\"\\\"})\\n\",\"format\":\"table\",\"instant\":true,\"legendFormat\":\"\"}],\"title\":\"Disk Space Usage\",\"transformations\":[{\"id\":\"groupBy\",\"options\":{\"fields\":{\"Value #A\":{\"aggregations\":[\"lastNotNull\"],\"operation\":\"aggregate\"},\"Value #B\":{\"aggregations\":[\"lastNotNull\"],\"operation\":\"aggregate\"},\"mountpoint\":{\"aggregations\":[],\"operation\":\"groupby\"}}}},{\"id\":\"merge\"},{\"id\":\"calculateField\",\"options\":{\"alias\":\"Used\",\"binary\":{\"left\":\"Value #A (lastNotNull)\",\"operator\":\"-\",\"reducer\":\"sum\",\"right\":\"Value #B (lastNotNull)\"},\"mode\":\"binary\",\"reduce\":{\"reducer\":\"sum\"}}},{\"id\":\"calculateField\",\"options\":{\"alias\":\"Used, %\",\"binary\":{\"left\":\"Used\",\"operator\":\"/\",\"reducer\":\"sum\",\"right\":\"Value #A (lastNotNull)\"},\"mode\":\"binary\",\"reduce\":{\"reducer\":\"sum\"}}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{},\"indexByName\":{},\"renameByName\":{\"Value #A (lastNotNull)\":\"Size\",\"Value #B (lastNotNull)\":\"Available\",\"mountpoint\":\"Mounted on\"}}},{\"id\":\"sortBy\",\"options\":{\"fields\":{},\"sort\":[{\"field\":\"Mounted on\"}]}}],\"type\":\"table\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":26},\"id\":10,\"panels\":[],\"title\":\"Network\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"description\":\"Network received (bits/s)\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"showPoints\":\"never\"},\"min\":0,\"unit\":\"bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":27},\"id\":11,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_network_receive_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device!=\\\"lo\\\"}[$__rate_interval]) * 8\",\"intervalFactor\":1,\"legendFormat\":\"{{device}}\"}],\"title\":\"Network Received\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"description\":\"Network transmitted (bits/s)\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0},\"min\":0,\"unit\":\"bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":27},\"id\":12,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_network_transmit_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device!=\\\"lo\\\"}[$__rate_interval]) * 8\",\"intervalFactor\":1,\"legendFormat\":\"{{device}}\"}],\"title\":\"Network Transmitted\",\"type\":\"timeseries\"}],\"refresh\":\"30s\",\"schemaVersion\":39,\"tags\":[\"node-exporter-mixin\"],\"templating\":{\"list\":[{\"name\":\"datasource\",\"query\":\"prometheus\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"Cluster\",\"name\":\"cluster\",\"query\":\"label_values(node_uname_info{job=\\\"node-exporter\\\", sysname!=\\\"Darwin\\\"}, cluster)\",\"refresh\":2,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"label\":\"Instance\",\"name\":\"instance\",\"query\":\"label_values(node_uname_info{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\", sysname!=\\\"Darwin\\\"}, instance)\",\"refresh\":2,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Node Exporter / AIX\",\"uid\":\"7e0a61e486f727d763fb1d86fdd629c2\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-nodes-darwin" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "nodes-darwin.json": "{\"graphTooltip\":1,\"panels\":[{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"panels\":[],\"title\":\"CPU\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"max\":1,\"min\":0,\"unit\":\"percentunit\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":1},\"id\":2,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"(\\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\\\"node-exporter\\\", mode=~\\\"idle|iowait|steal\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}[$__rate_interval])))\\n/ ignoring(cpu) group_left\\n count without (cpu, mode) (node_cpu_seconds_total{job=\\\"node-exporter\\\", mode=\\\"idle\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"})\\n)\\n\",\"intervalFactor\":5,\"legendFormat\":\"{{cpu}}\"}],\"title\":\"CPU Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"showPoints\":\"never\"},\"min\":0,\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":1},\"id\":3,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_load1{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"1m load average\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_load5{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"5m load average\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_load15{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"15m load average\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"count(node_cpu_seconds_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", mode=\\\"idle\\\"})\",\"legendFormat\":\"logical cores\"}],\"title\":\"Load Average\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":8},\"id\":4,\"title\":\"Memory\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"none\"}},\"min\":0,\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":18,\"x\":0,\"y\":9},\"id\":5,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_memory_total_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"Physical Memory\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"(\\n node_memory_internal_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} -\\n node_memory_purgeable_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} +\\n node_memory_wired_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} +\\n node_memory_compressed_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\\n)\\n\",\"legendFormat\":\"Memory Used\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"(\\n node_memory_internal_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"} -\\n node_memory_purgeable_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\\n)\\n\",\"legendFormat\":\"App Memory\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_memory_wired_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"Wired Memory\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"node_memory_compressed_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}\",\"legendFormat\":\"Compressed\"}],\"title\":\"Memory Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"max\":100,\"min\":0,\"thresholds\":{\"steps\":[{\"color\":\"rgba(50, 172, 45, 0.97)\"},{\"color\":\"rgba(237, 129, 40, 0.89)\",\"value\":80},{\"color\":\"rgba(245, 54, 54, 0.9)\",\"value\":90}]},\"unit\":\"percent\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":18,\"y\":9},\"id\":6,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"(\\n (\\n avg(node_memory_internal_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}) -\\n avg(node_memory_purgeable_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}) +\\n avg(node_memory_wired_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"}) +\\n avg(node_memory_compressed_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"})\\n ) /\\n avg(node_memory_total_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\"})\\n)\\n*\\n100\\n\"}],\"title\":\"Memory Usage\",\"type\":\"gauge\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":18},\"id\":7,\"panels\":[],\"title\":\"Disk\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"showPoints\":\"never\"},\"min\":0},\"overrides\":[{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/ read| written/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"Bps\"}]},{\"matcher\":{\"id\":\"byRegexp\",\"options\":\"/ io time/\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"}]}]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":19},\"id\":8,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_disk_read_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\"}[$__rate_interval])\",\"intervalFactor\":1,\"legendFormat\":\"{{device}} read\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_disk_written_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\"}[$__rate_interval])\",\"intervalFactor\":1,\"legendFormat\":\"{{device}} written\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_disk_io_time_seconds_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device=~\\\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\\\"}[$__rate_interval])\",\"intervalFactor\":1,\"legendFormat\":\"{{device}} io time\"}],\"title\":\"Disk I/O\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"fieldConfig\":{\"defaults\":{\"thresholds\":{\"steps\":[{\"color\":\"green\"},{\"color\":\"yellow\",\"value\":0.8},{\"color\":\"red\",\"value\":0.9}]},\"unit\":\"decbytes\"},\"overrides\":[{\"matcher\":{\"id\":\"byName\",\"options\":\"Mounted on\"},\"properties\":[{\"id\":\"custom.width\",\"value\":260}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Size\"},\"properties\":[{\"id\":\"custom.width\",\"value\":93}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Used\"},\"properties\":[{\"id\":\"custom.width\",\"value\":72}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Available\"},\"properties\":[{\"id\":\"custom.width\",\"value\":88}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Used, %\"},\"properties\":[{\"id\":\"unit\",\"value\":\"percentunit\"},{\"id\":\"custom.cellOptions\",\"value\":{\"type\":\"gauge\"}},{\"id\":\"max\",\"value\":1},{\"id\":\"min\",\"value\":0}]}]},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":19},\"id\":9,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"max by (mountpoint) (node_filesystem_size_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", fstype!=\\\"\\\", mountpoint!=\\\"\\\"})\\n\",\"format\":\"table\",\"instant\":true,\"legendFormat\":\"\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"max by (mountpoint) (node_filesystem_avail_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", fstype!=\\\"\\\", mountpoint!=\\\"\\\"})\\n\",\"format\":\"table\",\"instant\":true,\"legendFormat\":\"\"}],\"title\":\"Disk Space Usage\",\"transformations\":[{\"id\":\"groupBy\",\"options\":{\"fields\":{\"Value #A\":{\"aggregations\":[\"lastNotNull\"],\"operation\":\"aggregate\"},\"Value #B\":{\"aggregations\":[\"lastNotNull\"],\"operation\":\"aggregate\"},\"mountpoint\":{\"aggregations\":[],\"operation\":\"groupby\"}}}},{\"id\":\"merge\"},{\"id\":\"calculateField\",\"options\":{\"alias\":\"Used\",\"binary\":{\"left\":\"Value #A (lastNotNull)\",\"operator\":\"-\",\"reducer\":\"sum\",\"right\":\"Value #B (lastNotNull)\"},\"mode\":\"binary\",\"reduce\":{\"reducer\":\"sum\"}}},{\"id\":\"calculateField\",\"options\":{\"alias\":\"Used, %\",\"binary\":{\"left\":\"Used\",\"operator\":\"/\",\"reducer\":\"sum\",\"right\":\"Value #A (lastNotNull)\"},\"mode\":\"binary\",\"reduce\":{\"reducer\":\"sum\"}}},{\"id\":\"organize\",\"options\":{\"excludeByName\":{},\"indexByName\":{},\"renameByName\":{\"Value #A (lastNotNull)\":\"Size\",\"Value #B (lastNotNull)\":\"Available\",\"mountpoint\":\"Mounted on\"}}},{\"id\":\"sortBy\",\"options\":{\"fields\":{},\"sort\":[{\"field\":\"Mounted on\"}]}}],\"type\":\"table\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":26},\"id\":10,\"panels\":[],\"title\":\"Network\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"description\":\"Network received (bits/s)\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0,\"showPoints\":\"never\"},\"min\":0,\"unit\":\"bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":27},\"id\":11,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_network_receive_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device!=\\\"lo\\\"}[$__rate_interval]) * 8\",\"intervalFactor\":1,\"legendFormat\":\"{{device}}\"}],\"title\":\"Network Received\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"description\":\"Network transmitted (bits/s)\",\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":0},\"min\":0,\"unit\":\"bps\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":27},\"id\":12,\"options\":{\"tooltip\":{\"mode\":\"multi\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(node_network_transmit_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", cluster=\\\"$cluster\\\", device!=\\\"lo\\\"}[$__rate_interval]) * 8\",\"intervalFactor\":1,\"legendFormat\":\"{{device}}\"}],\"title\":\"Network Transmitted\",\"type\":\"timeseries\"}],\"refresh\":\"30s\",\"schemaVersion\":39,\"tags\":[\"node-exporter-mixin\"],\"templating\":{\"list\":[{\"name\":\"datasource\",\"query\":\"prometheus\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"Cluster\",\"name\":\"cluster\",\"query\":\"label_values(node_uname_info{job=\\\"node-exporter\\\", sysname=\\\"Darwin\\\"}, cluster)\",\"refresh\":2,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"label\":\"Instance\",\"name\":\"instance\",\"query\":\"label_values(node_uname_info{job=\\\"node-exporter\\\", cluster=\\\"$cluster\\\", sysname=\\\"Darwin\\\"}, instance)\",\"refresh\":2,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Node Exporter / MacOS\",\"uid\":\"629701ea43bf69291922ea45f4a87d37\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-persistentvolumesusage" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "persistentvolumesusage.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":18,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(\\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\", persistentvolumeclaim=\\\"$volume\\\"})))\\n -\\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\", persistentvolumeclaim=\\\"$volume\\\"})))\\n)\\n\",\"legendFormat\":\"Used Space\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\", persistentvolumeclaim=\\\"$volume\\\"})))\\n\",\"legendFormat\":\"Free Space\"}],\"title\":\"Volume Space Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"thresholds\"},\"max\":100,\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":0},{\"color\":\"orange\",\"value\":80},{\"color\":\"red\",\"value\":90}]},\"unit\":\"percent\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":18,\"y\":0},\"id\":2,\"interval\":\"1m\",\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"max without(instance,node) (\\n(\\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\", persistentvolumeclaim=\\\"$volume\\\"})\\n -\\n topk(1, kubelet_volume_stats_available_bytes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\", persistentvolumeclaim=\\\"$volume\\\"})\\n)\\n/\\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\", persistentvolumeclaim=\\\"$volume\\\"})\\n* 100)\\n\",\"instant\":true}],\"title\":\"Volume Space Usage\",\"type\":\"gauge\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"none\"}},\"gridPos\":{\"h\":7,\"w\":18,\"y\":7},\"id\":3,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\", persistentvolumeclaim=\\\"$volume\\\"})))\",\"legendFormat\":\"Used inodes\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"(\\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\", persistentvolumeclaim=\\\"$volume\\\"})))\\n -\\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\", persistentvolumeclaim=\\\"$volume\\\"})))\\n)\\n\",\"legendFormat\":\"Free inodes\"}],\"title\":\"Volume inodes Usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"mode\":\"thresholds\"},\"max\":100,\"min\":0,\"thresholds\":{\"mode\":\"absolute\",\"steps\":[{\"color\":\"green\",\"value\":0},{\"color\":\"orange\",\"value\":80},{\"color\":\"red\",\"value\":90}]},\"unit\":\"percent\"}},\"gridPos\":{\"h\":7,\"w\":6,\"x\":18,\"y\":7},\"id\":4,\"interval\":\"1m\",\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"max without(instance,node) (\\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\", persistentvolumeclaim=\\\"$volume\\\"})\\n/\\ntopk(1, kubelet_volume_stats_inodes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\", persistentvolumeclaim=\\\"$volume\\\"})\\n* 100)\\n\",\"instant\":true}],\"title\":\"Volume inodes Usage\",\"type\":\"gauge\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(kubelet_volume_stats_capacity_bytes{job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"Namespace\",\"name\":\"namespace\",\"query\":\"label_values(kubelet_volume_stats_capacity_bytes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\"}, namespace)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"PersistentVolumeClaim\",\"name\":\"volume\",\"query\":\"label_values(kubelet_volume_stats_capacity_bytes{cluster=\\\"$cluster\\\", job=\\\"kubelet\\\", metrics_path=\\\"/metrics\\\", namespace=\\\"$namespace\\\"}, persistentvolumeclaim)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Persistent Volumes\",\"uid\":\"919b92a8e8041bd567af9edab12c840c\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-pod-total" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "pod-total.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"displayName\":\"$pod\",\"max\":10000000000,\"min\":0,\"thresholds\":{\"steps\":[{\"color\":\"dark-green\",\"index\":0,\"value\":null},{\"color\":\"dark-yellow\",\"index\":1,\"value\":5000000000},{\"color\":\"dark-red\",\"index\":2,\"value\":7000000000}]},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval]))\",\"legendFormat\":\"__auto\"}],\"title\":\"Current Rate of Bytes Received\",\"type\":\"gauge\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"displayName\":\"$pod\",\"max\":10000000000,\"min\":0,\"thresholds\":{\"steps\":[{\"color\":\"dark-green\",\"index\":0,\"value\":null},{\"color\":\"dark-yellow\",\"index\":1,\"value\":5000000000},{\"color\":\"dark-red\",\"index\":2,\"value\":7000000000}]},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":0},\"id\":2,\"interval\":\"1m\",\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval]))\",\"legendFormat\":\"__auto\"}],\"title\":\"Current Rate of Bytes Transmitted\",\"type\":\"gauge\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":9},\"id\":3,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_bytes_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Receive Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":9},\"id\":4,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_bytes_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Transmit Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":18},\"id\":5,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_packets_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":18},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_packets_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":27},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_receive_packets_dropped_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"showPoints\":\"never\"},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":27},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(container_network_transmit_packets_dropped_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", pod=~\\\"$pod\\\"}[$__rate_interval])) by (pod)\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets Dropped\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"allValue\":\".+\",\"current\":{\"selected\":false,\"text\":\"kube-system\",\"value\":\"kube-system\"},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"label\":\"namespace\",\"name\":\"namespace\",\"query\":\"label_values(container_network_receive_packets_total{cluster=\\\"$cluster\\\"}, namespace)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"},{\"current\":{\"selected\":false,\"text\":\"kube-system\",\"value\":\"kube-system\"},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"pod\",\"name\":\"pod\",\"query\":\"label_values(container_network_receive_packets_total{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}, pod)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Networking / Pod\",\"uid\":\"7a18067ce943a40ae25454675c19ff5c\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-prometheus" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "prometheus.json": "{\"panels\":[{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":0},\"id\":1,\"panels\":[],\"title\":\"Prometheus Stats\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"decimals\":2,\"displayName\":\"\",\"unit\":\"short\"},\"overrides\":[{\"matcher\":{\"id\":\"byName\",\"options\":\"Time\"},\"properties\":[{\"id\":\"displayName\",\"value\":\"Time\"},{\"id\":\"custom.align\",\"value\":null},{\"id\":\"custom.hidden\",\"value\":\"true\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"cluster\"},\"properties\":[{\"id\":\"custom.align\",\"value\":null},{\"id\":\"unit\",\"value\":\"short\"},{\"id\":\"decimals\",\"value\":2},{\"id\":\"displayName\",\"value\":\"Cluster\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"job\"},\"properties\":[{\"id\":\"custom.align\",\"value\":null},{\"id\":\"unit\",\"value\":\"short\"},{\"id\":\"decimals\",\"value\":2},{\"id\":\"displayName\",\"value\":\"Job\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"instance\"},\"properties\":[{\"id\":\"displayName\",\"value\":\"Instance\"},{\"id\":\"custom.align\",\"value\":null},{\"id\":\"unit\",\"value\":\"short\"},{\"id\":\"decimals\",\"value\":2}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"version\"},\"properties\":[{\"id\":\"displayName\",\"value\":\"Version\"},{\"id\":\"custom.align\",\"value\":null},{\"id\":\"unit\",\"value\":\"short\"},{\"id\":\"decimals\",\"value\":2}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Value #A\"},\"properties\":[{\"id\":\"displayName\",\"value\":\"Count\"},{\"id\":\"custom.align\",\"value\":null},{\"id\":\"unit\",\"value\":\"short\"},{\"id\":\"decimals\",\"value\":2},{\"id\":\"custom.hidden\",\"value\":\"true\"}]},{\"matcher\":{\"id\":\"byName\",\"options\":\"Value #B\"},\"properties\":[{\"id\":\"displayName\",\"value\":\"Uptime\"},{\"id\":\"custom.align\",\"value\":null},{\"id\":\"unit\",\"value\":\"s\"}]}]},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":1},\"id\":2,\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"count by (cluster, job, instance, version) (prometheus_build_info{cluster=~\\\"$cluster\\\", job=~\\\"$job\\\", instance=~\\\"$instance\\\"})\",\"format\":\"table\",\"instant\":true,\"legendFormat\":\"\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"max by (cluster, job, instance) (time() - process_start_time_seconds{cluster=~\\\"$cluster\\\", job=~\\\"$job\\\", instance=~\\\"$instance\\\"})\",\"format\":\"table\",\"instant\":true,\"legendFormat\":\"\"}],\"title\":\"Prometheus Stats\",\"type\":\"table\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":8},\"id\":3,\"panels\":[],\"title\":\"Discovery\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\"},\"min\":0,\"unit\":\"ms\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":9},\"id\":4,\"options\":{\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~\\\"$cluster\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[5m])) by (cluster, job, scrape_job, instance) * 1e3\",\"format\":\"time_series\",\"legendFormat\":\"{{cluster}}:{{job}}:{{instance}}:{{scrape_job}}\"}],\"title\":\"Target Sync\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"lineWidth\":0,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"min\":0,\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":9},\"id\":5,\"options\":{\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~\\\"$cluster\\\", job=~\\\"$job\\\",instance=~\\\"$instance\\\"})\",\"format\":\"time_series\",\"legendFormat\":\"{{cluster}}:{{job}}:{{instance}}\"}],\"title\":\"Targets\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":16},\"id\":6,\"panels\":[],\"title\":\"Retrieval\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\"},\"min\":0,\"unit\":\"ms\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":17},\"id\":7,\"options\":{\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(prometheus_target_interval_length_seconds_sum{cluster=~\\\"$cluster\\\", job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{cluster=~\\\"$cluster\\\", job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[5m]) * 1e3\",\"format\":\"time_series\",\"legendFormat\":\"{{cluster}}:{{job}}:{{instance}} {{interval}} configured\"}],\"title\":\"Average Scrape Interval Duration\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"lineWidth\":0,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"min\":0,\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":8,\"y\":17},\"id\":8,\"options\":{\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total{cluster=~\\\"$cluster\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[1m]))\",\"format\":\"time_series\",\"legendFormat\":\"exceeded body size limit: {{cluster}} {{job}} {{instance}}\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{cluster=~\\\"$cluster\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[1m]))\",\"format\":\"time_series\",\"legendFormat\":\"exceeded sample limit: {{cluster}} {{job}} {{instance}}\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{cluster=~\\\"$cluster\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[1m]))\",\"format\":\"time_series\",\"legendFormat\":\"duplicate timestamp: {{cluster}} {{job}} {{instance}}\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~\\\"$cluster\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[1m]))\",\"format\":\"time_series\",\"legendFormat\":\"out of bounds: {{cluster}} {{job}} {{instance}}\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~\\\"$cluster\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[1m]))\",\"format\":\"time_series\",\"legendFormat\":\"out of order: {{cluster}} {{job}} {{instance}}\"}],\"title\":\"Scrape failures\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"lineWidth\":0,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"min\":0,\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":16,\"y\":17},\"id\":9,\"options\":{\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(prometheus_tsdb_head_samples_appended_total{cluster=~\\\"$cluster\\\", job=~\\\"$job\\\",instance=~\\\"$instance\\\"}[5m])\",\"format\":\"time_series\",\"legendFormat\":\"{{cluster}} {{job}} {{instance}}\"}],\"title\":\"Appended Samples\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":24},\"id\":10,\"panels\":[],\"title\":\"Storage\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"lineWidth\":0,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"min\":0,\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":25},\"id\":11,\"options\":{\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"prometheus_tsdb_head_series{cluster=~\\\"$cluster\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}\",\"format\":\"time_series\",\"legendFormat\":\"{{cluster}} {{job}} {{instance}} head series\"}],\"title\":\"Head Series\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"lineWidth\":0,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"min\":0,\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":25},\"id\":12,\"options\":{\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"prometheus_tsdb_head_chunks{cluster=~\\\"$cluster\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\"}\",\"format\":\"time_series\",\"legendFormat\":\"{{cluster}} {{job}} {{instance}} head chunks\"}],\"title\":\"Head Chunks\",\"type\":\"timeseries\"},{\"collapsed\":false,\"gridPos\":{\"h\":1,\"w\":24,\"x\":0,\"y\":32},\"id\":13,\"panels\":[],\"title\":\"Query\",\"type\":\"row\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"lineWidth\":0,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"min\":0,\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":33},\"id\":14,\"options\":{\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"rate(prometheus_engine_query_duration_seconds_count{cluster=~\\\"$cluster\\\",job=~\\\"$job\\\",instance=~\\\"$instance\\\",slice=\\\"inner_eval\\\"}[5m])\",\"format\":\"time_series\",\"legendFormat\":\"{{cluster}} {{job}} {{instance}}\"}],\"title\":\"Query Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":100,\"lineWidth\":0,\"showPoints\":\"never\",\"stacking\":{\"mode\":\"normal\"}},\"min\":0,\"unit\":\"ms\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":33},\"id\":15,\"options\":{\"tooltip\":{\"mode\":\"multi\",\"sort\":\"desc\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"$datasource\"},\"expr\":\"max by (slice) (prometheus_engine_query_duration_seconds{quantile=\\\"0.9\\\",cluster=~\\\"$cluster\\\", job=~\\\"$job\\\",instance=~\\\"$instance\\\"}) * 1e3\",\"format\":\"time_series\",\"legendFormat\":\"{{slice}}\"}],\"title\":\"Stage Duration\",\"type\":\"timeseries\"}],\"schemaVersion\":39,\"tags\":[\"prometheus-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":false,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"type\":\"datasource\"},{\"allValue\":\".*\",\"current\":{\"selected\":false,\"text\":[\"$__all\"],\"value\":[\"$__all\"]},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"includeAll\":true,\"label\":\"cluster\",\"multi\":true,\"name\":\"cluster\",\"query\":\"label_values(prometheus_build_info{}, cluster)\",\"refresh\":2,\"sort\":2,\"type\":\"query\"},{\"allValue\":\".+\",\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"includeAll\":true,\"label\":\"job\",\"multi\":true,\"name\":\"job\",\"query\":\"label_values(prometheus_build_info{cluster=~\\\"$cluster\\\"}, job)\",\"refresh\":2,\"sort\":2,\"type\":\"query\"},{\"allValue\":\".+\",\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"includeAll\":true,\"label\":\"instance\",\"multi\":true,\"name\":\"instance\",\"query\":\"label_values(prometheus_build_info{cluster=~\\\"$cluster\\\", job=~\\\"$job\\\"}, instance)\",\"refresh\":2,\"sort\":2,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timepicker\":{\"refresh_intervals\":[\"60s\"]},\"timezone\": \"utc\",\"title\":\"Prometheus / Overview\",\"uid\":\"9fa0d141-d019-4ad7-8bc5-42196ee308bd\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-proxy" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "proxy.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"none\"}},\"gridPos\":{\"h\":7,\"w\":4,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(up{cluster=\\\"$cluster\\\", job=\\\"kube-proxy\\\"})\",\"instant\":true}],\"title\":\"Up\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":10,\"x\":4,\"y\":0},\"id\":2,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\\\"$cluster\\\", job=\\\"kube-proxy\\\", instance=~\\\"$instance\\\"}[$__rate_interval]))\",\"legendFormat\":\"rate\"}],\"title\":\"Rules Sync Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":10,\"x\":14,\"y\":0},\"id\":3,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-proxy\\\", instance=~\\\"$instance\\\"}[$__rate_interval]))\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Rules Sync Latency 99th Quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":0,\"y\":7},\"id\":4,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\\\"$cluster\\\", job=\\\"kube-proxy\\\", instance=~\\\"$instance\\\"}[$__rate_interval]))\",\"legendFormat\":\"rate\"}],\"title\":\"Network Programming Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":12,\"x\":12,\"y\":7},\"id\":5,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-proxy\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (instance, le))\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Network Programming Latency 99th Quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":14},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\",job=\\\"kube-proxy\\\", instance=~\\\"$instance\\\",code=~\\\"2..\\\"}[$__rate_interval]))\",\"legendFormat\":\"2xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\",job=\\\"kube-proxy\\\", instance=~\\\"$instance\\\",code=~\\\"3..\\\"}[$__rate_interval]))\",\"legendFormat\":\"3xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\",job=\\\"kube-proxy\\\", instance=~\\\"$instance\\\",code=~\\\"4..\\\"}[$__rate_interval]))\",\"legendFormat\":\"4xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\",job=\\\"kube-proxy\\\", instance=~\\\"$instance\\\",code=~\\\"5..\\\"}[$__rate_interval]))\",\"legendFormat\":\"5xx\"}],\"title\":\"Kube API Request Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":16,\"x\":8,\"y\":14},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-proxy\\\",instance=~\\\"$instance\\\",verb=\\\"POST\\\"}[$__rate_interval])) by (verb, le))\",\"legendFormat\":\"{{verb}}\"}],\"title\":\"Post Request Latency 99th Quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":21},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-proxy\\\", instance=~\\\"$instance\\\", verb=\\\"GET\\\"}[$__rate_interval])) by (verb, le))\",\"legendFormat\":\"{{verb}}\"}],\"title\":\"Get Request Latency 99th Quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":28},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"process_resident_memory_bytes{cluster=\\\"$cluster\\\", job=\\\"kube-proxy\\\",instance=~\\\"$instance\\\"}\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Memory\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":8,\"y\":28},\"id\":10,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"rate(process_cpu_seconds_total{cluster=\\\"$cluster\\\", job=\\\"kube-proxy\\\",instance=~\\\"$instance\\\"}[$__rate_interval])\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"CPU usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":16,\"y\":28},\"id\":11,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"go_goroutines{cluster=\\\"$cluster\\\", job=\\\"kube-proxy\\\",instance=~\\\"$instance\\\"}\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Goroutines\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kube-proxy\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"allValue\":\".+\",\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"label\":\"instance\",\"name\":\"instance\",\"query\":\"label_values(up{job=\\\"kube-proxy\\\", cluster=\\\"$cluster\\\", job=\\\"kube-proxy\\\"}, instance)\",\"refresh\":2,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Proxy\",\"uid\":\"632e265de029684c40b21cb76bca4f94\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-scheduler" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "scheduler.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"unit\":\"none\"}},\"gridPos\":{\"h\":7,\"w\":4,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"colorMode\":\"none\"},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(up{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\"})\",\"instant\":true}],\"title\":\"Up\",\"type\":\"stat\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":10,\"x\":4,\"y\":0},\"id\":2,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (cluster, instance)\",\"legendFormat\":\"{{cluster}} {{instance}} e2e\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(scheduler_binding_duration_seconds_count{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (cluster, instance)\",\"legendFormat\":\"{{cluster}} {{instance}} binding\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (cluster, instance)\",\"legendFormat\":\"{{cluster}} {{instance}} scheduling algorithm\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\"}[$__rate_interval])) by (cluster, instance)\",\"legendFormat\":\"{{cluster}} {{instance}} volume\"}],\"title\":\"Scheduling Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":10,\"x\":14,\"y\":0},\"id\":3,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (cluster, instance, le))\",\"legendFormat\":\"{{cluster}} {{instance}} e2e\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (cluster, instance, le))\",\"legendFormat\":\"{{cluster}} {{instance}} binding\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (cluster, instance, le))\",\"legendFormat\":\"{{cluster}} {{instance}} scheduling algorithm\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\",instance=~\\\"$instance\\\"}[$__rate_interval])) by (cluster, instance, le))\",\"legendFormat\":\"{{cluster}} {{instance}} volume\"}],\"title\":\"Scheduling latency 99th Quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":7},\"id\":4,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\",code=~\\\"2..\\\"}[$__rate_interval]))\",\"legendFormat\":\"2xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\",code=~\\\"3..\\\"}[$__rate_interval]))\",\"legendFormat\":\"3xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\",code=~\\\"4..\\\"}[$__rate_interval]))\",\"legendFormat\":\"4xx\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sum(rate(rest_client_requests_total{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\",code=~\\\"5..\\\"}[$__rate_interval]))\",\"legendFormat\":\"5xx\"}],\"title\":\"Kube API Request Rate\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"ops\"}},\"gridPos\":{\"h\":7,\"w\":16,\"x\":8,\"y\":7},\"id\":5,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\", verb=\\\"POST\\\"}[$__rate_interval])) by (verb, le))\",\"legendFormat\":\"{{verb}}\"}],\"title\":\"Post Request Latency 99th Quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"s\"}},\"gridPos\":{\"h\":7,\"w\":24,\"x\":0,\"y\":14},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\", verb=\\\"GET\\\"}[$__rate_interval])) by (verb, le))\",\"legendFormat\":\"{{verb}}\"}],\"title\":\"Get Request Latency 99th Quantile\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"bytes\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":0,\"y\":21},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"process_resident_memory_bytes{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\"}\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Memory\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":8,\"y\":21},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"rate(process_cpu_seconds_total{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\", instance=~\\\"$instance\\\"}[$__rate_interval])\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"CPU usage\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"short\"}},\"gridPos\":{\"h\":7,\"w\":8,\"x\":16,\"y\":21},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"go_goroutines{cluster=\\\"$cluster\\\", job=\\\"kube-scheduler\\\",instance=~\\\"$instance\\\"}\",\"legendFormat\":\"{{instance}}\"}],\"title\":\"Goroutines\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(up{job=\\\"kube-scheduler\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"allValue\":\".+\",\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"label\":\"instance\",\"name\":\"instance\",\"query\":\"label_values(up{job=\\\"kube-scheduler\\\", cluster=\\\"$cluster\\\"}, instance)\",\"refresh\":2,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Scheduler\",\"uid\":\"2e6b6a3b4bddf1427b3a55aa1311c656\"}" +--- +"metadata": + "name": "kube-prometheus-stack-1761-workload-total" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-grafana" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "grafana_dashboard": "1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"data": + "workload-total.json": "{\"editable\":true,\"links\":[{\"asDropdown\":true,\"includeVars\":true,\"keepTime\":true,\"tags\":[\"kubernetes-mixin\"],\"targetBlank\":false,\"title\":\"Kubernetes\",\"type\":\"dashboards\"}],\"panels\":[{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"fixedColor\":\"green\",\"mode\":\"fixed\"},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":0},\"id\":1,\"interval\":\"1m\",\"options\":{\"displayMode\":\"basic\",\"showUnfilled\":false},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Current Rate of Bytes Received\",\"type\":\"bargauge\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"fixedColor\":\"green\",\"mode\":\"fixed\"},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":0},\"id\":2,\"interval\":\"1m\",\"options\":{\"displayMode\":\"basic\",\"showUnfilled\":false},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Current Rate of Bytes Transmitted\",\"type\":\"bargauge\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"fixedColor\":\"green\",\"mode\":\"fixed\"},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":9},\"id\":3,\"interval\":\"1m\",\"options\":{\"displayMode\":\"basic\",\"showUnfilled\":false},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(avg(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Rate of Bytes Received\",\"type\":\"bargauge\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"color\":{\"fixedColor\":\"green\",\"mode\":\"fixed\"},\"unit\":\"Bps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":9},\"id\":4,\"interval\":\"1m\",\"options\":{\"displayMode\":\"basic\",\"showUnfilled\":false},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(avg(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Average Rate of Bytes Transmitted\",\"type\":\"bargauge\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":18},\"id\":5,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_receive_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Receive Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"binBps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":18},\"id\":6,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_transmit_bytes_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Transmit Bandwidth\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":27},\"id\":7,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_receive_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":27},\"id\":8,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_transmit_packets_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":0,\"y\":36},\"id\":9,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_receive_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Received Packets Dropped\",\"type\":\"timeseries\"},{\"datasource\":{\"type\":\"datasource\",\"uid\":\"-- Mixed --\"},\"fieldConfig\":{\"defaults\":{\"custom\":{\"fillOpacity\":10,\"showPoints\":\"never\",\"spanNulls\":true},\"unit\":\"pps\"}},\"gridPos\":{\"h\":9,\"w\":12,\"x\":12,\"y\":36},\"id\":10,\"interval\":\"1m\",\"options\":{\"legend\":{\"asTable\":true,\"calcs\":[\"lastNotNull\"],\"displayMode\":\"table\",\"placement\":\"right\",\"showLegend\":true},\"tooltip\":{\"mode\":\"single\"}},\"pluginVersion\":\"v11.4.0\",\"targets\":[{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"expr\":\"sort_desc(sum(rate(container_network_transmit_packets_dropped_total{job=\\\"kubelet\\\", metrics_path=\\\"/metrics/cadvisor\\\", cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\"}[$__rate_interval])\\n* on (namespace,pod)\\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\",namespace=~\\\"$namespace\\\", workload=~\\\"$workload\\\", workload_type=~\\\"$type\\\"}) by (pod))\\n\",\"legendFormat\":\"__auto\"}],\"title\":\"Rate of Transmitted Packets Dropped\",\"type\":\"timeseries\"}],\"refresh\":\"10s\",\"schemaVersion\":39,\"tags\":[\"kubernetes-mixin\"],\"templating\":{\"list\":[{\"current\":{\"selected\":true,\"text\":\"default\",\"value\":\"default\"},\"hide\":0,\"label\":\"Data source\",\"name\":\"datasource\",\"query\":\"prometheus\",\"regex\":\"\",\"type\":\"datasource\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":2,\"label\":\"cluster\",\"name\":\"cluster\",\"query\":\"label_values(kube_pod_info{job=\\\"kube-state-metrics\\\"}, cluster)\",\"refresh\":2,\"sort\":1,\"type\":\"query\",\"allValue\":\".*\"},{\"allValue\":\".+\",\"current\":{\"selected\":false,\"text\":\"kube-system\",\"value\":\"kube-system\"},\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"label\":\"namespace\",\"name\":\"namespace\",\"query\":\"label_values(container_network_receive_packets_total{cluster=\\\"$cluster\\\"}, namespace)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"},{\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"label\":\"workload\",\"name\":\"workload\",\"query\":\"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=~\\\"$namespace\\\", workload=~\\\".+\\\"}, workload)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"},{\"allValue\":\".+\",\"datasource\":{\"type\":\"prometheus\",\"uid\":\"${datasource}\"},\"hide\":0,\"includeAll\":true,\"label\":\"workload_type\",\"name\":\"type\",\"query\":\"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\\\"$cluster\\\", namespace=~\\\"$namespace\\\", workload=~\\\"$workload\\\"}, workload_type)\",\"refresh\":2,\"sort\":1,\"type\":\"query\"}]},\"time\":{\"from\":\"now-1h\",\"to\":\"now\"},\"timezone\": \"utc\",\"title\":\"Kubernetes / Networking / Workload\",\"uid\":\"728bf77cc1166d2f3133bf25846876cc\"}" +--- +"metadata": + "name": "loki-log-dashboard" + "namespace": "kube-mon" + "labels": + "grafana_dashboard": "1" +"data": + "loki-log-overview.json": | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "gridPos": { "h": 12, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "expr": "{namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Live Logs", + "type": "logs" + }, + { + "datasource": "$datasource", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 12 }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "expr": "{namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\" |~ \"(?i)(error|exception|fatal|panic|failed)\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Error Logs", + "type": "logs" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 22 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\"[5m]))", + "refId": "A" + } + ], + "title": "Logs/s", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 5 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 22 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\" |~ \"(?i)(error|exception|fatal|panic|failed)\"[5m]))", + "refId": "A" + } + ], + "title": "Error Logs/s", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 5 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 22 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\" |~ \"(?i)(warn|warning)\"[5m]))", + "refId": "A" + } + ], + "title": "Warn Logs/s", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 22 }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(count by (pod) (rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"}[5m]) > 0))", + "refId": "A" + } + ], + "title": "Active Pods (Logs)", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, + "id": 7, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "sum by (namespace) (rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\"[5m]))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Log Rate by Namespace", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, + "id": 8, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(10, sum by (namespace,pod) (rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\"[5m])))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "title": "Top Pods by Log Rate", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 }, + "id": 9, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "sum by (namespace) (rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\" |~ \"(?i)(error|exception|fatal|panic|failed)\"[5m]))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Error Rate by Namespace", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 }, + "id": 10, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(10, sum by (namespace,pod) (rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\" |~ \"(?i)(error|exception|fatal|panic|failed)\"[5m])))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "title": "Top Pods by Error Rate", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 42 }, + "id": 11, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "sum(rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\" |~ \"(?i)debug\"[5m]))", + "legendFormat": "debug", + "refId": "A" + }, + { + "expr": "sum(rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\" |~ \"(?i)info\"[5m]))", + "legendFormat": "info", + "refId": "B" + }, + { + "expr": "sum(rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\" |~ \"(?i)warn|warning\"[5m]))", + "legendFormat": "warn", + "refId": "C" + }, + { + "expr": "sum(rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\" |~ \"(?i)error|exception|fatal|panic|failed\"[5m]))", + "legendFormat": "error", + "refId": "D" + } + ], + "title": "Log Level Trend", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 50 }, + "id": 12, + "options": { + "legend": { "displayMode": "table", "placement": "right" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "expr": "topk(12, sum by (namespace,pod,container) (rate({namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\"} |= \"$search\"[5m])))", + "legendFormat": "{{namespace}}/{{pod}}/{{container}}", + "refId": "A" + } + ], + "title": "Top Containers by Log Volume", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 40, + "tags": ["logs", "loki", "kubernetes"], + "templating": { + "list": [ + { + "current": { "selected": false, "text": "", "value": "" }, + "name": "datasource", + "query": "loki", + "refresh": 1, + "type": "datasource" + }, + { + "allValue": ".+", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values({namespace!=\"\"}, namespace)", + "includeAll": true, + "multi": true, + "name": "namespace", + "options": [], + "query": "label_values({namespace!=\"\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values({namespace=~\"$namespace\"}, pod)", + "includeAll": true, + "multi": true, + "name": "pod", + "options": [], + "query": "label_values({namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values({namespace=~\"$namespace\", pod=~\"$pod\"}, container)", + "includeAll": true, + "multi": true, + "name": "container", + "options": [], + "query": "label_values({namespace=~\"$namespace\", pod=~\"$pod\"}, container)", + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { "selected": false, "text": "", "value": "" }, + "label": "Search", + "name": "search", + "options": [], + "query": "", + "type": "textbox" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": { + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h"] + }, + "timezone": "browser", + "title": "Loki Log Overview", + "uid": "loki-log-overview", + "version": 1, + "weekStart": "" + } + +--- +"metadata": + "name": "microk8s-cluster-dashboard" + "namespace": "kube-mon" + "labels": + "grafana_dashboard": "1" +"data": + "microk8s-cluster-overview.json": | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})", "instant": true, "refId": "A" } + ], + "title": "Ready Nodes", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 3, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"false\"})", "instant": true, "refId": "A" } + ], + "title": "NotReady Nodes", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 4, "w": 3, "x": 6, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "expr": "sum(kube_pod_status_phase{phase=\"Running\",namespace=~\"$namespace\"})", "instant": true, "refId": "A" } + ], + "title": "Running Pods", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 10 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 9, "y": 0 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "expr": "sum(kube_pod_status_phase{phase=\"Pending\",namespace=~\"$namespace\"})", "instant": true, "refId": "A" } + ], + "title": "Pending Pods", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 12, "y": 0 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "expr": "sum(kube_pod_status_phase{phase=~\"Failed|Unknown\",namespace=~\"$namespace\"})", "instant": true, "refId": "A" } + ], + "title": "Failed/Unknown Pods", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 5 }, + { "color": "red", "value": 20 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 15, "y": 0 }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[1h]))", "instant": true, "refId": "A" } + ], + "title": "Restarts (1h)", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 18, "y": 0 }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "expr": "sum(kube_pod_container_status_waiting_reason{namespace=~\"$namespace\",reason=\"CrashLoopBackOff\"})", "instant": true, "refId": "A" } + ], + "title": "CrashLoopBackOff", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "expr": "sum(kube_pod_container_status_waiting_reason{namespace=~\"$namespace\",reason=~\"ImagePullBackOff|ErrImagePull\"})", "instant": true, "refId": "A" } + ], + "title": "Image Pull Errors", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "id": 9, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "expr": "100 * sum(rate(container_cpu_usage_seconds_total{container!=\"\",image!=\"\",namespace=~\"$namespace\"}[5m])) / sum(count by (instance) (node_cpu_seconds_total{mode=\"idle\"}))", + "legendFormat": "Cluster CPU", + "refId": "A" + } + ], + "title": "Cluster CPU Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "id": 10, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "expr": "100 * (1 - (sum(node_memory_MemAvailable_bytes and on(instance) node_uname_info{nodename=~\"$node\"}) / sum(node_memory_MemTotal_bytes and on(instance) node_uname_info{nodename=~\"$node\"})))", + "legendFormat": "Cluster Memory", + "refId": "A" + } + ], + "title": "Cluster Memory Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "id": 11, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "expr": "100 * (1 - (sum(node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay\",mountpoint=\"/\"} and on(instance) node_uname_info{nodename=~\"$node\"}) / sum(node_filesystem_size_bytes{fstype!~\"tmpfs|overlay\",mountpoint=\"/\"} and on(instance) node_uname_info{nodename=~\"$node\"})))", + "legendFormat": "Cluster RootFS", + "refId": "A" + } + ], + "title": "Cluster RootFS Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "id": 12, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "sum(rate(container_network_receive_bytes_total{interface!=\"lo\",namespace=~\"$namespace\"}[5m]))", + "legendFormat": "RX", + "refId": "A" + }, + { + "expr": "sum(rate(container_network_transmit_bytes_total{interface!=\"lo\",namespace=~\"$namespace\"}[5m]))", + "legendFormat": "TX", + "refId": "B" + } + ], + "title": "Cluster Network Throughput", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 20 }, + "id": 13, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "100 - (avg by (nodename) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]) * on(instance) group_left(nodename) node_uname_info{nodename=~\"$node\"}) * 100)", + "legendFormat": "{{nodename}}", + "refId": "A" + } + ], + "title": "Node CPU Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 20 }, + "id": 14, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "100 * (1 - avg by (nodename) ((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info{nodename=~\"$node\"}))", + "legendFormat": "{{nodename}}", + "refId": "A" + } + ], + "title": "Node Memory Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 20 }, + "id": 15, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "100 * (1 - avg by (nodename) ((node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay\",mountpoint=\"/\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay\",mountpoint=\"/\"}) * on(instance) group_left(nodename) node_uname_info{nodename=~\"$node\"}))", + "legendFormat": "{{nodename}}", + "refId": "A" + } + ], + "title": "Node RootFS Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "cores" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, + "id": 16, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(10, sum by (namespace) (rate(container_cpu_usage_seconds_total{container!=\"\",image!=\"\",namespace=~\"$namespace\"}[5m])))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Top Namespaces by CPU (cores)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, + "id": 17, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(10, sum by (namespace) (container_memory_working_set_bytes{container!=\"\",image!=\"\",namespace=~\"$namespace\"}))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Top Namespaces by Memory", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "cores" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, + "id": 18, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(15, sum by (namespace,pod) (rate(container_cpu_usage_seconds_total{container!=\"\",image!=\"\",namespace=~\"$namespace\"}[5m])))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "title": "Top Pods by CPU (cores)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, + "id": 19, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(15, sum by (namespace,pod) (container_memory_working_set_bytes{container!=\"\",image!=\"\",namespace=~\"$namespace\"}))", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "title": "Top Pods by Memory", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 44 }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(clamp_min(kube_deployment_spec_replicas{namespace=~\"$namespace\"} - kube_deployment_status_replicas_available{namespace=~\"$namespace\"}, 0))", + "instant": true, + "refId": "A" + } + ], + "title": "Deployment Replica Gaps", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 44 }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(clamp_min(kube_statefulset_replicas{namespace=~\"$namespace\"} - kube_statefulset_status_replicas_ready{namespace=~\"$namespace\"}, 0))", + "instant": true, + "refId": "A" + } + ], + "title": "StatefulSet Replica Gaps", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 44 }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(kube_daemonset_status_number_unavailable{namespace=~\"$namespace\"})", + "instant": true, + "refId": "A" + } + ], + "title": "DaemonSet Unavailable", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 44 }, + "id": 23, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(kube_job_status_failed{namespace=~\"$namespace\"})", + "instant": true, + "refId": "A" + } + ], + "title": "Failed Jobs", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 48 }, + "id": 24, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "sum by (phase) (kube_pod_status_phase{namespace=~\"$namespace\"})", + "legendFormat": "{{phase}}", + "refId": "A" + } + ], + "title": "Pods by Phase", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 }, + "id": 25, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "max by (namespace,pod,phase) (kube_pod_status_phase{namespace=~\"$namespace\",phase=~\"Pending|Failed|Unknown\"} == 1)", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Problem Pods (Current)", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 }, + "id": 26, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "max by (namespace,pod,container,reason) (kube_pod_container_status_waiting_reason{namespace=~\"$namespace\",reason!=\"\"} == 1)", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Waiting Containers (Current)", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 64 }, + "id": 27, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "topk(20, sum by (namespace,pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[6h])))", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Top Restarting Pods (6h)", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 64 }, + "id": 28, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "topk(20, clamp_min(kube_deployment_spec_replicas{namespace=~\"$namespace\"} - kube_deployment_status_replicas_available{namespace=~\"$namespace\"}, 0))", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Deployments with Replica Gaps", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 72 }, + "id": 29, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "topk(20, clamp_min(kube_statefulset_replicas{namespace=~\"$namespace\"} - kube_statefulset_status_replicas_ready{namespace=~\"$namespace\"}, 0))", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "StatefulSets with Replica Gaps", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 72 }, + "id": 30, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "topk(20, kube_daemonset_status_number_unavailable{namespace=~\"$namespace\"})", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "DaemonSets Unavailable", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 80 }, + "id": 31, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "100 * sum(kube_pod_container_resource_requests{resource=\"cpu\",namespace=~\"$namespace\"}) / sum(count by (instance) (node_cpu_seconds_total{mode=\"idle\"}))", + "legendFormat": "CPU Requests / Capacity", + "refId": "A" + }, + { + "expr": "100 * sum(kube_pod_container_resource_limits{resource=\"cpu\",namespace=~\"$namespace\"}) / sum(count by (instance) (node_cpu_seconds_total{mode=\"idle\"}))", + "legendFormat": "CPU Limits / Capacity", + "refId": "B" + } + ], + "title": "CPU Requests/Limits vs Cluster Capacity (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 80 }, + "id": 32, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "100 * sum(kube_pod_container_resource_requests{resource=\"memory\",namespace=~\"$namespace\"}) / sum(node_memory_MemTotal_bytes)", + "legendFormat": "Memory Requests / Capacity", + "refId": "A" + }, + { + "expr": "100 * sum(kube_pod_container_resource_limits{resource=\"memory\",namespace=~\"$namespace\"}) / sum(node_memory_MemTotal_bytes)", + "legendFormat": "Memory Limits / Capacity", + "refId": "B" + } + ], + "title": "Memory Requests/Limits vs Cluster Capacity (%)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 40, + "tags": ["microk8s", "kubernetes", "prometheus", "operations"], + "templating": { + "list": [ + { + "current": { "selected": true, "text": "Prometheus", "value": "Prometheus" }, + "name": "datasource", + "query": "prometheus", + "refresh": 1, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values(kube_pod_info, namespace)", + "includeAll": true, + "multi": true, + "name": "namespace", + "options": [], + "query": { "query": "label_values(kube_pod_info, namespace)", "refId": "Prometheus-namespace-Variable-Query" }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values(node_uname_info, nodename)", + "includeAll": true, + "multi": true, + "name": "node", + "options": [], + "query": { "query": "label_values(node_uname_info, nodename)", "refId": "Prometheus-node-Variable-Query" }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": { + "refresh_intervals": ["10s", "30s", "1m", "5m", "15m", "30m", "1h", "6h"] + }, + "timezone": "browser", + "title": "MicroK8s Cluster Overview", + "uid": "microk8s-cluster-overview", + "version": 2, + "weekStart": "" + } + +--- +"metadata": + "name": "node-vmware-memory-dashboard" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "grafana_dashboard": "1" +"data": + "node-vs-vmware-memory.json": | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Vergleicht Linux-Node-Speicher aus node_exporter mit VMware-VM-Speicher aus vmware-exporter und Proxmox-Gast-Speicher aus pve-exporter, um Abweichungen zwischen Gast-OS und Hypervisor sichtbar zu machen.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "content": "Dieses Dashboard zeigt bewusst verschiedene Speicherdefinitionen nebeneinander.\\n\\n- Linux `Used (MemAvailable)` ist der bessere Indikator fuer echten Speicherdruck auf dem Node.\\n- Linux `Used (Free only)` zaehlt Dateicache als belegt und wirkt deshalb meist deutlich hoeher.\\n- ESXi `Active` zeigt aktiv genutzten Speicher aus Hypervisor-Sicht.\\n- ESXi `Consumed` liegt oft nahe am konfigurierten RAM der VM und ist nicht mit Linux `Used` gleichzusetzen.\\n- Proxmox `pve_memory_usage_bytes` und `pve_memory_size_bytes` kommen aus PVE-Sicht und sind ebenfalls Hypervisor-Werte, nicht Linux `MemAvailable` im Gast.", + "mode": "markdown" + }, + "pluginVersion": "12.2.1", + "title": "Lesart", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Empfohlene Linux-Sicht: belegt = Total - MemAvailable.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} / node_memory_MemTotal_bytes{job=\"node-exporter\",instance=~\"$node_instance\"}))", + "legendFormat": "Linux Used (MemAvailable)", + "refId": "A" + } + ], + "title": "Linux Used (MemAvailable)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Klassische, oft irrefuehrende Linux-Sicht: belegt = Total - MemFree.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (1 - (node_memory_MemFree_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} / node_memory_MemTotal_bytes{job=\"node-exporter\",instance=~\"$node_instance\"}))", + "legendFormat": "Linux Used (Free only)", + "refId": "A" + } + ], + "title": "Linux Used (Free only)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Anteil des auf dem Node leicht reclaimbaren Speichers durch Cache, Buffer und reclaimable Slab.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 10 + }, + { + "color": "yellow", + "value": 25 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 4 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * ((node_memory_Cached_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} + node_memory_Buffers_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} + node_memory_SReclaimable_bytes{job=\"node-exporter\",instance=~\"$node_instance\"}) / node_memory_MemTotal_bytes{job=\"node-exporter\",instance=~\"$node_instance\"})", + "legendFormat": "Linux reclaimable cache", + "refId": "A" + } + ], + "title": "Linux Cache/Buffer/Reclaimable", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "ESXi Active Memory relativ zum konfigurierten VM-RAM.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 4 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (vmware_vm_mem_active_average{vm_name=~\"$node\",job=~\"$vmware_job\"} / (vmware_vm_memory_max{vm_name=~\"$node\",job=~\"$vmware_job\"} * 1024))", + "legendFormat": "ESXi Active ({{host_name}})", + "refId": "A" + } + ], + "title": "ESXi Active", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "ESXi Consumed Memory relativ zum konfigurierten VM-RAM. Dieser Wert ist oft deutlich naeher am zugewiesenen RAM als an echtem Linux-Speicherdruck.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 4 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (vmware_vm_mem_consumed_average{vm_name=~\"$node\",job=~\"$vmware_job\"} / (vmware_vm_memory_max{vm_name=~\"$node\",job=~\"$vmware_job\"} * 1024))", + "legendFormat": "ESXi Consumed ({{host_name}})", + "refId": "A" + } + ], + "title": "ESXi Consumed", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Aktuell belegter RAM der Proxmox-VM aus Hypervisor-Sicht.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GiB" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "pve_memory_usage_bytes{id=~\"$pve_guest_id\",job=~\"$pve_job\"} / 1073741824", + "legendFormat": "Proxmox used", + "refId": "A" + } + ], + "title": "Proxmox Guest Used", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Freier RAM der Proxmox-VM aus Hypervisor-Sicht, berechnet als Groesse minus Usage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + }, + "unit": "suffix: GiB" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 18, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "(pve_memory_size_bytes{id=~\"$pve_guest_id\",job=~\"$pve_job\"} - pve_memory_usage_bytes{id=~\"$pve_guest_id\",job=~\"$pve_job\"}) / 1073741824", + "legendFormat": "Proxmox free", + "refId": "A" + } + ], + "title": "Proxmox Guest Free", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Konfigurierter RAM der Proxmox-VM.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GiB" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "pve_memory_size_bytes{id=~\"$pve_guest_id\",job=~\"$pve_job\"} / 1073741824", + "legendFormat": "Proxmox total", + "refId": "A" + } + ], + "title": "Proxmox Guest Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Speicherauslastung der Proxmox-VM in Prozent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (pve_memory_usage_bytes{id=~\"$pve_guest_id\",job=~\"$pve_job\"} / pve_memory_size_bytes{id=~\"$pve_guest_id\",job=~\"$pve_job\"})", + "legendFormat": "Proxmox used %", + "refId": "A" + } + ], + "title": "Proxmox Guest Used %", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "In VMware konfiguriertes RAM der VM.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 4 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "vmware_vm_memory_max{vm_name=~\"$node\",job=~\"$vmware_job\"} * 1024 * 1024", + "legendFormat": "Configured RAM ({{host_name}})", + "refId": "A" + } + ], + "title": "Configured VM RAM", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Nutzbarer freier RAM in der VM aus Sicht des Gast-OS. Das ist die sinnvollste Antwort auf 'wie viel RAM ist in der VM noch frei?'.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + }, + "unit": "suffix: GiB" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 8 + }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_MemAvailable_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} / 1073741824", + "legendFormat": "VM free usable", + "refId": "A" + } + ], + "title": "VM Free Usable", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Roh frei gemeldeter RAM in der VM ohne Cache-Reclaim. Meist kleiner als MemAvailable.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GiB" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 8 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} / 1073741824", + "legendFormat": "VM free raw", + "refId": "A" + } + ], + "title": "VM Free Raw", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Aktuell belegter RAM in der VM, berechnet als Total - MemAvailable.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GiB" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 8 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "(node_memory_MemTotal_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} - node_memory_MemAvailable_bytes{job=\"node-exporter\",instance=~\"$node_instance\"}) / 1073741824", + "legendFormat": "VM used", + "refId": "A" + } + ], + "title": "VM Used", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Gesamter RAM der VM aus Sicht des Gast-OS.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GiB" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 8 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} / 1073741824", + "legendFormat": "VM total", + "refId": "A" + } + ], + "title": "VM Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Aktiver ESXi-Speicher der VM in GiB.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GiB" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 8 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "vmware_vm_mem_active_average{vm_name=~\"$node\",job=~\"$vmware_job\"} / 1048576", + "legendFormat": "ESXi active", + "refId": "A" + } + ], + "title": "ESXi Active", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Consumed Memory der VM in ESXi in GiB.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GiB" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 8 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "vmware_vm_mem_consumed_average{vm_name=~\"$node\",job=~\"$vmware_job\"} / 1048576", + "legendFormat": "ESXi consumed", + "refId": "A" + } + ], + "title": "ESXi Consumed", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Direkter Prozentvergleich der unterschiedlichen Sichten auf denselben Knoten bzw. dieselbe VM.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 12, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (1 - (node_memory_MemAvailable_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} / node_memory_MemTotal_bytes{job=\"node-exporter\",instance=~\"$node_instance\"}))", + "legendFormat": "Linux Used (MemAvailable)", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "100 * (1 - (node_memory_MemFree_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} / node_memory_MemTotal_bytes{job=\"node-exporter\",instance=~\"$node_instance\"}))", + "legendFormat": "Linux Used (Free only)", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "100 * ((node_memory_Cached_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} + node_memory_Buffers_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} + node_memory_SReclaimable_bytes{job=\"node-exporter\",instance=~\"$node_instance\"}) / node_memory_MemTotal_bytes{job=\"node-exporter\",instance=~\"$node_instance\"})", + "legendFormat": "Linux reclaimable cache", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "100 * (vmware_vm_mem_active_average{vm_name=~\"$node\",job=~\"$vmware_job\"} / (vmware_vm_memory_max{vm_name=~\"$node\",job=~\"$vmware_job\"} * 1024))", + "legendFormat": "ESXi Active ({{host_name}})", + "refId": "D" + }, + { + "editorMode": "code", + "expr": "100 * (vmware_vm_mem_consumed_average{vm_name=~\"$node\",job=~\"$vmware_job\"} / (vmware_vm_memory_max{vm_name=~\"$node\",job=~\"$vmware_job\"} * 1024))", + "legendFormat": "ESXi Consumed ({{host_name}})", + "refId": "E" + } + ], + "title": "Why The Values Differ (%)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Linux-Speicherzusammensetzung aus Sicht des Gast-OS.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} - node_memory_MemAvailable_bytes{job=\"node-exporter\",instance=~\"$node_instance\"}", + "legendFormat": "Used (MemAvailable basis)", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "node_memory_Cached_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} + node_memory_Buffers_bytes{job=\"node-exporter\",instance=~\"$node_instance\"} + node_memory_SReclaimable_bytes{job=\"node-exporter\",instance=~\"$node_instance\"}", + "legendFormat": "Reclaimable cache/buffer/slab", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "node_memory_MemAvailable_bytes{job=\"node-exporter\",instance=~\"$node_instance\"}", + "legendFormat": "MemAvailable", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\",instance=~\"$node_instance\"}", + "legendFormat": "MemFree", + "refId": "D" + } + ], + "title": "Linux Memory Breakdown (bytes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "VMware-Speicherwerte aus Hypervisor-Sicht fuer dieselbe VM.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "vmware_vm_mem_active_average{vm_name=~\"$node\",job=~\"$vmware_job\"} * 1024", + "legendFormat": "Active ({{host_name}})", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "vmware_vm_mem_consumed_average{vm_name=~\"$node\",job=~\"$vmware_job\"} * 1024", + "legendFormat": "Consumed ({{host_name}})", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "vmware_vm_memory_max{vm_name=~\"$node\",job=~\"$vmware_job\"} * 1024 * 1024", + "legendFormat": "Configured RAM ({{host_name}})", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "vmware_vm_mem_vmmemctl_average{vm_name=~\"$node\",job=~\"$vmware_job\"} * 1024", + "legendFormat": "Balloon ({{host_name}})", + "refId": "D" + }, + { + "editorMode": "code", + "expr": "vmware_vm_mem_swapped_average{vm_name=~\"$node\",job=~\"$vmware_job\"} * 1024", + "legendFormat": "Swapped ({{host_name}})", + "refId": "E" + } + ], + "title": "VMware Memory Breakdown (bytes)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 40, + "tags": [ + "kubernetes", + "memory", + "prometheus", + "vmware" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "prometheus" + }, + "label": "Datasource", + "name": "datasource", + "query": "prometheus", + "refresh": 1, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "worker05", + "value": "worker05" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_uname_info, nodename)", + "label": "Node", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(node_uname_info, nodename)", + "refId": "Prometheus-node" + }, + "refresh": 1, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "192.168.1.22:9100", + "value": "192.168.1.22:9100" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_uname_info{nodename=~\"$node\"}, instance)", + "hide": 2, + "label": "Node Instance", + "multi": false, + "name": "node_instance", + "options": [], + "query": { + "query": "label_values(node_uname_info{nodename=~\"$node\"}, instance)", + "refId": "Prometheus-node-instance" + }, + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "vmware-exporter", + "value": "vmware-exporter" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(vmware_vm_memory_max{vm_name=~\"$node\"}, job)", + "label": "VMware Job", + "multi": false, + "name": "vmware_job", + "options": [], + "query": { + "query": "label_values(vmware_vm_memory_max{vm_name=~\"$node\"}, job)", + "refId": "Prometheus-vmware-job" + }, + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(pve_guest_info{name=~\"$node\",type=\"qemu\"}, job)", + "label": "Proxmox Job", + "multi": false, + "name": "pve_job", + "options": [], + "query": { + "query": "label_values(pve_guest_info{name=~\"$node\",type=\"qemu\"}, job)", + "refId": "Prometheus-pve-job" + }, + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(pve_guest_info{name=~\"$node\",job=~\"$pve_job\",type=\"qemu\"}, id)", + "hide": 2, + "label": "Proxmox Guest ID", + "multi": false, + "name": "pve_guest_id", + "options": [], + "query": { + "query": "label_values(pve_guest_info{name=~\"$node\",job=~\"$pve_job\",type=\"qemu\"}, id)", + "refId": "Prometheus-pve-guest-id" + }, + "refresh": 2, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "6h", + "12h", + "24h" + ] + }, + "timezone": "browser", + "title": "Kubernetes Node vs VMware VM Memory", + "uid": "k8s-node-vs-vmware-memory", + "version": 5, + "weekStart": "" + } + +--- +"metadata": + "name": "proxmox-pve-dashboard" + "namespace": "kube-mon" + "labels": + "grafana_dashboard": "1" +"data": + "proxmox-pve-overview.json": | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 4, "w": 3, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(max by (job,id) (pve_up{job=~\"$job\",id=~\"node/.*\"}))", + "instant": true, + "refId": "A" + } + ], + "title": "Nodes Up", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 4, "w": 3, "x": 3, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(max by (job,id) (pve_up{job=~\"$job\",id=~\"(qemu|lxc)/.*\"}))", + "instant": true, + "refId": "A" + } + ], + "title": "Guests Total", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 6, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(max by (job,id) (pve_up{job=~\"$job\",id=~\"(qemu|lxc)/.*\"}))", + "instant": true, + "refId": "A" + } + ], + "title": "Guests Up", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 4, "w": 3, "x": 9, "y": 0 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(max by (job,id) (pve_storage_info{job=~\"$job\",node=~\"$node\",storage=~\"$storage\"}))", + "instant": true, + "refId": "A" + } + ], + "title": "Storages", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 3 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 12, "y": 0 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(pve_not_backed_up_total{job=~\"$job\"})", + "instant": true, + "refId": "A" + } + ], + "title": "Not Backed Up", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 15, "y": 0 }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(max by (job,id) (pve_subscription_status{job=~\"$job\",status=\"active\"} == 1))", + "instant": true, + "refId": "A" + } + ], + "title": "Subscriptions Active", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 3 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 18, "y": 0 }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(max by (job,id) (pve_subscription_status{job=~\"$job\",status=~\"expired|invalid|suspended|notfound\"} == 1))", + "instant": true, + "refId": "A" + } + ], + "title": "Subscriptions Problem", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(max by (job,id,state) (pve_lock_state{job=~\"$job\"} == 1))", + "instant": true, + "refId": "A" + } + ], + "title": "Active Locks", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 4 }, + "id": 9, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "100 * (pve_cpu_usage_ratio{job=~\"$job\",id=~\"node/.*\"} * on(id,instance,job) group_left(name) pve_node_info{job=~\"$job\",name=~\"$node\"})", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Node CPU Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 4 }, + "id": 10, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "100 * ((pve_memory_usage_bytes{job=~\"$job\",id=~\"node/.*\"} / pve_memory_size_bytes{job=~\"$job\",id=~\"node/.*\"}) * on(id,instance,job) group_left(name) pve_node_info{job=~\"$job\",name=~\"$node\"})", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Node Memory Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 4 }, + "id": 11, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "100 * ((pve_disk_usage_bytes{job=~\"$job\",id=~\"node/.*\"} / pve_disk_size_bytes{job=~\"$job\",id=~\"node/.*\"}) * on(id,instance,job) group_left(name) pve_node_info{job=~\"$job\",name=~\"$node\"})", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Node Disk Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "id": 12, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(15, 100 * (pve_cpu_usage_ratio{job=~\"$job\",id=~\"(qemu|lxc)/.*\"} * on(id,instance,job) group_left(name,node,type) pve_guest_info{job=~\"$job\",node=~\"$node\",type=~\"$guest_type\",name=~\"$guest\"}))", + "legendFormat": "{{name}} ({{node}})", + "refId": "A" + } + ], + "title": "Top Guests CPU Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "id": 13, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(15, 100 * ((pve_memory_usage_bytes{job=~\"$job\",id=~\"(qemu|lxc)/.*\"} / pve_memory_size_bytes{job=~\"$job\",id=~\"(qemu|lxc)/.*\"}) * on(id,instance,job) group_left(name,node,type) pve_guest_info{job=~\"$job\",node=~\"$node\",type=~\"$guest_type\",name=~\"$guest\"}))", + "legendFormat": "{{name}} ({{node}})", + "refId": "A" + } + ], + "title": "Top Guests Memory Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, + "id": 14, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(15, ((rate(pve_network_receive_bytes_total{job=~\"$job\",id=~\"(qemu|lxc)/.*\"}[5m]) + rate(pve_network_transmit_bytes_total{job=~\"$job\",id=~\"(qemu|lxc)/.*\"}[5m])) * on(id,instance,job) group_left(name,node,type) pve_guest_info{job=~\"$job\",node=~\"$node\",type=~\"$guest_type\",name=~\"$guest\"}))", + "legendFormat": "{{name}} ({{node}})", + "refId": "A" + } + ], + "title": "Top Guests Network Throughput", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, + "id": 15, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "100 * ((pve_disk_usage_bytes{job=~\"$job\",id=~\"storage/.*\"} / pve_disk_size_bytes{job=~\"$job\",id=~\"storage/.*\"}) * on(id,instance,job) group_left(storage,node,plugintype) pve_storage_info{job=~\"$job\",node=~\"$node\",storage=~\"$storage\"})", + "legendFormat": "{{node}}/{{storage}}", + "refId": "A" + } + ], + "title": "Storage Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, + "id": 16, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "max by (job,id,name,node,type) ((pve_not_backed_up_info{job=~\"$job\"} > 0) * on(id,instance,job) group_left(name,node,type) pve_guest_info{job=~\"$job\",node=~\"$node\",type=~\"$guest_type\",name=~\"$guest\"})", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Guests Not Backed Up", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, + "id": 17, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "max by (job,id,status) (pve_subscription_status{job=~\"$job\"} == 1)", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Subscription Status", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, + "id": 18, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "topk(20, pve_uptime_seconds{job=~\"$job\",id=~\"(qemu|lxc)/.*\"} * on(id,instance,job) group_left(name,node,type) pve_guest_info{job=~\"$job\",node=~\"$node\",type=~\"$guest_type\",name=~\"$guest\"})", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Guest Uptime", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, + "id": 19, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "max by (job,id,state) (pve_ha_state{job=~\"$job\",state!~\"started|online|stopped\"} == 1)", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "HA State Issues", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 44 }, + "id": 20, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "topk(30, ((pve_disk_size_bytes{job=~\"$job\",id=~\"storage/.*\"} - pve_disk_usage_bytes{job=~\"$job\",id=~\"storage/.*\"}) * on(id,instance,job) group_left(storage,node,plugintype) pve_storage_info{job=~\"$job\",node=~\"$node\",storage=~\"$storage\"}))", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Storage Free Space", + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 40, + "tags": ["proxmox", "pve", "infrastructure", "prometheus"], + "templating": { + "list": [ + { + "current": { "selected": true, "text": "Prometheus", "value": "Prometheus" }, + "name": "datasource", + "query": "prometheus", + "refresh": 1, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values(pve_up, job)", + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { "query": "label_values(pve_up, job)", "refId": "Prometheus-job-Variable-Query" }, + "refresh": 1, + "regex": "pve-exporter.*", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values(pve_node_info{job=~\"$job\"}, name)", + "includeAll": true, + "multi": true, + "name": "node", + "options": [], + "query": { "query": "label_values(pve_node_info{job=~\"$job\"}, name)", "refId": "Prometheus-node-Variable-Query" }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values(pve_guest_info{job=~\"$job\"}, type)", + "includeAll": true, + "multi": true, + "name": "guest_type", + "options": [], + "query": { "query": "label_values(pve_guest_info{job=~\"$job\"}, type)", "refId": "Prometheus-guest-type-Variable-Query" }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values(pve_guest_info{job=~\"$job\",node=~\"$node\",type=~\"$guest_type\"}, name)", + "includeAll": true, + "multi": true, + "name": "guest", + "options": [], + "query": { "query": "label_values(pve_guest_info{job=~\"$job\",node=~\"$node\",type=~\"$guest_type\"}, name)", "refId": "Prometheus-guest-Variable-Query" }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values(pve_storage_info{job=~\"$job\",node=~\"$node\"}, storage)", + "includeAll": true, + "multi": true, + "name": "storage", + "options": [], + "query": { "query": "label_values(pve_storage_info{job=~\"$job\",node=~\"$node\"}, storage)", "refId": "Prometheus-storage-Variable-Query" }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": { + "refresh_intervals": ["10s", "30s", "1m", "5m", "15m", "30m", "1h", "6h"] + }, + "timezone": "browser", + "title": "Proxmox PVE Overview", + "uid": "proxmox-pve-overview", + "version": 1, + "weekStart": "" + } + +--- +"metadata": + "name": "traefik-grafana-dashboard" + "namespace": "kube-mon" + "labels": + "grafana_dashboard": "1" + "release": "kube-prometheus-stack-1761819498" +"data": + "traefik-overview.json": | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 30, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Requests/s", + "type": "stat" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code=~\"4..\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "4xx/s", + "type": "stat" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code=~\"5..\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "5xx/s", + "type": "stat" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 250 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "1000 * histogram_quantile(0.95, sum by (le) (rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~\"$entrypoint\"}[$__rate_interval])))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "P95 Latency", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 31, + "panels": [], + "title": "Traffic, Latency & Routing", + "type": "row" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 5, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (entrypoint) (rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\"}[$__rate_interval]))", + "legendFormat": "{{entrypoint}}", + "refId": "A" + } + ], + "title": "Requests by Entrypoint", + "type": "timeseries" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (code) (rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code=~\"[2345]..\"}[$__rate_interval]))", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "title": "Status Codes", + "type": "timeseries" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "1000 * histogram_quantile(0.50, sum by (le) (rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~\"$entrypoint\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "1000 * histogram_quantile(0.95, sum by (le) (rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~\"$entrypoint\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "1000 * histogram_quantile(0.99, sum by (le) (rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~\"$entrypoint\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Latency Percentiles", + "type": "timeseries" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(10, sum by (router) (rate(traefik_router_requests_total{router=~\"$router\"}[$__rate_interval])))", + "legendFormat": "{{router}}", + "refId": "A" + } + ], + "title": "Top Routers by RPS", + "type": "bargauge" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 22 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (entrypoint) (traefik_open_connections{entrypoint=~\"$entrypoint\"})", + "legendFormat": "{{entrypoint}}", + "refId": "A" + } + ], + "title": "Open Connections", + "type": "timeseries" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 8, + "y": 22 + }, + "id": 10, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(traefik_tls_certs_not_after < (time() + 14 * 24 * 3600))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Certs expiring in 14d", + "type": "stat" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 11, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(10, sum by (router) (rate(traefik_router_requests_total{router=~\"$router\",code=~\"5..\"}[$__rate_interval])))", + "legendFormat": "{{router}}", + "refId": "A" + } + ], + "title": "Top Routers by 5xx/s", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 34, + "panels": [], + "title": "Prometheus Deep Dive", + "type": "row" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 50 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 31 + }, + "id": 35, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "100 * sum(rate(traefik_entrypoint_requests_tls_total{entrypoint=~\"$entrypoint\"}[5m])) / clamp_min(sum(rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\"}[5m])), 0.001)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "TLS Share", + "type": "stat", + "description": "Share of Traefik requests arriving over TLS for the selected entrypoints." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 31 + }, + "id": 36, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_requests_tls_total{entrypoint=~\"$entrypoint\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "TLS Requests/s", + "type": "stat", + "description": "TLS request rate across the selected entrypoints." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1048576 + }, + { + "color": "red", + "value": 10485760 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 31 + }, + "id": 37, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_requests_bytes_total{entrypoint=~\"$entrypoint\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Ingress Traffic", + "type": "stat", + "description": "Request bandwidth entering Traefik." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1048576 + }, + { + "color": "red", + "value": 10485760 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 31 + }, + "id": 38, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_responses_bytes_total{entrypoint=~\"$entrypoint\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Egress Traffic", + "type": "stat", + "description": "Response bandwidth leaving Traefik." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 31 + }, + "id": 39, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(increase(traefik_config_reloads_total[24h]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Config Reloads 24h", + "type": "stat", + "description": "How often Traefik reloaded its dynamic configuration in the last 24 hours." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 24 + }, + { + "color": "red", + "value": 168 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 31 + }, + "id": 40, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "(time() - max(traefik_config_last_reload_success)) / 3600", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Last Successful Reload Age (h)", + "type": "stat", + "description": "Hours since the last successful configuration reload reported by Traefik." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 35 + }, + "id": 41, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(12, sum by (exported_service) (rate(traefik_service_requests_total{exported_service=~\"$backend_regex\"}[$__rate_interval])))", + "legendFormat": "{{exported_service}}", + "refId": "A" + } + ], + "title": "Top Backends by RPS", + "type": "bargauge", + "description": "Backend services with the highest request rate according to Traefik service metrics." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 35 + }, + "id": 42, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(12, sum by (exported_service) (rate(traefik_service_responses_bytes_total{exported_service=~\"$backend_regex\"}[$__rate_interval])))", + "legendFormat": "{{exported_service}}", + "refId": "A" + } + ], + "title": "Top Backends by Response Throughput", + "type": "bargauge", + "description": "Backend services returning the most response traffic." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 35 + }, + "id": 43, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(12, 1000 * histogram_quantile(0.95, sum by (exported_service, le) (rate(traefik_service_request_duration_seconds_bucket{exported_service=~\"$backend_regex\"}[$__rate_interval]))))", + "legendFormat": "{{exported_service}}", + "refId": "A" + } + ], + "title": "Slowest Backends by P95", + "type": "bargauge", + "description": "Backends with the highest p95 request latency from Traefik's point of view." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 43 + }, + "id": 44, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(12, 1000 * histogram_quantile(0.95, sum by (router, le) (rate(traefik_router_request_duration_seconds_bucket{router=~\"$router\"}[$__rate_interval]))))", + "legendFormat": "{{router}}", + "refId": "A" + } + ], + "title": "Slowest Routers by P95", + "type": "bargauge", + "description": "Routers with the highest p95 latency." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 43 + }, + "id": 45, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "sum by (tls_version) (rate(traefik_service_requests_tls_total{exported_service=~\"$backend_regex\"}[$__rate_interval]))", + "legendFormat": "TLS {{tls_version}}", + "refId": "A" + } + ], + "title": "TLS Requests by Version", + "type": "bargauge", + "description": "Current TLS request split by negotiated TLS version." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 43 + }, + "id": 46, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "bottomk(12, (max by (cn) (traefik_tls_certs_not_after) - time()) / 86400)", + "legendFormat": "{{cn}}", + "refId": "A" + } + ], + "title": "Soonest Certificate Expiry (days)", + "type": "bargauge", + "description": "Certificates with the fewest days remaining before expiry, deduplicated by CN." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 32, + "panels": [], + "title": "Fail2Ban & Ban History", + "type": "row" + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 52 + }, + "id": 14, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*)\" [$__range]))", + "instant": true, + "queryType": "instant", + "refId": "A" + } + ], + "title": "Ban / Extend Events", + "type": "stat", + "description": "Log-based count of Fail2Ban ban or ban-extension events in the selected time range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 52 + }, + "id": 15, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(sum by (ip) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*)\" | regexp \"(?:Extend Ban|Ban) for (?P[0-9A-Fa-f:.]+)\" | ip =~ \"$ip_regex\" [$__range])))", + "instant": true, + "queryType": "instant", + "refId": "A" + } + ], + "title": "IPs Seen in Ban Events", + "type": "stat", + "description": "Unique IPs with at least one Fail2Ban ban event in the selected time range. This is log-derived, not a kernel live-ban list." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 16, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(25, sum by (ip) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*)\" | regexp \"(?:Extend Ban|Ban) for (?P[0-9A-Fa-f:.]+)\" | ip =~ \"$ip_regex\" [$__range])))", + "instant": true, + "legendFormat": "{{ip}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Blocked IPs by Ban Events", + "type": "bargauge", + "description": "Top IPs ranked by Fail2Ban ban events seen in logs during the selected time range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 17, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*)\" [$__interval]))", + "legendFormat": "block events", + "queryType": "range", + "refId": "A" + } + ], + "title": "Ban Events Over Time", + "type": "timeseries", + "description": "Time series of Fail2Ban ban events. Useful for spotting brute-force or scan spikes." + }, + { + "datasource": "$loki_ds", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 18, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "direction": "backward", + "editorMode": "code", + "expr": "{namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*|Un-Banned .*|Clearing out state for .*|Failed to get Client Identifier.*)\" | msg =~ \".*$ip_regex.*\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Fail2Ban Event Logs", + "type": "logs", + "description": "Raw Fail2Ban event lines, filterable via the shared IP regex variable." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 19, + "panels": [], + "title": "Client IP & Host Analysis", + "type": "row" + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 100 + }, + { + "color": "red", + "value": 250 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 65 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" [$__range])))", + "instant": true, + "queryType": "instant", + "refId": "A" + } + ], + "title": "Unique Client IPs", + "type": "stat", + "description": "Distinct client source IPs seen in Traefik access logs for the selected host/IP filters." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 25 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 65 + }, + "id": 21, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 400 | DownstreamStatus < 500 [$__range])))", + "instant": true, + "queryType": "instant", + "refId": "A" + } + ], + "title": "Unique 4xx Client IPs", + "type": "stat", + "description": "Distinct source IPs that triggered 4xx responses in the selected time range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 65 + }, + "id": 22, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 500 [$__range])))", + "instant": true, + "queryType": "instant", + "refId": "A" + } + ], + "title": "Unique 5xx Client IPs", + "type": "stat", + "description": "Distinct source IPs that hit upstream/server-side failures in the selected time range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 69 + }, + "id": 23, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(15, sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" [$__range])))", + "instant": true, + "legendFormat": "{{ClientHost}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Client IPs by Requests", + "type": "bargauge", + "description": "Request-heavy client IPs from Traefik access logs for the selected range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 69 + }, + "id": 24, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(15, sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 400 | DownstreamStatus < 500 [$__range])))", + "instant": true, + "legendFormat": "{{ClientHost}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Client IPs by 4xx", + "type": "bargauge", + "description": "Client IPs producing the most 4xx responses in the selected range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 69 + }, + "id": 25, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(15, sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 500 [$__range])))", + "instant": true, + "legendFormat": "{{ClientHost}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Client IPs by 5xx", + "type": "bargauge", + "description": "Client IPs associated with the most 5xx responses in the selected range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 26, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(12, sum by (RequestHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 400 | DownstreamStatus < 500 [$__range])))", + "instant": true, + "legendFormat": "{{RequestHost}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Request Hosts by 4xx", + "type": "bargauge", + "description": "Hosts currently attracting the most client-side errors or probes." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 27, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(12, sum by (RequestHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 500 [$__range])))", + "instant": true, + "legendFormat": "{{RequestHost}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Request Hosts by 5xx", + "type": "bargauge", + "description": "Hosts whose backends currently generate the most 5xx responses." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 28, + "options": { + "cellHeight": "sm", + "showHeader": true, + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "fields": "" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(50, sum by (ip) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*)\" | regexp \"(?:Extend Ban|Ban) for (?P[0-9A-Fa-f:.]+)\" | ip =~ \"$ip_regex\" [$__range])))", + "instant": true, + "legendFormat": "{{ip}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Blocked IPs in Selected Range", + "type": "table", + "description": "IP addresses that were blocked by Fail2Ban during the selected time range. This is log-derived history, not a live nftables/iptables inventory." + }, + { + "datasource": "$loki_ds", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 29, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "direction": "backward", + "editorMode": "code", + "expr": "{namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*|Un-Banned .*|Clearing out state for .*|Failed to get Client Identifier.*)\" | msg =~ \".*$ip_regex.*\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Recent Fail2Ban Ban / Unban Events", + "type": "logs", + "description": "Recent Fail2Ban state changes and related events, filtered by the shared IP regex when needed." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 33, + "panels": [], + "title": "Raw Traefik Logs", + "type": "row" + }, + { + "datasource": "$loki_ds", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 94 + }, + "id": 12, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "direction": "backward", + "editorMode": "code", + "expr": "{namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Access Logs (Filtered)", + "type": "logs" + }, + { + "datasource": "$loki_ds", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 94 + }, + "id": 13, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "direction": "backward", + "editorMode": "code", + "expr": "{namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 400", + "queryType": "range", + "refId": "A" + } + ], + "title": "Error Logs (4xx/5xx)", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 41, + "tags": [ + "traefik", + "prometheus", + "loki", + "security", + "fail2ban" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "prometheus_ds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "Loki Traefik", + "value": "Loki Traefik" + }, + "hide": 0, + "includeAll": false, + "label": "Loki", + "multi": false, + "name": "loki_ds", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$prometheus_ds", + "definition": "label_values(traefik_entrypoint_requests_total, entrypoint)", + "hide": 0, + "includeAll": true, + "label": "Entrypoint", + "multi": true, + "name": "entrypoint", + "options": [], + "query": { + "query": "label_values(traefik_entrypoint_requests_total, entrypoint)", + "refId": "EntrypointVar" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$prometheus_ds", + "definition": "label_values(traefik_router_requests_total, router)", + "hide": 0, + "includeAll": true, + "label": "Router", + "multi": true, + "name": "router", + "options": [], + "query": { + "query": "label_values(traefik_router_requests_total, router)", + "refId": "RouterVar" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "traefik-.*", + "value": "traefik-.*" + }, + "hide": 0, + "label": "Pod Regex", + "name": "pod_regex", + "options": [ + { + "selected": true, + "text": "traefik-.*", + "value": "traefik-.*" + } + ], + "query": "traefik-.*", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": ".*", + "value": ".*" + }, + "hide": 0, + "label": "Host Regex", + "name": "request_host", + "options": [ + { + "selected": true, + "text": ".*", + "value": ".*" + } + ], + "query": ".*", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": ".*", + "value": ".*" + }, + "hide": 0, + "label": "IP Regex", + "name": "ip_regex", + "options": [ + { + "selected": true, + "text": ".*", + "value": ".*" + } + ], + "query": ".*", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": ".*", + "value": ".*" + }, + "hide": 0, + "label": "Backend Regex", + "name": "backend_regex", + "options": [ + { + "selected": true, + "text": ".*", + "value": ".*" + } + ], + "query": ".*", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "Traefik Overview", + "uid": "traefik-overview", + "version": 10, + "weekStart": "" + } +--- +"metadata": + "name": "traefik-loki-datasource" + "namespace": "kube-mon" + "labels": + "grafana_datasource": "1" + "release": "kube-prometheus-stack-1761819498" +"data": + "datasource.yaml": | + apiVersion: 1 + datasources: + - name: Loki Traefik + type: loki + uid: loki-traefik + access: proxy + url: http://loki.loki.svc.cluster.local:3100 + isDefault: false + jsonData: + maxLines: 1000 +--- +"metadata": + "name": "vmware-esxi-dashboard" + "namespace": "kube-mon" + "labels": + "grafana_dashboard": "1" +"data": + "vmware-esxi-overview.json": | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 4, "w": 3, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(max by (job,host_name) (vmware_host_power_state{job=~\"$job\",host_name=~\"$host\"}))", + "instant": true, + "refId": "A" + } + ], + "title": "ESXi Hosts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 3, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(max by (job,host_name) (vmware_host_connection_state{job=~\"$job\",host_name=~\"$host\",state=\"connected\"}))", + "instant": true, + "refId": "A" + } + ], + "title": "Connected Hosts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 6, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(max by (job,host_name) (vmware_host_maintenance_mode{job=~\"$job\",host_name=~\"$host\"}))", + "instant": true, + "refId": "A" + } + ], + "title": "Hosts in Maintenance", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 4, "w": 3, "x": 9, "y": 0 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(max by (job,vm_name) (vmware_vm_power_state{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"}))", + "instant": true, + "refId": "A" + } + ], + "title": "VMs Total", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 12, "y": 0 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(max by (job,vm_name) (vmware_vm_power_state{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"}))", + "instant": true, + "refId": "A" + } + ], + "title": "VMs Powered On", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 15, "y": 0 }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(max by (job,vm_name) (vmware_vm_power_state{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"} == 0))", + "instant": true, + "refId": "A" + } + ], + "title": "VMs Powered Off", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 18, "y": 0 }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(max by (job,vm_name) (vmware_vm_snapshots{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"} > 0))", + "instant": true, + "refId": "A" + } + ], + "title": "VMs with Snapshots", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 3 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum((100 * (1 - (vmware_datastore_freespace_size{job=~\"$job\",ds_name=~\"$datastore\"} / vmware_datastore_capacity_size{job=~\"$job\",ds_name=~\"$datastore\"})) > bool 80) * (vmware_datastore_capacity_size{job=~\"$job\",ds_name=~\"$datastore\"} > bool 0))", + "instant": true, + "refId": "A" + } + ], + "title": "Datastores > 80%", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "id": 9, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "100 * (vmware_host_cpu_usage_average{job=~\"$job\",host_name=~\"$host\"} / vmware_host_cpu_max{job=~\"$job\",host_name=~\"$host\"})", + "legendFormat": "{{host_name}}", + "refId": "A" + } + ], + "title": "Host CPU Utilization (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "id": 10, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "100 * (vmware_host_memory_usage{job=~\"$job\",host_name=~\"$host\"} / vmware_host_memory_max{job=~\"$job\",host_name=~\"$host\"})", + "legendFormat": "{{host_name}}", + "refId": "A" + } + ], + "title": "Host Memory Utilization (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "id": 11, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "expr": "sum(vmware_host_net_bytesRx_average{job=~\"$job\",host_name=~\"$host\"} * 1024)", + "legendFormat": "RX", + "refId": "A" + }, + { + "expr": "sum(vmware_host_net_bytesTx_average{job=~\"$job\",host_name=~\"$host\"} * 1024)", + "legendFormat": "TX", + "refId": "B" + } + ], + "title": "Host Network Throughput", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "id": 12, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "(100 * (1 - (vmware_datastore_freespace_size{job=~\"$job\",ds_name=~\"$datastore\"} / vmware_datastore_capacity_size{job=~\"$job\",ds_name=~\"$datastore\"}))) and on(job,instance,dc_name,ds_name) (vmware_datastore_capacity_size{job=~\"$job\",ds_name=~\"$datastore\"} > 0)", + "legendFormat": "{{ds_name}}", + "refId": "A" + } + ], + "title": "Datastore Usage (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, + "id": 13, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(15, 100 * (vmware_vm_cpu_usage_average{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"} / vmware_vm_max_cpu_usage{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"}))", + "legendFormat": "{{vm_name}} ({{host_name}})", + "refId": "A" + } + ], + "title": "Top VMs by CPU Utilization (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, + "id": 14, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(15, 100 * (vmware_vm_mem_usage_average{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"} / vmware_vm_memory_max{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"}))", + "legendFormat": "{{vm_name}} ({{host_name}})", + "refId": "A" + } + ], + "title": "Top VMs by Memory Utilization (%)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, + "id": 15, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(20, (vmware_vm_net_received_average{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"} + vmware_vm_net_transmitted_average{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"}) * 1024)", + "legendFormat": "{{vm_name}}", + "refId": "A" + } + ], + "title": "Top VMs by Network Throughput", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, + "id": 16, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "expr": "topk(20, (vmware_vm_disk_read_average{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"} + vmware_vm_disk_write_average{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"}) * 1024)", + "legendFormat": "{{vm_name}}", + "refId": "A" + } + ], + "title": "Top VMs by Disk Throughput", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, + "id": 17, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "max by (job,host_name,vm_name,ds_name) (vmware_vm_power_state{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"} == 0)", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Powered-Off VMs", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, + "id": 18, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "topk(50, max by (job,host_name,vm_name,ds_name) (vmware_vm_snapshots{job=~\"$job\",host_name=~\"$host\",vm_name=~\"$vm\"}))", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "VM Snapshots", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 }, + "id": 19, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "max by (job,host_name) (vmware_host_maintenance_mode{job=~\"$job\",host_name=~\"$host\"} == 1)", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Hosts in Maintenance", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 }, + "id": 20, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "max by (job,host_name,state) (vmware_host_connection_state{job=~\"$job\",host_name=~\"$host\",state!=\"connected\"} == 1)", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Hosts Not Connected", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, + "id": 21, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "topk(30, (100 * (1 - (vmware_datastore_freespace_size{job=~\"$job\",ds_name=~\"$datastore\"} / vmware_datastore_capacity_size{job=~\"$job\",ds_name=~\"$datastore\"}))) and on(job,instance,dc_name,ds_name) (vmware_datastore_capacity_size{job=~\"$job\",ds_name=~\"$datastore\"} > 0))", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Datastore Usage (Top)", + "type": "table" + }, + { + "datasource": "$datasource", + "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 52 }, + "id": 22, + "options": { "showHeader": true }, + "targets": [ + { + "expr": "topk(30, vmware_datastore_freespace_size{job=~\"$job\",ds_name=~\"$datastore\"})", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } } + ], + "title": "Datastore Free Space", + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 40, + "tags": [ + "vmware", + "esxi", + "vcenter", + "prometheus" + ], + "templating": { + "list": [ + { + "current": { "selected": true, "text": "Prometheus", "value": "Prometheus" }, + "name": "datasource", + "query": "prometheus", + "refresh": 1, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values(vmware_host_power_state, job)", + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { "query": "label_values(vmware_host_power_state, job)", "refId": "Prometheus-job-Variable-Query" }, + "refresh": 1, + "regex": "vmware-exporter.*", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values(vmware_host_power_state{job=~\"$job\"}, host_name)", + "includeAll": true, + "multi": true, + "name": "host", + "options": [], + "query": { "query": "label_values(vmware_host_power_state{job=~\"$job\"}, host_name)", "refId": "Prometheus-host-Variable-Query" }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values(vmware_vm_power_state{job=~\"$job\",host_name=~\"$host\"}, vm_name)", + "includeAll": true, + "multi": true, + "name": "vm", + "options": [], + "query": { "query": "label_values(vmware_vm_power_state{job=~\"$job\",host_name=~\"$host\"}, vm_name)", "refId": "Prometheus-vm-Variable-Query" }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": "$datasource", + "definition": "label_values(vmware_datastore_capacity_size{job=~\"$job\"}, ds_name)", + "includeAll": true, + "multi": true, + "name": "datastore", + "options": [], + "query": { "query": "label_values(vmware_datastore_capacity_size{job=~\"$job\"}, ds_name)", "refId": "Prometheus-ds-Variable-Query" }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": { + "refresh_intervals": ["10s", "30s", "1m", "5m", "15m", "30m", "1h", "6h", "12h", "24h"] + }, + "timezone": "browser", + "title": "VMware ESXi Overview", + "uid": "vmware-esxi-overview", + "version": 1, + "weekStart": "" + } + diff --git a/live-monitoring-deployments/loki/01-core.yaml b/live-monitoring-deployments/loki/01-core.yaml new file mode 100644 index 0000000..b69e062 --- /dev/null +++ b/live-monitoring-deployments/loki/01-core.yaml @@ -0,0 +1,71 @@ +--- +"kind": "ServiceAccount" +"apiVersion": "v1" +"metadata": + "name": "loki" + "namespace": "loki" +--- +"kind": "ConfigMap" +"apiVersion": "v1" +"metadata": + "name": "loki-config" + "namespace": "loki" +"data": + "loki.yaml": | + auth_enabled: false + + server: + http_listen_port: 3100 + grpc_listen_port: 9096 + + common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + + schema_config: + configs: + - from: "2024-04-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + + limits_config: + allow_structured_metadata: true + reject_old_samples: true + reject_old_samples_max_age: 168h + ingestion_rate_mb: 16 + ingestion_burst_size_mb: 32 + max_cache_freshness_per_query: 10m + + storage_config: + filesystem: + directory: /loki/chunks + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + +--- +"kind": "PersistentVolumeClaim" +"apiVersion": "v1" +"metadata": + "name": "loki-data" + "namespace": "loki" +"spec": + "accessModes": + - "ReadWriteOnce" + "resources": + "requests": + "storage": "20Gi" + "storageClassName": "nfs-csi" + "volumeMode": "Filesystem" diff --git a/live-monitoring-deployments/loki/02-services.yaml b/live-monitoring-deployments/loki/02-services.yaml new file mode 100644 index 0000000..fee5954 --- /dev/null +++ b/live-monitoring-deployments/loki/02-services.yaml @@ -0,0 +1,50 @@ +--- +"kind": "Service" +"apiVersion": "v1" +"metadata": + "name": "loki" + "namespace": "loki" + "labels": + "k8slens-edit-resource-version": "v1" +"spec": + "ports": + - + "name": "http-metrics" + "protocol": "TCP" + "port": 3100 + "targetPort": "http-metrics" + "nodePort": 31940 + - + "name": "grpc" + "protocol": "TCP" + "port": 9096 + "targetPort": "grpc" + "nodePort": 30320 + "selector": + "app": "loki" + "type": "NodePort" + "sessionAffinity": "None" + "externalTrafficPolicy": "Cluster" +--- +"kind": "Service" +"apiVersion": "v1" +"metadata": + "name": "loki-headless" + "namespace": "loki" +"spec": + "ports": + - + "name": "http-metrics" + "protocol": "TCP" + "port": 3100 + "targetPort": "http-metrics" + - + "name": "grpc" + "protocol": "TCP" + "port": 9096 + "targetPort": "grpc" + "selector": + "app": "loki" + "type": "ClusterIP" + "sessionAffinity": "None" + "clusterIP": "None" diff --git a/live-monitoring-deployments/loki/03-statefulset.yaml b/live-monitoring-deployments/loki/03-statefulset.yaml new file mode 100644 index 0000000..d9857c4 --- /dev/null +++ b/live-monitoring-deployments/loki/03-statefulset.yaml @@ -0,0 +1,100 @@ +--- +"kind": "StatefulSet" +"apiVersion": "apps/v1" +"metadata": + "name": "loki" + "namespace": "loki" +"spec": + "replicas": 1 + "selector": + "matchLabels": + "app": "loki" + "template": + "metadata": + "labels": + "app": "loki" + "spec": + "volumes": + - + "name": "config" + "configMap": + "name": "loki-config" + "defaultMode": 420 + - + "name": "storage" + "persistentVolumeClaim": + "claimName": "loki-data" + "containers": + - + "name": "loki" + "image": "grafana/loki:3.4.2" + "args": + - "-config.file=/etc/loki/config/loki.yaml" + "ports": + - + "name": "http-metrics" + "containerPort": 3100 + "protocol": "TCP" + - + "name": "grpc" + "containerPort": 9096 + "protocol": "TCP" + "resources": + "limits": + "cpu": "1" + "memory": "1Gi" + "requests": + "cpu": "100m" + "memory": "256Mi" + "volumeMounts": + - + "name": "storage" + "mountPath": "/loki" + - + "name": "config" + "mountPath": "/etc/loki/config" + "livenessProbe": + "httpGet": + "path": "/ready" + "port": "http-metrics" + "scheme": "HTTP" + "initialDelaySeconds": 45 + "timeoutSeconds": 1 + "periodSeconds": 10 + "successThreshold": 1 + "failureThreshold": 3 + "readinessProbe": + "httpGet": + "path": "/ready" + "port": "http-metrics" + "scheme": "HTTP" + "initialDelaySeconds": 45 + "timeoutSeconds": 1 + "periodSeconds": 10 + "successThreshold": 1 + "failureThreshold": 3 + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "File" + "imagePullPolicy": "IfNotPresent" + "restartPolicy": "Always" + "terminationGracePeriodSeconds": 30 + "dnsPolicy": "ClusterFirst" + "serviceAccountName": "loki" + "serviceAccount": "loki" + "securityContext": + "runAsUser": 10001 + "runAsGroup": 10001 + "runAsNonRoot": true + "fsGroup": 10001 + "schedulerName": "default-scheduler" + "serviceName": "loki" + "podManagementPolicy": "OrderedReady" + "updateStrategy": + "type": "RollingUpdate" + "rollingUpdate": + "partition": 0 + "maxUnavailable": 1 + "revisionHistoryLimit": 10 + "persistentVolumeClaimRetentionPolicy": + "whenDeleted": "Retain" + "whenScaled": "Retain" diff --git a/live-monitoring-deployments/prometheus/01-rbac.yaml b/live-monitoring-deployments/prometheus/01-rbac.yaml new file mode 100644 index 0000000..9da14e9 --- /dev/null +++ b/live-monitoring-deployments/prometheus/01-rbac.yaml @@ -0,0 +1,102 @@ +--- +"kind": "ServiceAccount" +"apiVersion": "v1" +"metadata": + "name": "kube-prometheus-stack-1761-prometheus" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-prometheus" + "app.kubernetes.io/component": "prometheus" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/name": "kube-prometheus-stack-prometheus" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"automountServiceAccountToken": true +--- +"kind": "ClusterRole" +"apiVersion": "rbac.authorization.k8s.io/v1" +"metadata": + "name": "kube-prometheus-stack-1761-prometheus" + "labels": + "app": "kube-prometheus-stack-prometheus" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"rules": + - + "verbs": + - "get" + - "list" + - "watch" + "apiGroups": + - "" + "resources": + - "nodes" + - "nodes/metrics" + - "services" + - "endpoints" + - "pods" + - + "verbs": + - "get" + - "list" + - "watch" + "apiGroups": + - "discovery.k8s.io" + "resources": + - "endpointslices" + - + "verbs": + - "get" + - "list" + - "watch" + "apiGroups": + - "networking.k8s.io" + "resources": + - "ingresses" + - + "verbs": + - "get" + "nonResourceURLs": + - "/metrics" + - "/metrics/cadvisor" +--- +"kind": "ClusterRoleBinding" +"apiVersion": "rbac.authorization.k8s.io/v1" +"metadata": + "name": "kube-prometheus-stack-1761-prometheus" + "labels": + "app": "kube-prometheus-stack-prometheus" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"subjects": + - + "kind": "ServiceAccount" + "name": "kube-prometheus-stack-1761-prometheus" + "namespace": "kube-mon" +"roleRef": + "apiGroup": "rbac.authorization.k8s.io" + "kind": "ClusterRole" + "name": "kube-prometheus-stack-1761-prometheus" diff --git a/live-monitoring-deployments/prometheus/02-config.yaml b/live-monitoring-deployments/prometheus/02-config.yaml new file mode 100644 index 0000000..bcecf31 --- /dev/null +++ b/live-monitoring-deployments/prometheus/02-config.yaml @@ -0,0 +1,3198 @@ +--- +"kind": "Secret" +"apiVersion": "v1" +"metadata": + "name": "prometheus-kube-prometheus-stack-1761-prometheus" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/managed-by": "prometheus-operator" + "managed-by": "prometheus-operator" +"data": + "prometheus.yaml.gz": "H4sIAAAAAAAA/+ydS3PqOBbH9/kUXvQC7i2eCbkJWUx1Vc/ULKZ7uqa3t0olpAOokSVfSSZJjee7T1l+YJ4NCQETzsYJkiUdHT3+Pz8kT6QeUTm8CQLLDI2ACOXAzKkcBr2BvQkCeHFgFJVE0hFIm54ZBJHRIbgpxHYYzOIRtEKtOv6fRUzLOspmrd63+14ldCU5MRBJwegw+Knx+79/Ib/9/Ovfm2mpcypj6oRWFYNuu/bGxBLIWEiww5tW0AHHOovcOmmsrQS09rKplSbzeba6nS/tVxrKm9wbTKuxmPiy/tQjomgIw8CCmQsGv2olnDad0gFTGhn98tqCF9fp3gTBVCttCr8FYyot3ATeX0aBA0ssXxQQBK3AaAnDABSPtFAu81VapI0og9z1eUjxo1X6f3sThuCMYJZE1E2HQSf/eRMEBrxxy0ZYHRsGS83tK+//c9RMwGWRw4AQF0ak0pqFj3xGlKXtNwxmAJFPvDFnQkJwlFS9krk3O4/QKNrvzMiABeXKBAYm8DIMGpVWaT45E7/PuKJxSKSNK+paFrZw7RY/7siQcm7AWpK7eCYUPzhRaY+FiBrqtBkGT1UDf9McnhrtL808MJKUQQjKDYOf/tv734Y2VppDfevzu+aHVSfS/JDalKNvk2cqcXtnWHTZsmbLeebRh+QYab4ttwMrm+bEtHJUKDDb8ixPWBpG3OiDhlFaVDSldmn0NP5BhQSe/BEzBsCBN4/o2WIG29RJWivnFt1xPcEeA7zowqQMSSfJKbXTrTNoGVk4ov21+bTB2p4PK3yex+2yZf+SQ81jmcr5chlpfLi7Fy1lUwRwYelIArFTarhQk2rtfmr88c+f//NL8ylpf31qf10qz0/H+2jtDmGnEowLqaITMOfR4a2CC8p7Zepc1B8GuRhdhQpvbq9qUx1BnJdtNiChmGH2truaaLftaV976D3ePT4c3XILckzCrMsfZv5ayqIOqYUfjj9pv249w6i+vID8g/yD/PN2/tlnhCMA1QmAerUCIGQdZJ3PwToGpKYcDPIO8g7yzmflnX1HOTLPuZknEmmqk97x4TCmsXT+wcsUUsNTPE4TO2nz/LOzhbLAYgPEzkRE5mDE+HVhVxBkpue1X5jp4xj1z7uGQWdOTcfEqmOBGXC2szixLXQndxplTMfKdRhtM5PaNgJqwBCnZ6DelJNPeSHsxnQYaQX5UN4bJZaTFQOy7FNHh57I6LngcCDwLKWqAluW6CS3d/DZFrIOss5Fsc4RJtJtxZXgcqRbS/hg7UMZq7gxQ/aXcuL7XOlzudypwTFODPyIwbqklMsihFipN4aK9dAm4bHJXvixwLTiloxiNgP31Oh+b/cGSfd7u58ebv3B/77zB//vfXr4lh4e0sNj0vve7g/Soz98GyT95Da5TX/eJXfpn/vkW/KQPCa9QdLvJnfd5G6QDLrNxvd2t/m3Jdf58f9OPGXaAFf21I8j7at1EF4bBH7QDby8Da/63t0ez//wHSjkROTEa+HEP/XoX+k/J8XEfeYYpMUz35HzUelIMVpKMK0zvZFVItDb7tHliom34epAYFu6FBIZElmt6oNEhkSGRIZEVkciA8c43oa6dAhKWxGxB7GnVvVB7EHsQexB7Kkj9vg14Mg9l849vhkRfBB8alUfBB8EHwQfBJ86go9lU+CxxAdvyF7vZK+yJyF/IX/Vqj7IX8hfyF/IXzXkLwmb9oLMVSgLdCIE62gYLSI+FsiCgDpH2dR3bE4dLVNpDgsqRG47KbdVI4VeTGCHoNy2PKp8JOEY+2Aumz97sIejZzXR8Q3cYykjghqCGoLa1YBadb452YLGPeeYcreknNaWy6iecCh5IQXWbtUks4LoCLLljjaxThs6gUVQuRSyub4Wst0fJP32wC9bHCS9fje5737YqsUCYNf3EPsrgHWGslklglhHJSiwtWPc1R3Tu9t3TO8wyufC+t2mEI4RjhGOEY4RjhGOEY4RjhGO3wbHeTmLocqimDTY2BI3Ndo5CbyEYKcdlYnUlBM6B8/Mva5NMuZbOSu2YJaDmut4/EYTx5Y0hCYsNgaUS4T2oLtiQBH6DGIydWu1MEC5JSGYCfA8yAJz2pAsZino2Yh0xGZh+Y9q0uNVLYRQm1fSCGkUAfdkmdhnGh2vhEaaJ+FgmRGR08YmjtqZv0BwkLhp7hj68uYiW4FlOoKdFRxTIWNT+PRpKsBQw6avby/TX0KMC3VdK1eBe9ZmRtpfnhqMSpEwIUUcJkyJRL6wJEUTqRmVXNnExUo221+O5nMbATsoO1HgTaa6ZZbtr08fdpHZv8ynJFsuFSOjR34DR7xQxAtFvFDEC0W8UMQLRbxQxAvFa71QfCclZg9H9Ka3mD+SBovvqu5muAWkrXyPloFxttMlRV5kRw0pD4W1QivCaD79r2wK/heuuRxIO5jI1uhm3RWFF/DlZNy/ux71QUxDTDsGph1ESggxNYaYChnU67u0V8wNi4CrJofP8VE2/AAtQg5CzmeHHPwY2+VwDn5+FjkHOQc/Pousg6yDrIMfn/2MvJOJU2ti6JgqeqYnVKvrWAZ2EexECDp2u1e3rDznuhxaWnldRyjrqGLvfu1nKZ+zEcqJXmnK+y7eoEFoQWhBaDnay0LxCKoC4KdLspgud1px9D1zEKMuAaOyt7AddVC89XUWokL4uRr4We9xJ+Gg+jIDMhAyEDLQkRhoxwR0KvJB6qk99VTi0rm0BS+pbpx+1+YMft6wQyAi0zUh07buihssI0AhQCFA4QbLiE5HQqdofm4aes/DtWieGpfOqmGeufc/FF7lMKaxdJVmK2J6j/127/6h3Wt/u78cuDqYNJaxYn5ilKiv5CJCIEIgQuB9j88j3i1Fz7Tu6dwC3nu4vUYFTxscVRxVHFUcVRxV/IJV/EdMzSy2rdmDbdFInFDEV0p+p5b/uLalQCvuQzFGMUYxRjFGMb5gMZ6Hz9TU9Jb4VYjqSgOgqKKooqiiqKKofh5RbdluH4X1zMKaNgKKK4oriiuKK4rrpxLXWxTX84vrLYoriiuKK4oriuunEtcBiuv5xXWA4oriiuKK4ori+qnE9R7F9fzieo/iiuKK4oriiuJ6eeLqDIWxmJV/T6eneYlXvl3u+8Q49+GHCzCndjrS1Bw09aEKowqjCl+KCu81xFGK3yvF1mlDJ369qrN8lBWlY0f0mGjDwXi9I89Ccf08DLr2hkowTij/cWP/P9kobUUpPqrstLlZFRlLPSv8N419ZiFVdAImz8R/rTsyMBYvw6CTdftlWd3BAdtIYBMLrNBAsLbhWSTIHIz1VZr387qsVXuj4m0cqnuOrNJjsx2fhq747Qg2bFHeYG2T9v8HAAD//yPM0sEd8gAA" +"type": "Opaque" +--- +"kind": "Secret" +"apiVersion": "v1" +"metadata": + "name": "prometheus-kube-prometheus-stack-1761-prometheus-web-config" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/managed-by": "prometheus-operator" + "managed-by": "prometheus-operator" +"data": + "web-config.yaml": "" +"type": "Opaque" +--- +"kind": "Secret" +"apiVersion": "v1" +"metadata": + "name": "prometheus-kube-prometheus-stack-1761-prometheus-tls-assets-0" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/managed-by": "prometheus-operator" + "managed-by": "prometheus-operator" +"data": + "0_kube-mon_kube-prometheus-stack-1761-admission_ca": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkVENDQVJ5Z0F3SUJBZ0lSQU5Sbk1ta2lxa3pCVktsTXdnVTNxYTh3Q2dZSUtvWkl6ajBFQXdJd0R6RU4KTUFzR0ExVUVDaE1FYm1sc01UQWdGdzB5TlRFd016QXhNREV6TXpWYUdBOHlNVEkxTVRBd05qRXdNVE16TlZvdwpEekVOTUFzR0ExVUVDaE1FYm1sc01UQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJNQkhhakJ2CmNJREZYeUtmTkpyWm9hZExIeUtyUzQrQ0s0RnJFODlSVEp3d3VKYm5qTGFlVjRBRHY1cVk2UDdlYm1ZNXFxcDAKaVcwVHZNbDNvYktWakQralZ6QlZNQTRHQTFVZER3RUIvd1FFQXdJQ0JEQVRCZ05WSFNVRUREQUtCZ2dyQmdFRgpCUWNEQVRBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJRTVdlSWQyNldxU1c5QzhyVXFXU0JHCjExUEM2REFLQmdncWhrak9QUVFEQWdOSEFEQkVBaUJ0aCtVUWhqOEs1SjdFaEdWOVZQUWE3NzR5ZDYvRFUyS2cKaWNzRzJqNHJ6UUlnY0hTNVZiMlhrckRmNTRlbEpqem0zaEdlbzZZSTA3WlMrTmc2dTlGT2lpMD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=" +"type": "Opaque" +--- +"kind": "ConfigMap" +"apiVersion": "v1" +"metadata": + "name": "prometheus-kube-prometheus-stack-1761-prometheus-rulefiles-0" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/managed-by": "prometheus-operator" + "managed-by": "prometheus-operator" + "prometheus-name": "kube-prometheus-stack-1761-prometheus" +"data": + "kube-mon-kube-prometheus-stack-1761-alertmanager.rules-15c966ee-eb8b-4abb-9b70-3c5a925b0252.yaml": | + groups: + - name: alertmanager.rules + rules: + - alert: AlertmanagerFailedReload + annotations: + description: Configuration has failed to load for {{ $labels.namespace }}/{{ + $labels.pod}}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon"}[5m]) == 0 + for: 10m + labels: + severity: critical + - alert: AlertmanagerMembersInconsistent + annotations: + description: Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} has only + found {{ $value }} members of the {{$labels.job}} cluster. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster + members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon"}[5m]) + < on (namespace,service,cluster) group_left + count by (namespace,service,cluster) (max_over_time(alertmanager_cluster_members{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon"}[5m])) + for: 15m + labels: + severity: critical + - alert: AlertmanagerFailedToSendAlerts + annotations: + description: Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} failed to + send {{ $value | humanizePercentage }} of notifications to {{ $labels.integration + }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon"}[15m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon"}[15m]) + ) + > 0.01 + for: 5m + labels: + severity: warning + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: + description: The minimum notification failure rate to {{ $labels.integration + }} sent from any instance in the {{$labels.job}} cluster is {{ $value | humanizePercentage + }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications + to a critical integration. + expr: |- + min by (namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon", integration=~`.*`}[15m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon", integration=~`.*`}[15m]) + ) + > 0.01 + for: 5m + labels: + severity: critical + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: + description: The minimum notification failure rate to {{ $labels.integration + }} sent from any instance in the {{$labels.job}} cluster is {{ $value | humanizePercentage + }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications + to a non-critical integration. + expr: |- + min by (namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon", integration!~`.*`}[15m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon", integration!~`.*`}[15m]) + ) + > 0.01 + for: 5m + labels: + severity: warning + - alert: AlertmanagerConfigInconsistent + annotations: + description: Alertmanager instances within the {{$labels.job}} cluster have + different configurations. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. + expr: |- + count by (namespace,service,cluster) ( + count_values by (namespace,service,cluster) ("config_hash", alertmanager_config_hash{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon"}) + ) + != 1 + for: 20m + labels: + severity: critical + - alert: AlertmanagerClusterDown + annotations: + description: '{{ $value | humanizePercentage }} of Alertmanager instances within + the {{$labels.job}} cluster have been up for less than half of the last 5m.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster + are down. + expr: |- + ( + count by (namespace,service,cluster) ( + avg_over_time(up{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon"}[5m]) < 0.5 + ) + / + count by (namespace,service,cluster) ( + up{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon"} + ) + ) + >= 0.5 + for: 5m + labels: + severity: critical + - alert: AlertmanagerClusterCrashlooping + annotations: + description: '{{ $value | humanizePercentage }} of Alertmanager instances within + the {{$labels.job}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster + are crashlooping. + expr: |- + ( + count by (namespace,service,cluster) ( + changes(process_start_time_seconds{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon"}[10m]) > 4 + ) + / + count by (namespace,service,cluster) ( + up{job="kube-prometheus-stack-1761-alertmanager",container="alertmanager",namespace="kube-mon"} + ) + ) + >= 0.5 + for: 5m + labels: + severity: critical + + "kube-mon-kube-prometheus-stack-1761-config-reloaders-ac1cdf15-0ac7-4cdd-95dc-225ebbfe1b50.yaml": | + groups: + - name: config-reloaders + rules: + - alert: ConfigReloaderSidecarErrors + annotations: + description: |- + Errors encountered while the {{$labels.pod}} config-reloader sidecar attempts to sync config in {{$labels.namespace}} namespace. + As a result, configuration for service running in {{$labels.pod}} may be stale and cannot be updated anymore. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + for: 10m + labels: + severity: warning + + "kube-mon-kube-prometheus-stack-1761-etcd-2b0c1fbf-1a47-4909-9d44-6e22ba4561d6.yaml": | + groups: + - name: etcd + rules: + - alert: etcdMembersDown + annotations: + description: 'etcd cluster "{{ $labels.job }}": members are down ({{ $value + }}).' + summary: etcd cluster members are down. + expr: |- + max without (endpoint) ( + sum without (instance, pod) (up{job=~".*etcd.*"} == bool 0) + or + count without (To) ( + sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + ) + ) + > 0 + for: 20m + labels: + severity: warning + - alert: etcdInsufficientMembers + annotations: + description: 'etcd cluster "{{ $labels.job }}": insufficient members ({{ $value + }}).' + summary: etcd cluster has insufficient number of members. + expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance, pod) < ((count(up{job=~".*etcd.*"}) + without (instance, pod) + 1) / 2) + for: 3m + labels: + severity: critical + - alert: etcdNoLeader + annotations: + description: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance }} + has no leader.' + summary: etcd cluster has no leader. + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: 1m + labels: + severity: critical + - alert: etcdHighNumberOfLeaderChanges + annotations: + description: 'etcd cluster "{{ $labels.job }}": {{ $value }} leader changes + within the last 15 minutes. Frequent elections may be a sign of insufficient + resources, high network latency, or disruptions by other components and should + be investigated.' + summary: etcd cluster has high number of leader changes. + expr: increase((max without (instance, pod) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) + or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) + >= 4 + for: 5m + labels: + severity: warning + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: + description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for + {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 1 + for: 10m + labels: + severity: warning + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: + description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for + {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 5 + for: 5m + labels: + severity: critical + - alert: etcdGRPCRequestsSlow + annotations: + description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests + is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method + }} method.' + summary: etcd grpc requests are slow + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) + > 0.15 + for: 10m + labels: + severity: critical + - alert: etcdMemberCommunicationSlow + annotations: + description: 'etcd cluster "{{ $labels.job }}": member communication with {{ + $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance + }}.' + summary: etcd cluster member communication is slow. + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: 10m + labels: + severity: warning + - alert: etcdHighNumberOfFailedProposals + annotations: + description: 'etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures + within the last 30 minutes on etcd instance {{ $labels.instance }}.' + summary: etcd cluster has high number of proposal failures. + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: 15m + labels: + severity: warning + - alert: etcdHighFsyncDurations + annotations: + description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations + are {{ $value }}s on etcd instance {{ $labels.instance }}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: 10m + labels: + severity: warning + - alert: etcdHighFsyncDurations + annotations: + description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations + are {{ $value }}s on etcd instance {{ $labels.instance }}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 1 + for: 10m + labels: + severity: critical + - alert: etcdHighCommitDurations + annotations: + description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations + {{ $value }}s on etcd instance {{ $labels.instance }}.' + summary: etcd cluster 99th percentile commit durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: 10m + labels: + severity: warning + - alert: etcdDatabaseQuotaLowSpace + annotations: + description: 'etcd cluster "{{ $labels.job }}": database size exceeds the defined + quota on etcd instance {{ $labels.instance }}, please defrag or increase the + quota as the writes to etcd will be disabled when it is full.' + summary: etcd cluster database is running full. + expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / + last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > + 95 + for: 10m + labels: + severity: critical + - alert: etcdExcessiveDatabaseGrowth + annotations: + description: 'etcd cluster "{{ $labels.job }}": Predicting running out of disk + space in the next four hours, based on write observations within the past + four hours on etcd instance {{ $labels.instance }}, please check as it might + be disruptive.' + summary: etcd cluster database growing very fast. + expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) + > etcd_server_quota_backend_bytes{job=~".*etcd.*"} + for: 10m + labels: + severity: warning + - alert: etcdDatabaseHighFragmentationRatio + annotations: + description: 'etcd cluster "{{ $labels.job }}": database size in use on instance + {{ $labels.instance }} is {{ $value | humanizePercentage }} of the actual + allocated disk space, please run defragmentation (e.g. etcdctl defrag) to + retrieve the unused fragmented disk space.' + runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation + summary: etcd database size in use is less than 50% of the actual allocated + storage. + expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) + / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 + and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600 + for: 10m + labels: + severity: warning + + "kube-mon-kube-prometheus-stack-1761-general.rules-e6e7e2fc-5f23-4458-8fb6-2649daf88c98.yaml": | + groups: + - name: general.rules + rules: + - alert: TargetDown + annotations: + description: '{{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service + }} targets in {{ $labels.namespace }} namespace are down.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/targetdown + summary: One or more targets are unreachable. + expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) + BY (cluster, job, namespace, service)) > 10 + for: 10m + labels: + severity: warning + - alert: Watchdog + annotations: + description: | + This is an alert meant to ensure that the entire alerting pipeline is functional. + This alert is always firing, therefore it should always be firing in Alertmanager + and always fire against a receiver. There are integrations with various notification + mechanisms that send a notification when this alert is not firing. For example the + "DeadMansSnitch" integration in PagerDuty. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager + is working properly. + expr: vector(1) + labels: + severity: none + - alert: InfoInhibitor + annotations: + description: | + This is an alert that is used to inhibit info alerts. + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + other alerts. + This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a + severity of 'warning' or 'critical' starts firing on the same namespace. + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on (namespace) ALERTS{alertname != + "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: none + + "kube-mon-kube-prometheus-stack-1761-k8s.rules.container-cpu-usage-second-e8eb8b18-cdf0-4d8f-9a5a-dc2cc584b95f.yaml": | + groups: + - name: k8s.rules.container_cpu_usage_seconds_total + rules: + - expr: |- + sum by (cluster, namespace, pod, container) ( + rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate5m + - expr: |- + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + + "kube-mon-kube-prometheus-stack-1761-k8s.rules.container-memory-cache-31b80a6a-c6e0-47ff-a476-cb2685e628b2.yaml": | + groups: + - name: k8s.rules.container_memory_cache + rules: + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1, + max by (cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + + "kube-mon-kube-prometheus-stack-1761-k8s.rules.container-memory-rss-4e0ac934-42d7-4b4b-9ca6-23abedc3260e.yaml": | + groups: + - name: k8s.rules.container_memory_rss + rules: + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1, + max by (cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + + "kube-mon-kube-prometheus-stack-1761-k8s.rules.container-memory-swap-92e9016e-30e4-4b4d-9b40-75faa82c66d3.yaml": | + groups: + - name: k8s.rules.container_memory_swap + rules: + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1, + max by (cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + + "kube-mon-kube-prometheus-stack-1761-k8s.rules.container-memory-working-s-152e27cf-cf25-4594-ae97-cff59c627b06.yaml": | + groups: + - name: k8s.rules.container_memory_working_set_bytes + rules: + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1, + max by (cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + + "kube-mon-kube-prometheus-stack-1761-k8s.rules.container-resource-8cb14230-2967-447f-8807-f8c6094e3030.yaml": | + groups: + - name: k8s.rules.container_resource + rules: + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + + "kube-mon-kube-prometheus-stack-1761-k8s.rules.pod-owner-e171fc97-e41a-42b9-a9c1-4e56596eb49d.yaml": | + groups: + - name: k8s.rules.pod_owner + rules: + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on (cluster, replicaset, namespace) group_left(owner_name) topk by (cluster, replicaset, namespace) ( + 1, max by (cluster, replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics", owner_kind=""} + ) + ), + "workload", "$1", "replicaset", "(.*)" + ) + ) + labels: + workload_type: replicaset + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on (replicaset, namespace, cluster) group_left(owner_name) topk by (cluster, replicaset, namespace) ( + 1, max by (cluster, replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics", owner_kind="Deployment"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)") + ) + labels: + workload_type: statefulset + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + group by (cluster, namespace, workload, pod) ( + label_join( + group by (cluster, namespace, job_name, pod, owner_name) ( + label_join( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"} + , "job_name", "", "owner_name") + ) + * on (cluster, namespace, job_name) group_left() + group by (cluster, namespace, job_name) ( + kube_job_owner{job="kube-state-metrics", owner_kind=~"Pod|"} + ) + , "workload", "", "owner_name") + ) + labels: + workload_type: job + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="", owner_name=""}, + "workload", "$1", "pod", "(.+)") + ) + labels: + workload_type: barepod + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="Node"}, + "workload", "$1", "pod", "(.+)") + ) + labels: + workload_type: staticpod + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + group by (cluster, namespace, workload, workload_type, pod) ( + label_join( + label_join( + group by (cluster, namespace, job_name, pod) ( + label_join( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"} + , "job_name", "", "owner_name") + ) + * on (cluster, namespace, job_name) group_left(owner_kind, owner_name) + group by (cluster, namespace, job_name, owner_kind, owner_name) ( + kube_job_owner{job="kube-state-metrics", owner_kind!="Pod", owner_kind!=""} + ) + , "workload", "", "owner_name") + , "workload_type", "", "owner_kind") + + OR + + label_replace( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"} + , "replicaset", "$1", "owner_name", "(.+)" + ) + * on (cluster, namespace, replicaset) group_left(owner_kind, owner_name) + group by (cluster, namespace, replicaset, owner_kind, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics", owner_kind!="Deployment", owner_kind!=""} + ) + , "workload", "$1", "owner_name", "(.+)") + OR + label_replace( + group by (cluster, namespace, pod, owner_name, owner_kind) ( + kube_pod_owner{job="kube-state-metrics", owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""} + ) + , "workload", "$1", "owner_name", "(.+)" + ) + , "workload_type", "$1", "owner_kind", "(.+)") + ) + record: namespace_workload_pod:kube_pod_owner:relabel + + "kube-mon-kube-prometheus-stack-1761-kube-apiserver-availability.rules-f8e8efaa-dc08-4d49-be19-8ac2530ddfbe.yaml": | + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * + 30 + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + record: code:apiserver_request_total:increase30d + - expr: sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + record: code:apiserver_request_total:increase30d + - expr: sum by (cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h])) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h + - expr: sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) + * 24 * 30) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d + - expr: sum by (cluster, verb, scope) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h{le="+Inf"}) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h + - expr: sum by (cluster, verb, scope) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{le="+Inf"}) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d + - expr: |- + 1 - ( + ( + # write too slow + sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le=~"1(\\.0)?"} or vector(0)) + ) + + ( + # read too slow + sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le=~"1(\\.0)?"} or vector(0)) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le=~"5(\\.0)?"} or vector(0)) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le=~"30(\\.0)?"} or vector(0)) + ) + ) + + # errors + sum by (cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum by (cluster) (code:apiserver_request_total:increase30d) + labels: + verb: all + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + # too slow + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le=~"1(\\.0)?"} or vector(0)) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le=~"5(\\.0)?"} or vector(0)) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le=~"30(\\.0)?"} or vector(0)) + ) + + + # errors + sum by (cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum by (cluster) (code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le=~"1(\\.0)?"} or vector(0)) + ) + + + # errors + sum by (cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum by (cluster) (code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + record: apiserver_request:availability30d + - expr: sum by (cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + record: code_resource:apiserver_request_total:rate5m + - expr: sum by (cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + record: code_resource:apiserver_request_total:rate5m + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + + "kube-mon-kube-prometheus-stack-1761-kube-apiserver-burnrate.rules-93b05b40-1d3d-42e4-ab29-191ef2f26b37.yaml": | + groups: + - name: kube-apiserver-burnrate.rules + rules: + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[1d])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[1d])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + labels: + verb: read + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[1h])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[1h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + labels: + verb: read + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[2h])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[2h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + labels: + verb: read + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[30m])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[30m])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + labels: + verb: read + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[3d])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[3d])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + labels: + verb: read + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[5m])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[5m])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le=~"1(\\.0)?"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le=~"5(\\.0)?"}[6h])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le=~"30(\\.0)?"}[6h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + labels: + verb: read + record: apiserver_request:burnrate6h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[1d])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + labels: + verb: write + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[1h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + labels: + verb: write + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[2h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + labels: + verb: write + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[30m])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + labels: + verb: write + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[3d])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + labels: + verb: write + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[5m])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le=~"1(\\.0)?"}[6h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + labels: + verb: write + record: apiserver_request:burnrate6h + + "kube-mon-kube-prometheus-stack-1761-kube-apiserver-histogram.rules-200e8bdf-3d72-43da-8446-ca9ac60d3e66.yaml": | + groups: + - name: kube-apiserver-histogram.rules + rules: + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) + > 0 + labels: + quantile: "0.99" + verb: read + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) + > 0 + labels: + quantile: "0.99" + verb: write + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + + "kube-mon-kube-prometheus-stack-1761-kube-apiserver-slos-e494c35c-71bd-4575-9bdc-a00fe3a0fc21.yaml": | + groups: + - name: kube-apiserver-slos + rules: + - alert: KubeAPIErrorBudgetBurn + annotations: + description: The API server is burning too much error budget on cluster {{ $labels.cluster + }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum by (cluster) (apiserver_request:burnrate1h) > (14.40 * 0.01000) + and on (cluster) + sum by (cluster) (apiserver_request:burnrate5m) > (14.40 * 0.01000) + for: 2m + labels: + long: 1h + severity: critical + short: 5m + - alert: KubeAPIErrorBudgetBurn + annotations: + description: The API server is burning too much error budget on cluster {{ $labels.cluster + }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum by (cluster) (apiserver_request:burnrate6h) > (6.00 * 0.01000) + and on (cluster) + sum by (cluster) (apiserver_request:burnrate30m) > (6.00 * 0.01000) + for: 15m + labels: + long: 6h + severity: critical + short: 30m + - alert: KubeAPIErrorBudgetBurn + annotations: + description: The API server is burning too much error budget on cluster {{ $labels.cluster + }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum by (cluster) (apiserver_request:burnrate1d) > (3.00 * 0.01000) + and on (cluster) + sum by (cluster) (apiserver_request:burnrate2h) > (3.00 * 0.01000) + for: 1h + labels: + long: 1d + severity: warning + short: 2h + - alert: KubeAPIErrorBudgetBurn + annotations: + description: The API server is burning too much error budget on cluster {{ $labels.cluster + }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum by (cluster) (apiserver_request:burnrate3d) > (1.00 * 0.01000) + and on (cluster) + sum by (cluster) (apiserver_request:burnrate6h) > (1.00 * 0.01000) + for: 3h + labels: + long: 3d + severity: warning + short: 6h + + "kube-mon-kube-prometheus-stack-1761-kube-prometheus-general.rules-1e36e323-67b2-4d05-89e0-7a8f53a20695.yaml": | + groups: + - name: kube-prometheus-general.rules + rules: + - expr: count without(instance, pod, node) (up == 1) + record: count:up1 + - expr: count without(instance, pod, node) (up == 0) + record: count:up0 + + "kube-mon-kube-prometheus-stack-1761-kube-prometheus-node-recording.rules-273c2bf2-6448-43b0-b4bf-e36a19808388.yaml": | + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) + BY (instance) + record: instance:node_cpu:rate:sum + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) + record: instance:node_network_receive_bytes:rate:sum + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) + record: instance:node_network_transmit_bytes:rate:sum + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) + WITHOUT (cpu, mode) / ON (instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) + BY (instance, cpu)) BY (instance) + record: instance:node_cpu:ratio + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) + record: cluster:node_cpu:sum_rate5m + - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, + cpu)) + record: cluster:node_cpu:ratio + + "kube-mon-kube-prometheus-stack-1761-kube-scheduler.rules-bd7fd06f-73ca-45a2-a56a-be1cca5c57d4.yaml": | + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) + without(instance, pod)) + labels: + quantile: "0.99" + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) + without(instance, pod)) + labels: + quantile: "0.99" + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) + without(instance, pod)) + labels: + quantile: "0.99" + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) + without(instance, pod)) + labels: + quantile: "0.9" + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) + without(instance, pod)) + labels: + quantile: "0.9" + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) + without(instance, pod)) + labels: + quantile: "0.9" + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) + without(instance, pod)) + labels: + quantile: "0.5" + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) + without(instance, pod)) + labels: + quantile: "0.5" + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) + without(instance, pod)) + labels: + quantile: "0.5" + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + + "kube-mon-kube-prometheus-stack-1761-kube-state-metrics-8197c349-0c10-4bdd-b377-33a11765a448.yaml": | + groups: + - name: kube-state-metrics + rules: + - alert: KubeStateMetricsListErrors + annotations: + description: kube-state-metrics is experiencing errors at an elevated rate in + list operations. This is likely causing it to not be able to expose metrics + about Kubernetes objects correctly or at all. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricslisterrors + summary: kube-state-metrics is experiencing errors in list operations. + expr: |- + (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) by (cluster) + / + sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])) by (cluster)) + > 0.01 + for: 15m + labels: + severity: critical + - alert: KubeStateMetricsWatchErrors + annotations: + description: kube-state-metrics is experiencing errors at an elevated rate in + watch operations. This is likely causing it to not be able to expose metrics + about Kubernetes objects correctly or at all. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricswatcherrors + summary: kube-state-metrics is experiencing errors in watch operations. + expr: |- + (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) by (cluster) + / + sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])) by (cluster)) + > 0.01 + for: 15m + labels: + severity: critical + - alert: KubeStateMetricsShardingMismatch + annotations: + description: kube-state-metrics pods are running with different --total-shards + configuration, some Kubernetes objects may be exposed multiple times or not + exposed at all. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardingmismatch + summary: kube-state-metrics sharding is misconfigured. + expr: stdvar (kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) + != 0 + for: 15m + labels: + severity: critical + - alert: KubeStateMetricsShardsMissing + annotations: + description: kube-state-metrics shards are missing, some Kubernetes objects + are not being exposed. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardsmissing + summary: kube-state-metrics shards are missing. + expr: |- + 2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) - 1 + - + sum( 2 ^ max by (cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) ) by (cluster) + != 0 + for: 15m + labels: + severity: critical + + "kube-mon-kube-prometheus-stack-1761-kubelet.rules-5550b0b4-a44a-4602-989a-e87c58ea73ae.yaml": | + groups: + - name: kubelet.rules + rules: + - expr: |- + histogram_quantile( + 0.99, + sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le) + * on (cluster, instance) group_left (node) + max by (cluster, instance, node) (kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + ) + labels: + quantile: "0.99" + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: |- + histogram_quantile( + 0.9, + sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le) + * on (cluster, instance) group_left (node) + max by (cluster, instance, node) (kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + ) + labels: + quantile: "0.9" + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: |- + histogram_quantile( + 0.5, + sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le) + * on (cluster, instance) group_left (node) + max by (cluster, instance, node) (kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + ) + labels: + quantile: "0.5" + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + + "kube-mon-kube-prometheus-stack-1761-kubernetes-apps-1ff209d8-4a23-4a49-9c38-2753b7e7b8d9.yaml": | + groups: + - name: kubernetes-apps + rules: + - alert: KubePodCrashLooping + annotations: + description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container + }}) is in waiting state (reason: "CrashLoopBackOff") on cluster {{ $labels.cluster + }}.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping + summary: Pod is crash looping. + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", + job="kube-state-metrics", namespace=~".*"}[5m]) >= 1 + for: 15m + labels: + severity: warning + - alert: KubePodNotReady + annotations: + description: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready + state for longer than 15 minutes on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready + summary: Pod has been in a non-ready state for more than 15 minutes. + expr: |- + sum by (namespace, pod, cluster) ( + max by (namespace, pod, cluster) ( + kube_pod_status_phase{job="kube-state-metrics", namespace=~".*", phase=~"Pending|Unknown|Failed"} + ) * on (namespace, pod, cluster) group_left(owner_kind) topk by (namespace, pod, cluster) ( + 1, max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: 15m + labels: + severity: warning + - alert: KubeDeploymentGenerationMismatch + annotations: + description: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment + }} does not match, this indicates that the Deployment has failed but has not + been rolled back on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentgenerationmismatch + summary: Deployment generation mismatch due to possible roll-back + expr: |- + kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~".*"} + != + kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~".*"} + for: 15m + labels: + severity: warning + - alert: KubeDeploymentReplicasMismatch + annotations: + description: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has + not matched the expected number of replicas for longer than 15 minutes on + cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~".*"} + > + kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~".*"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m]) + == + 0 + ) + for: 15m + labels: + severity: warning + - alert: KubeDeploymentRolloutStuck + annotations: + description: Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment + }} is not progressing for longer than 15 minutes on cluster {{ $labels.cluster + }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentrolloutstuck + summary: Deployment rollout is not progressing. + expr: |- + kube_deployment_status_condition{condition="Progressing", status="false",job="kube-state-metrics", namespace=~".*"} + != 0 + for: 15m + labels: + severity: warning + - alert: KubeStatefulSetReplicasMismatch + annotations: + description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has + not matched the expected number of replicas for longer than 15 minutes on + cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetreplicasmismatch + summary: StatefulSet has not matched the expected number of replicas. + expr: |- + ( + kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~".*"} + != + kube_statefulset_replicas{job="kube-state-metrics", namespace=~".*"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m]) + == + 0 + ) + for: 15m + labels: + severity: warning + - alert: KubeStatefulSetGenerationMismatch + annotations: + description: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset + }} does not match, this indicates that the StatefulSet has failed but has + not been rolled back on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetgenerationmismatch + summary: StatefulSet generation mismatch due to possible roll-back + expr: |- + kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~".*"} + != + kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~".*"} + for: 15m + labels: + severity: warning + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: + description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update + has not been rolled out on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout + summary: StatefulSet update has not been rolled out. + expr: |- + ( + max by (namespace, statefulset, job, cluster) ( + kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~".*"} + unless + kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~".*"} + ) + * on (namespace, statefulset, job, cluster) + ( + kube_statefulset_replicas{job="kube-state-metrics", namespace=~".*"} + != + kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"} + ) + ) and on (namespace, statefulset, job, cluster) ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[5m]) + == + 0 + ) + for: 15m + labels: + severity: warning + - alert: KubeDaemonSetRolloutStuck + annotations: + description: DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not + finished or progressed for at least 15m on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck + summary: DaemonSet rollout is stuck. + expr: |- + ( + ( + kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"} + ) or ( + kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"} + != + 0 + ) or ( + kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"} + ) or ( + kube_daemonset_status_number_available{job="kube-state-metrics", namespace=~".*"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"} + ) + ) and ( + changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"}[5m]) + == + 0 + ) + for: 15m + labels: + severity: warning + - alert: KubeContainerWaiting + annotations: + description: 'pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on + container {{ $labels.container}} has been in waiting state for longer than + 1 hour. (reason: "{{ $labels.reason }}") on cluster {{ $labels.cluster }}.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting + summary: Pod container waiting longer than 1 hour + expr: kube_pod_container_status_waiting_reason{reason!="CrashLoopBackOff", job="kube-state-metrics", + namespace=~".*"} > 0 + for: 1h + labels: + severity: warning + - alert: KubeDaemonSetNotScheduled + annotations: + description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset + }} are not scheduled on cluster {{ $labels.cluster }}.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetnotscheduled + summary: DaemonSet pods are not scheduled. + expr: |- + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"} + - + kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"} > 0 + for: 10m + labels: + severity: warning + - alert: KubeDaemonSetMisScheduled + annotations: + description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset + }} are running where they are not supposed to run on cluster {{ $labels.cluster + }}.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetmisscheduled + summary: DaemonSet pods are misscheduled. + expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"} + > 0 + for: 15m + labels: + severity: warning + - alert: KubeJobNotCompleted + annotations: + description: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more + than {{ "43200" | humanizeDuration }} to complete on cluster {{ $labels.cluster + }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted + summary: Job did not complete in time + expr: |- + time() - max by (namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics", namespace=~".*"} + and + kube_job_status_active{job="kube-state-metrics", namespace=~".*"} > 0) > 43200 + labels: + severity: warning + - alert: KubeJobFailed + annotations: + description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. + Removing failed job after investigation should clear this alert on cluster + {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed + summary: Job failed to complete. + expr: kube_job_failed{job="kube-state-metrics", namespace=~".*"} > 0 + for: 15m + labels: + severity: warning + - alert: KubeHpaReplicasMismatch + annotations: + description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} + has not matched the desired number of replicas for longer than 15 minutes + on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpareplicasmismatch + summary: HPA has not matched desired number of replicas. + expr: |- + (kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics", namespace=~".*"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics", namespace=~".*"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}[15m]) == 0 + for: 15m + labels: + severity: warning + - alert: KubeHpaMaxedOut + annotations: + description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} + has been running at max replicas for longer than 15 minutes on cluster {{ + $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpamaxedout + summary: HPA is running at max replicas + expr: |- + kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"} + for: 15m + labels: + severity: warning + - alert: KubePdbNotEnoughHealthyPods + annotations: + description: PDB {{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.poddisruptionbudget + }} expects {{ $value }} more healthy pods. The desired number of healthy pods + has not been met for at least 15m. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepdbnotenoughhealthypods + summary: PDB does not have enough healthy pods. + expr: |- + ( + kube_poddisruptionbudget_status_desired_healthy{job="kube-state-metrics", namespace=~".*"} + - + kube_poddisruptionbudget_status_current_healthy{job="kube-state-metrics", namespace=~".*"} + ) + > 0 + for: 15m + labels: + severity: warning + + "kube-mon-kube-prometheus-stack-1761-kubernetes-resources-93713bfc-63ee-4f4d-83c7-fedafb90dcae.yaml": | + groups: + - name: kubernetes-resources + rules: + - alert: KubeCPUOvercommit + annotations: + description: Cluster {{ $labels.cluster }} has overcommitted CPU resource requests + for Pods by {{ printf "%.2f" $value }} CPU shares and cannot tolerate node + failure. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + # Non-HA clusters. + ( + ( + sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum{}) + - + sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) > 0 + ) + and + count by (cluster) (max by (cluster, node) (kube_node_role{job="kube-state-metrics", role="control-plane"})) < 3 + ) + or + # HA clusters. + ( + sum by (cluster) (namespace_cpu:kube_pod_container_resource_requests:sum{}) + - + ( + # Skip clusters with only one allocatable node. + ( + sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) + - + max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) + ) > 0 + ) > 0 + ) + for: 10m + labels: + severity: warning + - alert: KubeMemoryOvercommit + annotations: + description: Cluster {{ $labels.cluster }} has overcommitted memory resource + requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node + failure. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + # Non-HA clusters. + ( + ( + sum by (cluster) (namespace_memory:kube_pod_container_resource_requests:sum{}) + - + sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"}) > 0 + ) + and + count by (cluster) (max by (cluster, node) (kube_node_role{job="kube-state-metrics", role="control-plane"})) < 3 + ) + or + # HA clusters. + ( + sum by (cluster) (namespace_memory:kube_pod_container_resource_requests:sum{}) + - + ( + # Skip clusters with only one allocatable node. + ( + sum by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"}) + - + max by (cluster) (kube_node_status_allocatable{job="kube-state-metrics",resource="memory"}) + ) > 0 + ) > 0 + ) + for: 10m + labels: + severity: warning + - alert: KubeCPUQuotaOvercommit + annotations: + description: Cluster {{ $labels.cluster }} has overcommitted CPU resource requests + for Namespaces. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum by (cluster) ( + min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"}) + ) + / + sum by (cluster) ( + kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"} + ) > 1.5 + for: 5m + labels: + severity: warning + - alert: KubeMemoryQuotaOvercommit + annotations: + description: Cluster {{ $labels.cluster }} has overcommitted memory resource + requests for Namespaces. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum by (cluster) ( + min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"}) + ) + / + sum by (cluster) ( + kube_node_status_allocatable{resource="memory", job="kube-state-metrics"} + ) > 1.5 + for: 5m + labels: + severity: warning + - alert: KubeQuotaAlmostFull + annotations: + description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage + }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaalmostfull + summary: Namespace quota is going to be full. + expr: |- + kube_resourcequota{job="kube-state-metrics", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) + > 0.9 < 1 + for: 15m + labels: + severity: info + - alert: KubeQuotaFullyUsed + annotations: + description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage + }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotafullyused + summary: Namespace quota is fully used. + expr: |- + kube_resourcequota{job="kube-state-metrics", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) + == 1 + for: 15m + labels: + severity: info + - alert: KubeQuotaExceeded + annotations: + description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage + }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaexceeded + summary: Namespace quota has exceeded the limits. + expr: |- + kube_resourcequota{job="kube-state-metrics", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) + > 1 + for: 15m + labels: + severity: warning + - alert: CPUThrottlingHigh + annotations: + description: '{{ $value | humanizePercentage }} throttling of CPU in namespace + {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod + }} on cluster {{ $labels.cluster }}.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh + summary: Processes experience elevated CPU throttling. + expr: |- + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", job="kubelet", metrics_path="/metrics/cadvisor", }[5m])) without (id, metrics_path, name, image, endpoint, job, node) + / on (cluster, namespace, pod, container, instance) group_left + sum(increase(container_cpu_cfs_periods_total{job="kubelet", metrics_path="/metrics/cadvisor", }[5m])) without (id, metrics_path, name, image, endpoint, job, node) + > ( 25 / 100 ) + for: 15m + labels: + severity: info + + "kube-mon-kube-prometheus-stack-1761-kubernetes-storage-44be9212-edc5-4094-853a-cb73b4d505e2.yaml": | + groups: + - name: kubernetes-storage + rules: + - alert: KubePersistentVolumeFillingUp + annotations: + description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim + }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster + {{ . }} {{- end }} is only {{ $value | humanizePercentage }} free. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_used_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0 + unless on (cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on (cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1m + labels: + severity: critical + - alert: KubePersistentVolumeFillingUp + annotations: + description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim + }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster + {{ . }} {{- end }} is expected to fill up within four days. Currently {{ $value + | humanizePercentage }} is available. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on (cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on (cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1h + labels: + severity: warning + - alert: KubePersistentVolumeInodesFillingUp + annotations: + description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim + }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster + {{ . }} {{- end }} only has {{ $value | humanizePercentage }} free inodes. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="kubelet", namespace=~".*", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_inodes_used{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0 + unless on (cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on (cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1m + labels: + severity: critical + - alert: KubePersistentVolumeInodesFillingUp + annotations: + description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim + }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster + {{ . }} {{- end }} is expected to run out of inodes within four days. Currently + {{ $value | humanizePercentage }} of its inodes are free. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="kubelet", namespace=~".*", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_inodes_used{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on (cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on (cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1h + labels: + severity: warning + - alert: KubePersistentVolumeErrors + annotations: + description: The persistent volume {{ $labels.persistentvolume }} {{ with $labels.cluster + -}} on Cluster {{ . }} {{- end }} has status {{ $labels.phase }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeerrors + summary: PersistentVolume is having issues with provisioning. + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} + > 0 + for: 5m + labels: + severity: critical + + "kube-mon-kube-prometheus-stack-1761-kubernetes-system-apiserver-5c004ea0-4dd4-4670-ab90-8c5c11efb4a9.yaml": | + groups: + - name: kubernetes-system-apiserver + rules: + - alert: KubeClientCertificateExpiration + annotations: + description: A client certificate used to authenticate to kubernetes apiserver + is expiring in less than 7.0 days on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: |- + histogram_quantile(0.01, sum without (namespace, service, endpoint) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + and + on (job, cluster, instance) apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 + for: 5m + labels: + severity: warning + - alert: KubeClientCertificateExpiration + annotations: + description: A client certificate used to authenticate to kubernetes apiserver + is expiring in less than 24.0 hours on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: |- + histogram_quantile(0.01, sum without (namespace, service, endpoint) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + and + on (job, cluster, instance) apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 + for: 5m + labels: + severity: critical + - alert: KubeAggregatedAPIErrors + annotations: + description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name + }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster + }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. + expr: sum by (cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) + > 0 + for: 10m + labels: + severity: warning + - alert: KubeAggregatedAPIDown + annotations: + description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace + }} has been only {{ $value | humanize }}% available over the last 10m on cluster + {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. + expr: (1 - max by (name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) + * 100 < 85 + for: 5m + labels: + severity: warning + - alert: KubeAPIDown + annotations: + description: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapidown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="apiserver"} == 1) + for: 15m + labels: + severity: critical + - alert: KubeAPITerminatedRequests + annotations: + description: The kubernetes apiserver has terminated {{ $value | humanizePercentage + }} of its incoming requests on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests + summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage + }} of its incoming requests. + expr: sum by (cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) + / ( sum by (cluster) (rate(apiserver_request_total{job="apiserver"}[10m])) + + sum by (cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) + ) > 0.20 + for: 5m + labels: + severity: warning + + "kube-mon-kube-prometheus-stack-1761-kubernetes-system-bcfa5d6a-96ab-4a9e-9263-092f214f0415.yaml": | + groups: + - name: kubernetes-system + rules: + - alert: KubeVersionMismatch + annotations: + description: There are {{ $value }} different semantic versions of Kubernetes + components running on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeversionmismatch + summary: Different semantic versions of Kubernetes components running. + expr: count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) + > 1 + for: 15m + labels: + severity: warning + - alert: KubeClientErrors + annotations: + description: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance + }}' is experiencing {{ $value | humanizePercentage }} errors on cluster {{ + $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclienterrors + summary: Kubernetes API server client is experiencing errors. + expr: |- + (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by (cluster, instance, job, namespace) + / + sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by (cluster, instance, job, namespace)) + > 0.01 + for: 15m + labels: + severity: warning + + "kube-mon-kube-prometheus-stack-1761-kubernetes-system-controller-manager-0ebd353f-4ae6-4f5b-98e5-6c781a041d24.yaml": | + groups: + - name: kubernetes-system-controller-manager + rules: + - alert: KubeControllerManagerDown + annotations: + description: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontrollermanagerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="kube-controller-manager"} == 1) + for: 15m + labels: + severity: critical + + "kube-mon-kube-prometheus-stack-1761-kubernetes-system-kube-proxy-ff165a90-0461-487e-803d-c52765f16c74.yaml": | + groups: + - name: kubernetes-system-kube-proxy + rules: + - alert: KubeProxyDown + annotations: + description: KubeProxy has disappeared from Prometheus target discovery. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeproxydown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="kube-proxy"} == 1) + for: 15m + labels: + severity: critical + + "kube-mon-kube-prometheus-stack-1761-kubernetes-system-kubelet-01a4fc74-17a4-40e3-a4d0-bced6a05fbdb.yaml": | + groups: + - name: kubernetes-system-kubelet + rules: + - alert: KubeNodeNotReady + annotations: + description: '{{ $labels.node }} has been unready for more than 15 minutes on + cluster {{ $labels.cluster }}.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodenotready + summary: Node is not ready. + expr: |- + kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0 + and on (cluster, node) + kube_node_spec_unschedulable{job="kube-state-metrics"} == 0 + for: 15m + labels: + severity: warning + - alert: KubeNodePressure + annotations: + description: '{{ $labels.node }} on cluster {{ $labels.cluster }} has active + Condition {{ $labels.condition }}. This is caused by resource usage exceeding + eviction thresholds.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodepressure + summary: Node has as active Condition. + expr: |- + kube_node_status_condition{job="kube-state-metrics",condition=~"(MemoryPressure|DiskPressure|PIDPressure)",status="true"} == 1 + and on (cluster, node) + kube_node_spec_unschedulable{job="kube-state-metrics"} == 0 + for: 10m + labels: + severity: info + - alert: KubeNodeUnreachable + annotations: + description: '{{ $labels.node }} is unreachable and some workloads may be rescheduled + on cluster {{ $labels.cluster }}.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodeunreachable + summary: Node is unreachable. + expr: (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} + unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) + == 1 + for: 15m + labels: + severity: warning + - alert: KubeletTooManyPods + annotations: + description: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage + }} of its Pod capacity on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubelettoomanypods + summary: Kubelet is running at capacity. + expr: |- + ( + max by (cluster, instance) ( + kubelet_running_pods{job="kubelet", metrics_path="/metrics"} > 1 + ) + * on (cluster, instance) group_left(node) + max by (cluster, instance, node) ( + kubelet_node_name{job="kubelet", metrics_path="/metrics"} + ) + ) + / on (cluster, node) group_left() + max by (cluster, node) ( + kube_node_status_capacity{job="kube-state-metrics", resource="pods"} != 1 + ) > 0.95 + for: 15m + labels: + severity: info + - alert: KubeNodeReadinessFlapping + annotations: + description: The readiness status of node {{ $labels.node }} has changed {{ + $value }} times in the last 15 minutes on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping + summary: Node readiness status is flapping. + expr: |- + sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready"}[15m])) by (cluster, node) > 2 + and on (cluster, node) + kube_node_spec_unschedulable{job="kube-state-metrics"} == 0 + for: 15m + labels: + severity: warning + - alert: KubeNodeEviction + annotations: + description: Node {{ $labels.node }} on {{ $labels.cluster }} is evicting Pods + due to {{ $labels.eviction_signal }}. Eviction occurs when eviction thresholds + are crossed, typically caused by Pods exceeding RAM/ephemeral-storage limits. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodeeviction + summary: Node is evicting pods. + expr: |- + sum(rate(kubelet_evictions{job="kubelet", metrics_path="/metrics"}[15m])) by (cluster, eviction_signal, instance) + * on (cluster, instance) group_left(node) + max by (cluster, instance, node) ( + kubelet_node_name{job="kubelet", metrics_path="/metrics"} + ) + > 0 + for: 0s + labels: + severity: info + - alert: KubeletPlegDurationHigh + annotations: + description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile + duration of {{ $value }} seconds on node {{ $labels.node }} on cluster {{ + $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletplegdurationhigh + summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist. + expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} + >= 10 + for: 5m + labels: + severity: warning + - alert: KubeletPodStartUpLatencyHigh + annotations: + description: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds + on node {{ $labels.node }} on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletpodstartuplatencyhigh + summary: Kubelet Pod startup latency is too high. + expr: |- + histogram_quantile(0.99, + sum by (cluster, instance, le) ( + topk by (cluster, instance, le, operation_type) (1, + rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m]) + ) + ) + ) + * on (cluster, instance) group_left(node) + topk by (cluster, instance, node) (1, + kubelet_node_name{job="kubelet", metrics_path="/metrics"} + ) + > 60 + for: 15m + labels: + severity: warning + - alert: KubeletClientCertificateExpiration + annotations: + description: Client certificate for Kubelet on node {{ $labels.node }} expires + in {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 604800 + labels: + severity: warning + - alert: KubeletClientCertificateExpiration + annotations: + description: Client certificate for Kubelet on node {{ $labels.node }} expires + in {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 86400 + labels: + severity: critical + - alert: KubeletServerCertificateExpiration + annotations: + description: Server certificate for Kubelet on node {{ $labels.node }} expires + in {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 604800 + labels: + severity: warning + - alert: KubeletServerCertificateExpiration + annotations: + description: Server certificate for Kubelet on node {{ $labels.node }} expires + in {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 86400 + labels: + severity: critical + - alert: KubeletClientCertificateRenewalErrors + annotations: + description: Kubelet on node {{ $labels.node }} has failed to renew its client + certificate ({{ $value | humanize }} errors in the last 5 minutes) on cluster + {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificaterenewalerrors + summary: Kubelet has failed to renew its client certificate. + expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) + > 0 + for: 15m + labels: + severity: warning + - alert: KubeletServerCertificateRenewalErrors + annotations: + description: Kubelet on node {{ $labels.node }} has failed to renew its server + certificate ({{ $value | humanize }} errors in the last 5 minutes) on cluster + {{ $labels.cluster }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificaterenewalerrors + summary: Kubelet has failed to renew its server certificate. + expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0 + for: 15m + labels: + severity: warning + - alert: KubeletDown + annotations: + description: Kubelet has disappeared from Prometheus target discovery. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="kubelet", metrics_path="/metrics"} == 1) + for: 15m + labels: + severity: critical + + "kube-mon-kube-prometheus-stack-1761-kubernetes-system-scheduler-d8f0b9d8-1cd8-4e7b-abff-19c637dbeae0.yaml": | + groups: + - name: kubernetes-system-scheduler + rules: + - alert: KubeSchedulerDown + annotations: + description: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeschedulerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="kube-scheduler"} == 1) + for: 15m + labels: + severity: critical + + "kube-mon-kube-prometheus-stack-1761-node-exporter-08ab1b90-11d6-4be2-a0d4-4db5ea20393f.yaml": | + groups: + - name: node-exporter + rules: + - alert: NodeFilesystemSpaceFillingUp + annotations: + description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint + }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available + space left and is filling up. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 24 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: 1h + labels: + severity: warning + - alert: NodeFilesystemSpaceFillingUp + annotations: + description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint + }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available + space left and is filling up fast. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 4 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: 1h + labels: + severity: critical + - alert: NodeFilesystemAlmostOutOfSpace + annotations: + description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint + }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available + space left. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 5% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: 30m + labels: + severity: warning + - alert: NodeFilesystemAlmostOutOfSpace + annotations: + description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint + }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available + space left. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 3% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: 30m + labels: + severity: critical + - alert: NodeFilesystemFilesFillingUp + annotations: + description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint + }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available + inodes left and is filling up. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 24 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: 1h + labels: + severity: warning + - alert: NodeFilesystemFilesFillingUp + annotations: + description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint + }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available + inodes left and is filling up fast. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 4 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: 1h + labels: + severity: critical + - alert: NodeFilesystemAlmostOutOfFiles + annotations: + description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint + }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available + inodes left. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 5% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: 1h + labels: + severity: warning + - alert: NodeFilesystemAlmostOutOfFiles + annotations: + description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint + }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available + inodes left. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 3% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: 1h + labels: + severity: critical + - alert: NodeNetworkReceiveErrs + annotations: + description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered + {{ printf "%.0f" $value }} receive errors in the last two minutes.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworkreceiveerrs + summary: Network interface is reporting many receive errors. + expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) + > 0.01 + for: 1h + labels: + severity: warning + - alert: NodeNetworkTransmitErrs + annotations: + description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered + {{ printf "%.0f" $value }} transmit errors in the last two minutes.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworktransmiterrs + summary: Network interface is reporting many transmit errors. + expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) + > 0.01 + for: 1h + labels: + severity: warning + - alert: NodeHighNumberConntrackEntriesUsed + annotations: + description: '{{ $labels.instance }} {{ $value | humanizePercentage }} of conntrack + entries are used.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesused + summary: Number of conntrack are getting close to the limit. + expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) + > 0.75 + labels: + severity: warning + - alert: NodeTextFileCollectorScrapeError + annotations: + description: Node Exporter text file collector on {{ $labels.instance }} failed + to scrape. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: warning + - alert: NodeClockSkewDetected + annotations: + description: Clock at {{ $labels.instance }} is out of sync by more than 0.05s. + Ensure NTP is configured correctly on this host. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetected + summary: Clock skew detected. + expr: |- + ( + node_timex_offset_seconds{job="node-exporter"} > 0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds{job="node-exporter"} < -0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0 + ) + for: 10m + labels: + severity: warning + - alert: NodeClockNotSynchronising + annotations: + description: Clock at {{ $labels.instance }} is not synchronising. Ensure NTP + is configured on this host. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclocknotsynchronising + summary: Clock not synchronising. + expr: |- + min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0 + and + node_timex_maxerror_seconds{job="node-exporter"} >= 16 + for: 10m + labels: + severity: warning + - alert: NodeRAIDDegraded + annotations: + description: RAID array '{{ $labels.device }}' at {{ $labels.instance }} is + in degraded state due to one or more disks failures. Number of spare drives + is insufficient to fix issue automatically. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegraded + summary: RAID Array is degraded. + expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} + - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) + > 0 + for: 15m + labels: + severity: critical + - alert: NodeRAIDDiskFailure + annotations: + description: At least one device in RAID array at {{ $labels.instance }} failed. + Array '{{ $labels.device }}' needs attention and possibly a disk swap. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailure + summary: Failed device in RAID array. + expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} + > 0 + labels: + severity: warning + - alert: NodeFileDescriptorLimit + annotations: + description: File descriptors limit at {{ $labels.instance }} is currently at + {{ printf "%.2f" $value }}%. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70 + ) + for: 15m + labels: + severity: warning + - alert: NodeFileDescriptorLimit + annotations: + description: File descriptors limit at {{ $labels.instance }} is currently at + {{ printf "%.2f" $value }}%. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90 + ) + for: 15m + labels: + severity: critical + - alert: NodeCPUHighUsage + annotations: + description: | + CPU usage at {{ $labels.instance }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodecpuhighusage + summary: High CPU usage. + expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", + mode!~"idle|iowait"}[2m]))) * 100 > 90 + for: 15m + labels: + severity: info + - alert: NodeSystemSaturation + annotations: + description: | + System load per core at {{ $labels.instance }} has been above 2 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}. + This might indicate this instance resources saturation and can cause it becoming unresponsive. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemsaturation + summary: System saturated, load per core is very high. + expr: |- + node_load1{job="node-exporter"} + / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2 + for: 15m + labels: + severity: warning + - alert: NodeMemoryMajorPagesFaults + annotations: + description: | + Memory major pages are occurring at very high rate at {{ $labels.instance }}, 500 major page faults per second for the last 15 minutes, is currently at {{ printf "%.2f" $value }}. + Please check that there is enough memory available at this instance. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememorymajorpagesfaults + summary: Memory major page faults are occurring at very high rate. + expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500 + for: 15m + labels: + severity: warning + - alert: NodeMemoryHighUtilization + annotations: + description: | + Memory is filling up at {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememoryhighutilization + summary: Host is running out of memory. + expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} + * 100) > 90 + for: 15m + labels: + severity: warning + - alert: NodeDiskIOSaturation + annotations: + description: | + Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}. + This symptom might indicate disk saturation. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodediskiosaturation + summary: Disk IO queue is high. + expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + > 10 + for: 30m + labels: + severity: warning + - alert: NodeSystemdServiceFailed + annotations: + description: Systemd service {{ $labels.name }} has entered failed state at + {{ $labels.instance }} + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicefailed + summary: Systemd service has entered failed state. + expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1 + for: 5m + labels: + severity: warning + - alert: NodeSystemdServiceCrashlooping + annotations: + description: Systemd service {{ $labels.name }} has being restarted too many + times at {{ $labels.instance }} for the last 15 minutes. Please check if service + is crash looping. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicecrashlooping + summary: Systemd service keeps restaring, possibly crash looping. + expr: increase(node_systemd_service_restart_total{job="node-exporter"}[5m]) > + 2 + for: 15m + labels: + severity: warning + - alert: NodeBondingDegraded + annotations: + description: Bonding interface {{ $labels.master }} on {{ $labels.instance }} + is in degraded state due to one or more slave failures. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodebondingdegraded + summary: Bonding interface is degraded. + expr: (node_bonding_slaves{job="node-exporter"} - node_bonding_active{job="node-exporter"}) + != 0 + for: 5m + labels: + severity: warning + + "kube-mon-kube-prometheus-stack-1761-node-exporter.rules-f6c67ed3-387a-4bab-bbcd-1caa5ae910b3.yaml": | + groups: + - name: node-exporter.rules + rules: + - expr: |- + count without (cpu, mode) ( + node_cpu_seconds_total{job="node-exporter",mode="idle"} + ) + record: instance:node_num_cpu:sum + - expr: |- + 1 - avg without (cpu) ( + sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m])) + ) + record: instance:node_cpu_utilisation:rate5m + - expr: |- + ( + node_load1{job="node-exporter"} + / + instance:node_num_cpu:sum{job="node-exporter"} + ) + record: instance:node_load1_per_cpu:ratio + - expr: |- + 1 - ( + ( + node_memory_MemAvailable_bytes{job="node-exporter"} + or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + + node_memory_Cached_bytes{job="node-exporter"} + + + node_memory_MemFree_bytes{job="node-exporter"} + + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) + / + node_memory_MemTotal_bytes{job="node-exporter"} + ) + record: instance:node_memory_utilisation:ratio + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) + record: instance:node_vmstat_pgmajfault:rate5m + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_seconds:rate5m + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate5m + - expr: |- + sum without (device) ( + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_bytes_excluding_lo:rate5m + - expr: |- + sum without (device) ( + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_bytes_excluding_lo:rate5m + - expr: |- + sum without (device) ( + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_drop_excluding_lo:rate5m + - expr: |- + sum without (device) ( + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_drop_excluding_lo:rate5m + + "kube-mon-kube-prometheus-stack-1761-node-network-07480cd5-476b-43c2-be6e-554551eaa301.yaml": | + groups: + - name: node-network + rules: + - alert: NodeNetworkInterfaceFlapping + annotations: + description: Network interface "{{ $labels.device }}" changing its up status + often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }} + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/nodenetworkinterfaceflapping + summary: Network interface is often changing its status + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: 2m + labels: + severity: warning + + "kube-mon-kube-prometheus-stack-1761-node.rules-25ee75df-36d2-4be7-a757-57acd6654fa7.yaml": | + groups: + - name: node.rules + rules: + - expr: |- + topk by (cluster, namespace, pod) (1, + max by (cluster, node, namespace, pod) ( + label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)") + )) + record: 'node_namespace_pod:kube_pod_info:' + - expr: |- + count by (cluster, node) ( + node_cpu_seconds_total{mode="idle",job="node-exporter"} + * on (cluster, namespace, pod) group_left(node) + topk by (cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:) + ) + record: node:node_num_cpu:sum + - expr: |- + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by (cluster) + record: :node_memory_MemAvailable_bytes:sum + - expr: |- + avg by (cluster, node) ( + sum without (mode) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m]) + ) + ) + record: node:node_cpu_utilization:ratio_rate5m + - expr: |- + avg by (cluster) ( + node:node_cpu_utilization:ratio_rate5m + ) + record: cluster:node_cpu:ratio_rate5m + + "kube-mon-kube-prometheus-stack-1761-prometheus-5edc6798-1849-44da-b947-7f0b5176e3d7.yaml": | + groups: + - name: prometheus + rules: + - alert: PrometheusBadConfig + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to + reload its configuration. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusbadconfig + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) == 0 + for: 10m + labels: + severity: critical + - alert: PrometheusSDRefreshFailure + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to + refresh SD with mechanism {{$labels.mechanism}}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheussdrefreshfailure + summary: Failed Prometheus SD refresh. + expr: increase(prometheus_sd_refresh_failures_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[10m]) + > 0 + for: 20m + labels: + severity: warning + - alert: PrometheusKubernetesListWatchFailures + annotations: + description: Kubernetes service discovery of Prometheus {{$labels.namespace}}/{{$labels.pod}} + is experiencing {{ printf "%.0f" $value }} failures with LIST/WATCH requests + to the Kubernetes API in the last 5 minutes. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuskuberneteslistwatchfailures + summary: Requests in Kubernetes SD are failing. + expr: increase(prometheus_sd_kubernetes_failures_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + > 0 + for: 15m + labels: + severity: warning + - alert: PrometheusNotificationQueueRunningFull + annotations: + description: Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} + is running full. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotificationqueuerunningfull + summary: Prometheus alert notification queue predicted to run full in less than + 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + ) + for: 15m + labels: + severity: warning + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: + description: '{{ printf "%.1f" $value }}% of alerts sent by Prometheus {{$labels.namespace}}/{{$labels.pod}} + to Alertmanager {{$labels.alertmanager}} were affected by errors.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuserrorsendingalertstosomealertmanagers + summary: More than 1% of alerts sent by Prometheus to a specific Alertmanager + were affected by errors. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + / + rate(prometheus_notifications_sent_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + ) + * 100 + > 1 + for: 15m + labels: + severity: warning + - alert: PrometheusNotConnectedToAlertmanagers + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected + to any Alertmanagers. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotconnectedtoalertmanagers + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) < 1 + for: 10m + labels: + severity: warning + - alert: PrometheusTSDBReloadsFailing + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value + | humanize}} reload failures over the last 3h. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustsdbreloadsfailing + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[3h]) + > 0 + for: 4h + labels: + severity: warning + - alert: PrometheusTSDBCompactionsFailing + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value + | humanize}} compaction failures over the last 3h. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustsdbcompactionsfailing + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[3h]) + > 0 + for: 4h + labels: + severity: warning + - alert: PrometheusNotIngestingSamples + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting + samples. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotingestingsamples + summary: Prometheus is not ingesting samples. + expr: |- + ( + sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m])) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}) > 0 + ) + ) + for: 10m + labels: + severity: warning + - alert: PrometheusDuplicateTimestamps + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ + printf "%.4g" $value }} samples/s with different values but duplicated timestamp. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusduplicatetimestamps + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + > 0 + for: 10m + labels: + severity: warning + - alert: PrometheusOutOfOrderTimestamps + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ + printf "%.4g" $value }} samples/s with timestamps arriving out of order. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusoutofordertimestamps + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + > 0 + for: 10m + labels: + severity: warning + - alert: PrometheusRemoteStorageFailures + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send + {{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{ + $labels.url }} + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotestoragefailures + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + (rate(prometheus_remote_storage_failed_samples_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m])) + / + ( + (rate(prometheus_remote_storage_failed_samples_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m])) + + + (rate(prometheus_remote_storage_succeeded_samples_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) or rate(prometheus_remote_storage_samples_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m])) + ) + ) + * 100 + > 1 + for: 15m + labels: + severity: critical + - alert: PrometheusRemoteWriteBehind + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write is + {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url + }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotewritebehind + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_queue_highest_timestamp_seconds{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + - + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + ) + > 120 + for: 15m + labels: + severity: critical + - alert: PrometheusRemoteWriteDesiredShards + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired + shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ + $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}` + $labels.instance | query | first | value }}. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotewritedesiredshards + summary: Prometheus remote write desired shards calculation wants to run more + than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + ) + for: 15m + labels: + severity: warning + - alert: PrometheusRuleFailures + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to + evaluate {{ printf "%.0f" $value }} rules in the last 5m. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusrulefailures + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + > 0 + for: 15m + labels: + severity: critical + - alert: PrometheusMissingRuleEvaluations + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ + printf "%.0f" $value }} rule group evaluations in the last 5m. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusmissingruleevaluations + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + > 0 + for: 15m + labels: + severity: warning + - alert: PrometheusTargetLimitHit + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ + printf "%.0f" $value }} targets because the number of targets exceeded the + configured target_limit. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustargetlimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded + the targets limit. + expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + > 0 + for: 15m + labels: + severity: warning + - alert: PrometheusLabelLimitHit + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ + printf "%.0f" $value }} targets because some samples exceeded the configured + label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded + the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + > 0 + for: 15m + labels: + severity: warning + - alert: PrometheusScrapeBodySizeLimitHit + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ + printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded + the configured body_size_limit. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + > 0 + for: 15m + labels: + severity: warning + - alert: PrometheusScrapeSampleLimitHit + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ + printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded + the configured sample_limit. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample + limit. + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + > 0 + for: 15m + labels: + severity: warning + - alert: PrometheusTargetSyncFailure + annotations: + description: '{{ printf "%.0f" $value }} targets in Prometheus {{$labels.namespace}}/{{$labels.pod}} + have failed to sync because invalid configuration was supplied.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[30m]) + > 0 + for: 5m + labels: + severity: critical + - alert: PrometheusHighQueryLoad + annotations: + description: Prometheus {{$labels.namespace}}/{{$labels.pod}} query API has + less than 20% available capacity in its query engine for the last 15 minutes. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheushighqueryload + summary: Prometheus is reaching its maximum capacity serving concurrent requests. + expr: avg_over_time(prometheus_engine_queries{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + / max_over_time(prometheus_engine_queries_concurrent_max{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon"}[5m]) + > 0.8 + for: 15m + labels: + severity: warning + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: + description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts + from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="kube-prometheus-stack-1761-prometheus",namespace="kube-mon",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: 15m + labels: + severity: critical + + "kube-mon-kube-prometheus-stack-1761-prometheus-operator-1a5eabbf-3752-4a2f-9913-068f3ff065f3.yaml": | + groups: + - name: prometheus-operator + rules: + - alert: PrometheusOperatorListErrors + annotations: + description: Errors while performing List operations in controller {{$labels.controller}} + in {{$labels.namespace}} namespace. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorlisterrors + summary: Errors while performing list operations in controller. + expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[10m])) + / sum by (cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[10m]))) + > 0.4 + for: 15m + labels: + severity: warning + - alert: PrometheusOperatorWatchErrors + annotations: + description: Errors while performing watch operations in controller {{$labels.controller}} + in {{$labels.namespace}} namespace. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorwatcherrors + summary: Errors while performing watch operations in controller. + expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[5m])) + / sum by (cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[5m]))) + > 0.4 + for: 15m + labels: + severity: warning + - alert: PrometheusOperatorSyncFailed + annotations: + description: Controller {{ $labels.controller }} in {{ $labels.namespace }} + namespace fails to reconcile {{ $value }} objects. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorsyncfailed + summary: Last controller reconciliation failed + expr: min_over_time(prometheus_operator_syncs{status="failed",job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[5m]) + > 0 + for: 10m + labels: + severity: warning + - alert: PrometheusOperatorReconcileErrors + annotations: + description: '{{ $value | humanizePercentage }} of reconciling operations failed + for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorreconcileerrors + summary: Errors while reconciling objects. + expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[5m]))) + / (sum by (cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[5m]))) + > 0.1 + for: 10m + labels: + severity: warning + - alert: PrometheusOperatorStatusUpdateErrors + annotations: + description: '{{ $value | humanizePercentage }} of status update operations + failed for {{ $labels.controller }} controller in {{ $labels.namespace }} + namespace.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorstatusupdateerrors + summary: Errors while updating objects status. + expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[5m]))) + / (sum by (cluster,controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[5m]))) + > 0.1 + for: 10m + labels: + severity: warning + - alert: PrometheusOperatorNodeLookupErrors + annotations: + description: Errors while reconciling Prometheus in {{ $labels.namespace }} + Namespace. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatornodelookuperrors + summary: Errors while reconciling Prometheus. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[5m]) + > 0.1 + for: 10m + labels: + severity: warning + - alert: PrometheusOperatorNotReady + annotations: + description: Prometheus operator in {{ $labels.namespace }} namespace isn't + ready to reconcile {{ $labels.controller }} resources. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatornotready + summary: Prometheus operator not ready + expr: min by (cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[5m]) + == 0) + for: 5m + labels: + severity: warning + - alert: PrometheusOperatorRejectedResources + annotations: + description: Prometheus operator in {{ $labels.namespace }} namespace rejected + {{ printf "%0.0f" $value }} {{ $labels.controller }}/{{ $labels.resource }} + resources. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorrejectedresources + summary: Resources rejected by Prometheus operator + expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="kube-prometheus-stack-1761-operator",namespace="kube-mon"}[5m]) + > 0 + for: 5m + labels: + severity: warning + diff --git a/live-monitoring-deployments/prometheus/03-services.yaml b/live-monitoring-deployments/prometheus/03-services.yaml new file mode 100644 index 0000000..31967d6 --- /dev/null +++ b/live-monitoring-deployments/prometheus/03-services.yaml @@ -0,0 +1,80 @@ +--- +"kind": "Service" +"apiVersion": "v1" +"metadata": + "name": "kube-prometheus-stack-1761-prometheus" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-prometheus" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761819498" + "app.kubernetes.io/managed-by": "Helm" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "heritage": "Helm" + "release": "kube-prometheus-stack-1761819498" + "self-monitor": "true" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" +"spec": + "ports": + - + "name": "http-web" + "protocol": "TCP" + "port": 9090 + "targetPort": 9090 + - + "name": "reloader-web" + "protocol": "TCP" + "appProtocol": "http" + "port": 8080 + "targetPort": "reloader-web" + "selector": + "app.kubernetes.io/name": "prometheus" + "operator.prometheus.io/name": "kube-prometheus-stack-1761-prometheus" + "type": "ClusterIP" + "sessionAffinity": "None" +--- +"kind": "Service" +"apiVersion": "v1" +"metadata": + "name": "prometheus-operated" + "namespace": "kube-mon" + "labels": + "app.kubernetes.io/managed-by": "prometheus-operator" + "managed-by": "prometheus-operator" + "operated-prometheus": "true" +"spec": + "ports": + - + "name": "http-web" + "protocol": "TCP" + "port": 9090 + "targetPort": "http-web" + "selector": + "app.kubernetes.io/name": "prometheus" + "type": "ClusterIP" + "sessionAffinity": "None" + "clusterIP": "None" +--- +"kind": "Service" +"apiVersion": "v1" +"metadata": + "name": "prometheus-operated-nodeport" + "namespace": "kube-mon" + "labels": + "k8slens-edit-resource-version": "v1" +"spec": + "ports": + - + "name": "http-web" + "protocol": "TCP" + "port": 9090 + "targetPort": "http-web" + "nodePort": 31595 + "selector": + "app.kubernetes.io/name": "prometheus" + "type": "NodePort" + "sessionAffinity": "None" + "externalTrafficPolicy": "Cluster" diff --git a/live-monitoring-deployments/prometheus/04-statefulset.yaml b/live-monitoring-deployments/prometheus/04-statefulset.yaml new file mode 100644 index 0000000..fba8b45 --- /dev/null +++ b/live-monitoring-deployments/prometheus/04-statefulset.yaml @@ -0,0 +1,302 @@ +--- +"kind": "StatefulSet" +"apiVersion": "apps/v1" +"metadata": + "name": "prometheus-kube-prometheus-stack-1761-prometheus" + "namespace": "kube-mon" + "labels": + "app": "kube-prometheus-stack-prometheus" + "app.kubernetes.io/instance": "kube-prometheus-stack-1761-prometheus" + "app.kubernetes.io/managed-by": "prometheus-operator" + "app.kubernetes.io/name": "prometheus" + "app.kubernetes.io/part-of": "kube-prometheus-stack" + "app.kubernetes.io/version": "79.0.1" + "chart": "kube-prometheus-stack-79.0.1" + "heritage": "Helm" + "k8slens-edit-resource-version": "v1" + "managed-by": "prometheus-operator" + "operator.prometheus.io/mode": "server" + "operator.prometheus.io/name": "kube-prometheus-stack-1761-prometheus" + "operator.prometheus.io/shard": "0" + "prometheus": "kube-prometheus-stack-1761-prometheus" + "release": "kube-prometheus-stack-1761819498" + "annotations": + "meta.helm.sh/release-name": "kube-prometheus-stack-1761819498" + "meta.helm.sh/release-namespace": "kube-mon" + "prometheus-operator-input-hash": "5891682108210749018" +"spec": + "replicas": 1 + "selector": + "matchLabels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761-prometheus" + "app.kubernetes.io/managed-by": "prometheus-operator" + "app.kubernetes.io/name": "prometheus" + "operator.prometheus.io/name": "kube-prometheus-stack-1761-prometheus" + "operator.prometheus.io/shard": "0" + "prometheus": "kube-prometheus-stack-1761-prometheus" + "template": + "metadata": + "labels": + "app.kubernetes.io/instance": "kube-prometheus-stack-1761-prometheus" + "app.kubernetes.io/managed-by": "prometheus-operator" + "app.kubernetes.io/name": "prometheus" + "app.kubernetes.io/version": "3.7.2" + "operator.prometheus.io/name": "kube-prometheus-stack-1761-prometheus" + "operator.prometheus.io/shard": "0" + "prometheus": "kube-prometheus-stack-1761-prometheus" + "annotations": + "kubectl.kubernetes.io/default-container": "prometheus" + "spec": + "volumes": + - + "name": "config" + "secret": + "secretName": "prometheus-kube-prometheus-stack-1761-prometheus" + "defaultMode": 420 + - + "name": "tls-assets" + "projected": + "sources": + - + "secret": + "name": "prometheus-kube-prometheus-stack-1761-prometheus-tls-assets-0" + "defaultMode": 420 + - + "name": "config-out" + "emptyDir": + "medium": "Memory" + - + "name": "prometheus-kube-prometheus-stack-1761-prometheus-rulefiles-0" + "configMap": + "name": "prometheus-kube-prometheus-stack-1761-prometheus-rulefiles-0" + "defaultMode": 420 + - + "name": "web-config" + "secret": + "secretName": "prometheus-kube-prometheus-stack-1761-prometheus-web-config" + "defaultMode": 420 + "initContainers": + - + "name": "init-config-reloader" + "image": "quay.io/prometheus-operator/prometheus-config-reloader:v0.86.1" + "command": + - "/bin/prometheus-config-reloader" + "args": + - "--watch-interval=0" + - "--listen-address=:8081" + - "--config-file=/etc/prometheus/config/prometheus.yaml.gz" + - "--config-envsubst-file=/etc/prometheus/config_out/prometheus.env.yaml" + - "--watched-dir=/etc/prometheus/rules/prometheus-kube-prometheus-stack-1761-prometheus-rulefiles-0" + "ports": + - + "name": "reloader-init" + "containerPort": 8081 + "protocol": "TCP" + "env": + - + "name": "POD_NAME" + "valueFrom": + "fieldRef": + "apiVersion": "v1" + "fieldPath": "metadata.name" + - + "name": "SHARD" + "value": "0" + "resources": {} + "volumeMounts": + - + "name": "config" + "mountPath": "/etc/prometheus/config" + - + "name": "config-out" + "mountPath": "/etc/prometheus/config_out" + - + "name": "prometheus-kube-prometheus-stack-1761-prometheus-rulefiles-0" + "mountPath": "/etc/prometheus/rules/prometheus-kube-prometheus-stack-1761-prometheus-rulefiles-0" + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "FallbackToLogsOnError" + "imagePullPolicy": "IfNotPresent" + "securityContext": + "capabilities": + "drop": + - "ALL" + "readOnlyRootFilesystem": true + "allowPrivilegeEscalation": false + "containers": + - + "name": "prometheus" + "image": "quay.io/prometheus/prometheus:v3.7.2" + "args": + - "--config.file=/etc/prometheus/config_out/prometheus.env.yaml" + - "--web.enable-lifecycle" + - "--web.external-url=http://kube-prometheus-stack-1761-prometheus.kube-mon:9090" + - "--web.route-prefix=/" + - "--storage.tsdb.retention.time=10d" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.wal-compression" + - "--web.config.file=/etc/prometheus/web_config/web-config.yaml" + "ports": + - + "name": "http-web" + "containerPort": 9090 + "protocol": "TCP" + "resources": {} + "volumeMounts": + - + "name": "config-out" + "readOnly": true + "mountPath": "/etc/prometheus/config_out" + - + "name": "tls-assets" + "readOnly": true + "mountPath": "/etc/prometheus/certs" + - + "name": "prometheus-kube-prometheus-stack-1761-prometheus-db" + "mountPath": "/prometheus" + "subPath": "prometheus-db" + - + "name": "prometheus-kube-prometheus-stack-1761-prometheus-rulefiles-0" + "mountPath": "/etc/prometheus/rules/prometheus-kube-prometheus-stack-1761-prometheus-rulefiles-0" + - + "name": "web-config" + "readOnly": true + "mountPath": "/etc/prometheus/web_config/web-config.yaml" + "subPath": "web-config.yaml" + "livenessProbe": + "httpGet": + "path": "/-/healthy" + "port": "http-web" + "scheme": "HTTP" + "timeoutSeconds": 3 + "periodSeconds": 5 + "successThreshold": 1 + "failureThreshold": 6 + "readinessProbe": + "httpGet": + "path": "/-/ready" + "port": "http-web" + "scheme": "HTTP" + "timeoutSeconds": 3 + "periodSeconds": 5 + "successThreshold": 1 + "failureThreshold": 3 + "startupProbe": + "httpGet": + "path": "/-/ready" + "port": "http-web" + "scheme": "HTTP" + "timeoutSeconds": 3 + "periodSeconds": 15 + "successThreshold": 1 + "failureThreshold": 60 + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "FallbackToLogsOnError" + "imagePullPolicy": "IfNotPresent" + "securityContext": + "capabilities": + "drop": + - "ALL" + "readOnlyRootFilesystem": true + "allowPrivilegeEscalation": false + - + "name": "config-reloader" + "image": "quay.io/prometheus-operator/prometheus-config-reloader:v0.86.1" + "command": + - "/bin/prometheus-config-reloader" + "args": + - "--listen-address=:8080" + - "--reload-url=http://127.0.0.1:9090/-/reload" + - "--config-file=/etc/prometheus/config/prometheus.yaml.gz" + - "--config-envsubst-file=/etc/prometheus/config_out/prometheus.env.yaml" + - "--watched-dir=/etc/prometheus/rules/prometheus-kube-prometheus-stack-1761-prometheus-rulefiles-0" + "ports": + - + "name": "reloader-web" + "containerPort": 8080 + "protocol": "TCP" + "env": + - + "name": "POD_NAME" + "valueFrom": + "fieldRef": + "apiVersion": "v1" + "fieldPath": "metadata.name" + - + "name": "SHARD" + "value": "0" + "resources": {} + "volumeMounts": + - + "name": "config" + "mountPath": "/etc/prometheus/config" + - + "name": "config-out" + "mountPath": "/etc/prometheus/config_out" + - + "name": "prometheus-kube-prometheus-stack-1761-prometheus-rulefiles-0" + "mountPath": "/etc/prometheus/rules/prometheus-kube-prometheus-stack-1761-prometheus-rulefiles-0" + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "FallbackToLogsOnError" + "imagePullPolicy": "IfNotPresent" + "securityContext": + "capabilities": + "drop": + - "ALL" + "readOnlyRootFilesystem": true + "allowPrivilegeEscalation": false + "restartPolicy": "Always" + "terminationGracePeriodSeconds": 600 + "dnsPolicy": "ClusterFirst" + "serviceAccountName": "kube-prometheus-stack-1761-prometheus" + "serviceAccount": "kube-prometheus-stack-1761-prometheus" + "automountServiceAccountToken": true + "shareProcessNamespace": false + "securityContext": + "runAsUser": 1000 + "runAsGroup": 2000 + "runAsNonRoot": true + "fsGroup": 2000 + "seccompProfile": + "type": "RuntimeDefault" + "affinity": + "podAntiAffinity": + "preferredDuringSchedulingIgnoredDuringExecution": + - + "weight": 100 + "podAffinityTerm": + "labelSelector": + "matchExpressions": + - + "key": "app.kubernetes.io/name" + "operator": "In" + "values": + - "prometheus" + - + "key": "app.kubernetes.io/instance" + "operator": "In" + "values": + - "kube-prometheus-stack-1761-prometheus" + "topologyKey": "kubernetes.io/hostname" + "schedulerName": "default-scheduler" + "volumeClaimTemplates": + - + "kind": "PersistentVolumeClaim" + "apiVersion": "v1" + "metadata": + "name": "prometheus-kube-prometheus-stack-1761-prometheus-db" + "spec": + "accessModes": + - "ReadWriteOnce" + "resources": + "requests": + "storage": "50Gi" + "storageClassName": "nfs-csi" + "volumeMode": "Filesystem" + "serviceName": "prometheus-operated" + "podManagementPolicy": "Parallel" + "updateStrategy": + "type": "RollingUpdate" + "revisionHistoryLimit": 10 + "persistentVolumeClaimRetentionPolicy": + "whenDeleted": "Retain" + "whenScaled": "Retain" diff --git a/live-monitoring-deployments/promtail/01-rbac.yaml b/live-monitoring-deployments/promtail/01-rbac.yaml new file mode 100644 index 0000000..445b393 --- /dev/null +++ b/live-monitoring-deployments/promtail/01-rbac.yaml @@ -0,0 +1,39 @@ +--- +"kind": "ServiceAccount" +"apiVersion": "v1" +"metadata": + "name": "promtail" + "namespace": "loki" +--- +"kind": "ClusterRole" +"apiVersion": "rbac.authorization.k8s.io/v1" +"metadata": + "name": "promtail" +"rules": + - + "verbs": + - "get" + - "list" + - "watch" + "apiGroups": + - "" + "resources": + - "nodes" + - "nodes/proxy" + - "services" + - "endpoints" + - "pods" +--- +"kind": "ClusterRoleBinding" +"apiVersion": "rbac.authorization.k8s.io/v1" +"metadata": + "name": "promtail" +"subjects": + - + "kind": "ServiceAccount" + "name": "promtail" + "namespace": "loki" +"roleRef": + "apiGroup": "rbac.authorization.k8s.io" + "kind": "ClusterRole" + "name": "promtail" diff --git a/live-monitoring-deployments/promtail/02-config.yaml b/live-monitoring-deployments/promtail/02-config.yaml new file mode 100644 index 0000000..395d7ba --- /dev/null +++ b/live-monitoring-deployments/promtail/02-config.yaml @@ -0,0 +1,50 @@ +--- +"kind": "ConfigMap" +"apiVersion": "v1" +"metadata": + "name": "promtail-config" + "namespace": "loki" +"data": + "promtail.yaml": | + server: + http_listen_port: 9080 + grpc_listen_port: 0 + + clients: + - url: http://loki.loki.svc.cluster.local:3100/loki/api/v1/push + + positions: + filename: /run/promtail/positions.yaml + + scrape_configs: + - job_name: kubernetes-pods + pipeline_stages: + - cri: {} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: replace + source_labels: [__meta_kubernetes_pod_node_name] + target_label: __host__ + - action: replace + source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + regex: (.+) + target_label: app + - action: replace + source_labels: [__meta_kubernetes_pod_label_app] + regex: (.+) + target_label: app + - action: replace + source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - action: replace + source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - action: replace + source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + - action: replace + source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name] + separator: / + replacement: /var/log/pods/*$1/*.log + target_label: __path__ diff --git a/live-monitoring-deployments/promtail/03-daemonset.yaml b/live-monitoring-deployments/promtail/03-daemonset.yaml new file mode 100644 index 0000000..e2a9c86 --- /dev/null +++ b/live-monitoring-deployments/promtail/03-daemonset.yaml @@ -0,0 +1,84 @@ +--- +"kind": "DaemonSet" +"apiVersion": "apps/v1" +"metadata": + "name": "promtail" + "namespace": "loki" +"spec": + "selector": + "matchLabels": + "app": "promtail" + "template": + "metadata": + "labels": + "app": "promtail" + "spec": + "volumes": + - + "name": "config" + "configMap": + "name": "promtail-config" + "defaultMode": 420 + - + "name": "run" + "hostPath": + "path": "/run/promtail" + "type": "" + - + "name": "pods" + "hostPath": + "path": "/var/log/pods" + "type": "" + "containers": + - + "name": "promtail" + "image": "grafana/promtail:3.4.2" + "args": + - "-config.file=/etc/promtail/promtail.yaml" + - "-config.expand-env=true" + "ports": + - + "name": "http-metrics" + "containerPort": 9080 + "protocol": "TCP" + "env": + - + "name": "HOSTNAME" + "valueFrom": + "fieldRef": + "apiVersion": "v1" + "fieldPath": "spec.nodeName" + "resources": {} + "volumeMounts": + - + "name": "config" + "mountPath": "/etc/promtail" + - + "name": "run" + "mountPath": "/run/promtail" + - + "name": "pods" + "readOnly": true + "mountPath": "/var/log/pods" + "terminationMessagePath": "/dev/termination-log" + "terminationMessagePolicy": "File" + "imagePullPolicy": "IfNotPresent" + "securityContext": + "privileged": true + "runAsUser": 0 + "restartPolicy": "Always" + "terminationGracePeriodSeconds": 30 + "dnsPolicy": "ClusterFirst" + "serviceAccountName": "promtail" + "serviceAccount": "promtail" + "securityContext": {} + "schedulerName": "default-scheduler" + "tolerations": + - + "operator": "Exists" + "updateStrategy": + "type": "RollingUpdate" + "rollingUpdate": + "maxUnavailable": 1 + "maxSurge": 0 + "revisionHistoryLimit": 10 diff --git a/traefik-grafana-dashboard.configmap.yaml b/traefik-grafana-dashboard.configmap.yaml new file mode 100644 index 0000000..7ff3173 --- /dev/null +++ b/traefik-grafana-dashboard.configmap.yaml @@ -0,0 +1,2213 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: traefik-grafana-dashboard + namespace: kube-mon + labels: + grafana_dashboard: "1" + release: "kube-prometheus-stack-1761819498" +data: + traefik-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 30, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Requests/s", + "type": "stat" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code=~\"4..\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "4xx/s", + "type": "stat" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code=~\"5..\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "5xx/s", + "type": "stat" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 250 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "1000 * histogram_quantile(0.95, sum by (le) (rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~\"$entrypoint\"}[$__rate_interval])))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "P95 Latency", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 31, + "panels": [], + "title": "Traffic, Latency & Routing", + "type": "row" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 5, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (entrypoint) (rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\"}[$__rate_interval]))", + "legendFormat": "{{entrypoint}}", + "refId": "A" + } + ], + "title": "Requests by Entrypoint", + "type": "timeseries" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (code) (rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code=~\"[2345]..\"}[$__rate_interval]))", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "title": "Status Codes", + "type": "timeseries" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "1000 * histogram_quantile(0.50, sum by (le) (rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~\"$entrypoint\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "1000 * histogram_quantile(0.95, sum by (le) (rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~\"$entrypoint\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "1000 * histogram_quantile(0.99, sum by (le) (rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~\"$entrypoint\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Latency Percentiles", + "type": "timeseries" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(10, sum by (router) (rate(traefik_router_requests_total{router=~\"$router\"}[$__rate_interval])))", + "legendFormat": "{{router}}", + "refId": "A" + } + ], + "title": "Top Routers by RPS", + "type": "bargauge" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 22 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (entrypoint) (traefik_open_connections{entrypoint=~\"$entrypoint\"})", + "legendFormat": "{{entrypoint}}", + "refId": "A" + } + ], + "title": "Open Connections", + "type": "timeseries" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 8, + "y": 22 + }, + "id": 10, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(traefik_tls_certs_not_after < (time() + 14 * 24 * 3600))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Certs expiring in 14d", + "type": "stat" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 11, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(10, sum by (router) (rate(traefik_router_requests_total{router=~\"$router\",code=~\"5..\"}[$__rate_interval])))", + "legendFormat": "{{router}}", + "refId": "A" + } + ], + "title": "Top Routers by 5xx/s", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 34, + "panels": [], + "title": "Prometheus Deep Dive", + "type": "row" + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 50 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 31 + }, + "id": 35, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "100 * sum(rate(traefik_entrypoint_requests_tls_total{entrypoint=~\"$entrypoint\"}[5m])) / clamp_min(sum(rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\"}[5m])), 0.001)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "TLS Share", + "type": "stat", + "description": "Share of Traefik requests arriving over TLS for the selected entrypoints." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 31 + }, + "id": 36, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_requests_tls_total{entrypoint=~\"$entrypoint\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "TLS Requests/s", + "type": "stat", + "description": "TLS request rate across the selected entrypoints." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1048576 + }, + { + "color": "red", + "value": 10485760 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 31 + }, + "id": 37, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_requests_bytes_total{entrypoint=~\"$entrypoint\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Ingress Traffic", + "type": "stat", + "description": "Request bandwidth entering Traefik." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1048576 + }, + { + "color": "red", + "value": 10485760 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 31 + }, + "id": 38, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(traefik_entrypoint_responses_bytes_total{entrypoint=~\"$entrypoint\"}[5m]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Egress Traffic", + "type": "stat", + "description": "Response bandwidth leaving Traefik." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 31 + }, + "id": 39, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(increase(traefik_config_reloads_total[24h]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Config Reloads 24h", + "type": "stat", + "description": "How often Traefik reloaded its dynamic configuration in the last 24 hours." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 24 + }, + { + "color": "red", + "value": 168 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 31 + }, + "id": 40, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "(time() - max(traefik_config_last_reload_success)) / 3600", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Last Successful Reload Age (h)", + "type": "stat", + "description": "Hours since the last successful configuration reload reported by Traefik." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 35 + }, + "id": 41, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(12, sum by (exported_service) (rate(traefik_service_requests_total{exported_service=~\"$backend_regex\"}[$__rate_interval])))", + "legendFormat": "{{exported_service}}", + "refId": "A" + } + ], + "title": "Top Backends by RPS", + "type": "bargauge", + "description": "Backend services with the highest request rate according to Traefik service metrics." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 35 + }, + "id": 42, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(12, sum by (exported_service) (rate(traefik_service_responses_bytes_total{exported_service=~\"$backend_regex\"}[$__rate_interval])))", + "legendFormat": "{{exported_service}}", + "refId": "A" + } + ], + "title": "Top Backends by Response Throughput", + "type": "bargauge", + "description": "Backend services returning the most response traffic." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 35 + }, + "id": 43, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(12, 1000 * histogram_quantile(0.95, sum by (exported_service, le) (rate(traefik_service_request_duration_seconds_bucket{exported_service=~\"$backend_regex\"}[$__rate_interval]))))", + "legendFormat": "{{exported_service}}", + "refId": "A" + } + ], + "title": "Slowest Backends by P95", + "type": "bargauge", + "description": "Backends with the highest p95 request latency from Traefik's point of view." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 43 + }, + "id": 44, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "topk(12, 1000 * histogram_quantile(0.95, sum by (router, le) (rate(traefik_router_request_duration_seconds_bucket{router=~\"$router\"}[$__rate_interval]))))", + "legendFormat": "{{router}}", + "refId": "A" + } + ], + "title": "Slowest Routers by P95", + "type": "bargauge", + "description": "Routers with the highest p95 latency." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 43 + }, + "id": 45, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "sum by (tls_version) (rate(traefik_service_requests_tls_total{exported_service=~\"$backend_regex\"}[$__rate_interval]))", + "legendFormat": "TLS {{tls_version}}", + "refId": "A" + } + ], + "title": "TLS Requests by Version", + "type": "bargauge", + "description": "Current TLS request split by negotiated TLS version." + }, + { + "datasource": "$prometheus_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 43 + }, + "id": 46, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "bottomk(12, (max by (cn) (traefik_tls_certs_not_after) - time()) / 86400)", + "legendFormat": "{{cn}}", + "refId": "A" + } + ], + "title": "Soonest Certificate Expiry (days)", + "type": "bargauge", + "description": "Certificates with the fewest days remaining before expiry, deduplicated by CN." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 32, + "panels": [], + "title": "Fail2Ban & Ban History", + "type": "row" + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 52 + }, + "id": 14, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*)\" [$__range]))", + "instant": true, + "queryType": "instant", + "refId": "A" + } + ], + "title": "Ban / Extend Events", + "type": "stat", + "description": "Log-based count of Fail2Ban ban or ban-extension events in the selected time range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 52 + }, + "id": 15, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(sum by (ip) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*)\" | regexp \"(?:Extend Ban|Ban) for (?P[0-9A-Fa-f:.]+)\" | ip =~ \"$ip_regex\" [$__range])))", + "instant": true, + "queryType": "instant", + "refId": "A" + } + ], + "title": "IPs Seen in Ban Events", + "type": "stat", + "description": "Unique IPs with at least one Fail2Ban ban event in the selected time range. This is log-derived, not a kernel live-ban list." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 16, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(25, sum by (ip) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*)\" | regexp \"(?:Extend Ban|Ban) for (?P[0-9A-Fa-f:.]+)\" | ip =~ \"$ip_regex\" [$__range])))", + "instant": true, + "legendFormat": "{{ip}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Blocked IPs by Ban Events", + "type": "bargauge", + "description": "Top IPs ranked by Fail2Ban ban events seen in logs during the selected time range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 17, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*)\" [$__interval]))", + "legendFormat": "block events", + "queryType": "range", + "refId": "A" + } + ], + "title": "Ban Events Over Time", + "type": "timeseries", + "description": "Time series of Fail2Ban ban events. Useful for spotting brute-force or scan spikes." + }, + { + "datasource": "$loki_ds", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 18, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "direction": "backward", + "editorMode": "code", + "expr": "{namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*|Un-Banned .*|Clearing out state for .*|Failed to get Client Identifier.*)\" | msg =~ \".*$ip_regex.*\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Fail2Ban Event Logs", + "type": "logs", + "description": "Raw Fail2Ban event lines, filterable via the shared IP regex variable." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 19, + "panels": [], + "title": "Client IP & Host Analysis", + "type": "row" + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 100 + }, + { + "color": "red", + "value": 250 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 65 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" [$__range])))", + "instant": true, + "queryType": "instant", + "refId": "A" + } + ], + "title": "Unique Client IPs", + "type": "stat", + "description": "Distinct client source IPs seen in Traefik access logs for the selected host/IP filters." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 25 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 65 + }, + "id": 21, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 400 | DownstreamStatus < 500 [$__range])))", + "instant": true, + "queryType": "instant", + "refId": "A" + } + ], + "title": "Unique 4xx Client IPs", + "type": "stat", + "description": "Distinct source IPs that triggered 4xx responses in the selected time range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 65 + }, + "id": 22, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 500 [$__range])))", + "instant": true, + "queryType": "instant", + "refId": "A" + } + ], + "title": "Unique 5xx Client IPs", + "type": "stat", + "description": "Distinct source IPs that hit upstream/server-side failures in the selected time range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 69 + }, + "id": 23, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(15, sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" [$__range])))", + "instant": true, + "legendFormat": "{{ClientHost}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Client IPs by Requests", + "type": "bargauge", + "description": "Request-heavy client IPs from Traefik access logs for the selected range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 69 + }, + "id": 24, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(15, sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 400 | DownstreamStatus < 500 [$__range])))", + "instant": true, + "legendFormat": "{{ClientHost}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Client IPs by 4xx", + "type": "bargauge", + "description": "Client IPs producing the most 4xx responses in the selected range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 69 + }, + "id": 25, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(15, sum by (ClientHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 500 [$__range])))", + "instant": true, + "legendFormat": "{{ClientHost}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Client IPs by 5xx", + "type": "bargauge", + "description": "Client IPs associated with the most 5xx responses in the selected range." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 26, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(12, sum by (RequestHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 400 | DownstreamStatus < 500 [$__range])))", + "instant": true, + "legendFormat": "{{RequestHost}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Request Hosts by 4xx", + "type": "bargauge", + "description": "Hosts currently attracting the most client-side errors or probes." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 27, + "options": { + "displayMode": "gradient", + "legend": { + "displayMode": "hidden", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(12, sum by (RequestHost) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 500 [$__range])))", + "instant": true, + "legendFormat": "{{RequestHost}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Request Hosts by 5xx", + "type": "bargauge", + "description": "Hosts whose backends currently generate the most 5xx responses." + }, + { + "datasource": "$loki_ds", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 28, + "options": { + "cellHeight": "sm", + "showHeader": true, + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "fields": "" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(50, sum by (ip) (count_over_time({namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*)\" | regexp \"(?:Extend Ban|Ban) for (?P[0-9A-Fa-f:.]+)\" | ip =~ \"$ip_regex\" [$__range])))", + "instant": true, + "legendFormat": "{{ip}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Blocked IPs in Selected Range", + "type": "table", + "description": "IP addresses that were blocked by Fail2Ban during the selected time range. This is log-derived history, not a live nftables/iptables inventory." + }, + { + "datasource": "$loki_ds", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 29, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "direction": "backward", + "editorMode": "code", + "expr": "{namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | prefix=\"Fail-2-Ban\" | msg =~ \"(Extend Ban for .*|Ban for .*|Un-Banned .*|Clearing out state for .*|Failed to get Client Identifier.*)\" | msg =~ \".*$ip_regex.*\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Recent Fail2Ban Ban / Unban Events", + "type": "logs", + "description": "Recent Fail2Ban state changes and related events, filtered by the shared IP regex when needed." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 33, + "panels": [], + "title": "Raw Traefik Logs", + "type": "row" + }, + { + "datasource": "$loki_ds", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 94 + }, + "id": 12, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "direction": "backward", + "editorMode": "code", + "expr": "{namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Access Logs (Filtered)", + "type": "logs" + }, + { + "datasource": "$loki_ds", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 94 + }, + "id": 13, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "direction": "backward", + "editorMode": "code", + "expr": "{namespace=\"traefik\", pod=~\"$pod_regex\", container=\"traefik\"} | json | __error__=\"\" | RequestHost =~ \"$request_host\" | ClientHost =~ \"$ip_regex\" | DownstreamStatus >= 400", + "queryType": "range", + "refId": "A" + } + ], + "title": "Error Logs (4xx/5xx)", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 41, + "tags": [ + "traefik", + "prometheus", + "loki", + "security", + "fail2ban" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "prometheus_ds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "Loki Traefik", + "value": "Loki Traefik" + }, + "hide": 0, + "includeAll": false, + "label": "Loki", + "multi": false, + "name": "loki_ds", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$prometheus_ds", + "definition": "label_values(traefik_entrypoint_requests_total, entrypoint)", + "hide": 0, + "includeAll": true, + "label": "Entrypoint", + "multi": true, + "name": "entrypoint", + "options": [], + "query": { + "query": "label_values(traefik_entrypoint_requests_total, entrypoint)", + "refId": "EntrypointVar" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$prometheus_ds", + "definition": "label_values(traefik_router_requests_total, router)", + "hide": 0, + "includeAll": true, + "label": "Router", + "multi": true, + "name": "router", + "options": [], + "query": { + "query": "label_values(traefik_router_requests_total, router)", + "refId": "RouterVar" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "traefik-.*", + "value": "traefik-.*" + }, + "hide": 0, + "label": "Pod Regex", + "name": "pod_regex", + "options": [ + { + "selected": true, + "text": "traefik-.*", + "value": "traefik-.*" + } + ], + "query": "traefik-.*", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": ".*", + "value": ".*" + }, + "hide": 0, + "label": "Host Regex", + "name": "request_host", + "options": [ + { + "selected": true, + "text": ".*", + "value": ".*" + } + ], + "query": ".*", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": ".*", + "value": ".*" + }, + "hide": 0, + "label": "IP Regex", + "name": "ip_regex", + "options": [ + { + "selected": true, + "text": ".*", + "value": ".*" + } + ], + "query": ".*", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": ".*", + "value": ".*" + }, + "hide": 0, + "label": "Backend Regex", + "name": "backend_regex", + "options": [ + { + "selected": true, + "text": ".*", + "value": ".*" + } + ], + "query": ".*", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "Traefik Overview", + "uid": "traefik-overview", + "version": 10, + "weekStart": "" + }