apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-alerting-traefik
  namespace: kube-mon
data:
  traefik-alert-rules.yaml: |
    apiVersion: 1
    groups:
      - orgId: 1
        name: traefik-prometheus
        folder: Traefik Alerts
        interval: 1m
        rules:
          - uid: traefik-instance-down
            title: Traefik Instance Down
            condition: C
            data:
              - refId: A
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: prometheus
                model:
                  editorMode: code
                  expr: max by (pod, instance) (up{job="traefik"} == 0)
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  range: true
                  refId: A
              - refId: B
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: __expr__
                model:
                  conditions:
                    - evaluator:
                        params: []
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                          - B
                      reducer:
                        params: []
                        type: last
                      type: query
                  datasource:
                    type: __expr__
                    uid: __expr__
                  expression: A
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  reducer: last
                  refId: B
                  settings:
                    mode: dropNN
                  type: reduce
              - refId: C
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: __expr__
                model:
                  conditions:
                    - evaluator:
                        params:
                          - 0.5
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                          - C
                      reducer:
                        params: []
                        type: last
                      type: query
                  datasource:
                    type: __expr__
                    uid: __expr__
                  expression: B
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  refId: C
                  type: threshold
            dashboardUid: traefik-overview
            panelId: 5
            noDataState: OK
            execErrState: Error
            for: 5m
            annotations:
              __dashboardUid__: traefik-overview
              __panelId__: "5"
              description: Die Traefik-Instanz {{ $labels.pod }} auf {{ $labels.instance }} liefert seit 5 Minuten keine erfolgreichen Prometheus-Scrapes.
              summary: Traefik-Instanz {{ $labels.pod }} ist nicht erreichbar
            labels:
              category: availability
              cluster: microk8s
              severity: critical
              source: traefik
              team: ops
            isPaused: false
            notification_settings:
              receiver: n8n
          - uid: traefik-websecure-5xx-ratio-high
            title: Traefik Websecure 5xx Ratio High
            condition: C
            data:
              - refId: A
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: prometheus
                model:
                  editorMode: code
                  expr: (100 * sum by (entrypoint) (rate(traefik_entrypoint_requests_total{entrypoint="websecure",code=~"5.."}[5m])) / clamp_min(sum by (entrypoint) (rate(traefik_entrypoint_requests_total{entrypoint="websecure"}[5m])), 0.001)) and on(entrypoint) (sum by (entrypoint) (rate(traefik_entrypoint_requests_total{entrypoint="websecure"}[5m])) > 0.3)
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  range: true
                  refId: A
              - refId: B
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: __expr__
                model:
                  conditions:
                    - evaluator:
                        params: []
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                          - B
                      reducer:
                        params: []
                        type: last
                      type: query
                  datasource:
                    type: __expr__
                    uid: __expr__
                  expression: A
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  reducer: last
                  refId: B
                  settings:
                    mode: dropNN
                  type: reduce
              - refId: C
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: __expr__
                model:
                  conditions:
                    - evaluator:
                        params:
                          - 15
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                          - C
                      reducer:
                        params: []
                        type: last
                      type: query
                  datasource:
                    type: __expr__
                    uid: __expr__
                  expression: B
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  refId: C
                  type: threshold
            dashboardUid: traefik-overview
            panelId: 6
            noDataState: OK
            execErrState: Error
            for: 10m
            annotations:
              __dashboardUid__: traefik-overview
              __panelId__: "6"
              description: Die 5xx-Quote auf {{ $labels.entrypoint }} liegt seit 10 Minuten ueber 15% (aktuell {{ printf "%.2f" $values.B.Value }}%).
              summary: Traefik Websecure 5xx-Quote ist erhoeht
            labels:
              category: errors
              cluster: microk8s
              severity: warning
              source: traefik
              team: ops
            isPaused: false
            notification_settings:
              receiver: n8n
          - uid: traefik-router-5xx-ratio-critical
            title: Traefik Router 5xx Ratio Critical
            condition: C
            data:
              - refId: A
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: prometheus
                model:
                  editorMode: code
                  expr: (100 * sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal",code=~"5.."}[5m])) / clamp_min(sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal"}[5m])), 0.001)) and on(router) (sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal"}[5m])) > 0.1)
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  range: true
                  refId: A
              - refId: B
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: __expr__
                model:
                  conditions:
                    - evaluator:
                        params: []
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                          - B
                      reducer:
                        params: []
                        type: last
                      type: query
                  datasource:
                    type: __expr__
                    uid: __expr__
                  expression: A
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  reducer: last
                  refId: B
                  settings:
                    mode: dropNN
                  type: reduce
              - refId: C
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: __expr__
                model:
                  conditions:
                    - evaluator:
                        params:
                          - 50
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                          - C
                      reducer:
                        params: []
                        type: last
                      type: query
                  datasource:
                    type: __expr__
                    uid: __expr__
                  expression: B
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  refId: C
                  type: threshold
            dashboardUid: traefik-overview
            panelId: 11
            noDataState: OK
            execErrState: Error
            for: 10m
            annotations:
              __dashboardUid__: traefik-overview
              __panelId__: "11"
              description: Der Router {{ $labels.router }} liegt seit 10 Minuten ueber 50% 5xx-Quote bei relevanter Last (aktuell {{ printf "%.2f" $values.B.Value }}%).
              summary: Traefik-Router {{ $labels.router }} produziert viele 5xx
            labels:
              category: errors
              cluster: microk8s
              severity: critical
              source: traefik
              team: ops
            isPaused: false
            notification_settings:
              receiver: n8n
          - uid: traefik-router-p95-latency-high
            title: Traefik Router P95 Latency High
            condition: C
            data:
              - refId: A
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: prometheus
                model:
                  editorMode: code
                  expr: (1000 * histogram_quantile(0.95, sum by (router, le) (rate(traefik_router_request_duration_seconds_bucket{router!~"ping@internal|prometheus@internal"}[5m])))) and on(router) (sum by (router) (rate(traefik_router_requests_total{router!~"ping@internal|prometheus@internal"}[5m])) > 0.05)
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  range: true
                  refId: A
              - refId: B
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: __expr__
                model:
                  conditions:
                    - evaluator:
                        params: []
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                          - B
                      reducer:
                        params: []
                        type: last
                      type: query
                  datasource:
                    type: __expr__
                    uid: __expr__
                  expression: A
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  reducer: last
                  refId: B
                  settings:
                    mode: dropNN
                  type: reduce
              - refId: C
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: __expr__
                model:
                  conditions:
                    - evaluator:
                        params:
                          - 3000
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                          - C
                      reducer:
                        params: []
                        type: last
                      type: query
                  datasource:
                    type: __expr__
                    uid: __expr__
                  expression: B
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  refId: C
                  type: threshold
            dashboardUid: traefik-overview
            panelId: 44
            noDataState: OK
            execErrState: Error
            for: 10m
            annotations:
              __dashboardUid__: traefik-overview
              __panelId__: "44"
              description: Der Router {{ $labels.router }} hat seit 10 Minuten eine p95-Latenz ueber 3000 ms (aktuell {{ printf "%.0f" $values.B.Value }} ms).
              summary: Traefik-Router {{ $labels.router }} ist langsam
            labels:
              category: latency
              cluster: microk8s
              severity: warning
              source: traefik
              team: ops
            isPaused: false
            notification_settings:
              receiver: n8n
          - uid: traefik-cert-expiry-soon
            title: Traefik Certificate Expiry Soon
            condition: C
            data:
              - refId: A
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: prometheus
                model:
                  editorMode: code
                  expr: (max by (cn) (traefik_tls_certs_not_after) - time()) / 86400
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  range: true
                  refId: A
              - refId: B
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: __expr__
                model:
                  conditions:
                    - evaluator:
                        params: []
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                          - B
                      reducer:
                        params: []
                        type: last
                      type: query
                  datasource:
                    type: __expr__
                    uid: __expr__
                  expression: A
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  reducer: last
                  refId: B
                  settings:
                    mode: dropNN
                  type: reduce
              - refId: C
                relativeTimeRange:
                  from: 600
                  to: 0
                datasourceUid: __expr__
                model:
                  conditions:
                    - evaluator:
                        params:
                          - 30
                        type: lt
                      operator:
                        type: and
                      query:
                        params:
                          - C
                      reducer:
                        params: []
                        type: last
                      type: query
                  datasource:
                    type: __expr__
                    uid: __expr__
                  expression: B
                  hide: false
                  intervalMs: 1000
                  maxDataPoints: 43200
                  refId: C
                  type: threshold
            dashboardUid: traefik-overview
            panelId: 46
            noDataState: OK
            execErrState: Error
            for: 1h
            annotations:
              __dashboardUid__: traefik-overview
              __panelId__: "46"
              description: Das Zertifikat {{ $labels.cn }} laeuft in weniger als 30 Tagen ab (aktuell {{ printf "%.1f" $values.B.Value }} Tage Restlaufzeit).
              summary: Traefik-Zertifikat {{ $labels.cn }} laeuft bald ab
            labels:
              category: certificate
              cluster: microk8s
              severity: warning
              source: traefik
              team: ops
            isPaused: false
            notification_settings:
              receiver: n8n
  contact-points.n8n.yaml: |
    apiVersion: 1
    contactPoints:
      - orgId: 1
        name: n8n
        receivers:
          - uid: dfjuxb8oyjh8ge
            type: webhook
            settings:
              httpMethod: POST
              url: https://n8n.apps.rsitservice.de/webhook/alerting
              password: __REPLACE_WITH_N8N_WEBHOOK_PASSWORD__
            disableResolveMessage: false