mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-08 06:44:42 +02:00
18da44a89f
Dashboards (ConfigMaps, auto-importés par sidecar Grafana) : - dashboard-kubernetes : cluster overview, pods, PVCs, restarts - dashboard-infrastructure : compute/storage-01/gpu-01 CPU/RAM/disque/réseau + GPU ROCm - dashboard-ai : llama-server GPU/CPU tokens/s, requêtes, VRAM, température GPU Alertes (PrometheusRule) : - alerts-node : CPU>90%, RAM>90%, disque>80%, storage-01/gpu-01 down, load élevé - alerts-k8s : NodeNotReady, PodCrashLoop, PodPending, DeploymentUnavailable, PVCNotBound, ArgoCDOutOfSync - alerts-ai : llama-server GPU/system/monitor down, GPU temp>80°C, VRAM>95%, queue élevée Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
106 lines
3.9 KiB
YAML
106 lines
3.9 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: funk-node-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
spec:
|
|
groups:
|
|
- name: funk.nodes
|
|
interval: 1m
|
|
rules:
|
|
|
|
# --- CPU ---
|
|
- alert: NodeHighCPU
|
|
expr: |
|
|
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "CPU élevé sur {{ $labels.instance }}"
|
|
description: "CPU à {{ $value | printf \"%.1f\" }}% depuis 5 min."
|
|
|
|
- alert: NodeCriticalCPU
|
|
expr: |
|
|
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 98
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "CPU critique sur {{ $labels.instance }}"
|
|
description: "CPU à {{ $value | printf \"%.1f\" }}% depuis 2 min."
|
|
|
|
# --- RAM ---
|
|
- alert: NodeHighMemory
|
|
expr: |
|
|
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Mémoire élevée sur {{ $labels.instance }}"
|
|
description: "RAM utilisée à {{ $value | printf \"%.1f\" }}% depuis 5 min."
|
|
|
|
- alert: NodeCriticalMemory
|
|
expr: |
|
|
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 97
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Mémoire critique sur {{ $labels.instance }}"
|
|
description: "RAM utilisée à {{ $value | printf \"%.1f\" }}%. Risque OOM."
|
|
|
|
# --- Disque ---
|
|
- alert: NodeDiskSpaceLow
|
|
expr: |
|
|
(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"}
|
|
/ node_filesystem_size_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"}) * 100 > 80
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Disque plein à {{ $value | printf \"%.0f\" }}% sur {{ $labels.instance }}:{{ $labels.mountpoint }}"
|
|
description: "Point de montage {{ $labels.mountpoint }} à {{ $value | printf \"%.1f\" }}%."
|
|
|
|
- alert: NodeDiskSpaceCritical
|
|
expr: |
|
|
(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"}
|
|
/ node_filesystem_size_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"}) * 100 > 93
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Disque presque plein sur {{ $labels.instance }}:{{ $labels.mountpoint }}"
|
|
description: "{{ $labels.mountpoint }} à {{ $value | printf \"%.1f\" }}% — intervention requise."
|
|
|
|
# --- Hors cluster : storage-01 et gpu-01 ---
|
|
- alert: Storage01Down
|
|
expr: up{job="storage-01"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "storage-01 ne répond plus"
|
|
description: "node_exporter de storage-01 inaccessible depuis 2 min. NFS, PostgreSQL, LiteLLM probablement down."
|
|
|
|
- alert: Gpu01Down
|
|
expr: up{job="gpu-01-node"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "gpu-01 ne répond plus"
|
|
description: "node_exporter de gpu-01 inaccessible depuis 2 min. LLM inference unavailable."
|
|
|
|
# --- Load average ---
|
|
- alert: NodeHighLoad
|
|
expr: node_load5 > 8
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Load average élevé sur {{ $labels.instance }}"
|
|
description: "Load5 = {{ $value | printf \"%.1f\" }} depuis 10 min."
|