mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-16 02:44:42 +02:00
feat: monitoring — dashboards Grafana + alertes Prometheus
Dashboards (ConfigMaps, auto-importés par sidecar Grafana) : - dashboard-kubernetes : cluster overview, pods, PVCs, restarts - dashboard-infrastructure : compute/storage-01/gpu-01 CPU/RAM/disque/réseau + GPU ROCm - dashboard-ai : llama-server GPU/CPU tokens/s, requêtes, VRAM, température GPU Alertes (PrometheusRule) : - alerts-node : CPU>90%, RAM>90%, disque>80%, storage-01/gpu-01 down, load élevé - alerts-k8s : NodeNotReady, PodCrashLoop, PodPending, DeploymentUnavailable, PVCNotBound, ArgoCDOutOfSync - alerts-ai : llama-server GPU/system/monitor down, GPU temp>80°C, VRAM>95%, queue élevée Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
83c10feae7
commit
18da44a89f
6 changed files with 2304 additions and 0 deletions
80
k8s/infra/monitoring/alerts/alerts-ai.yaml
Normal file
80
k8s/infra/monitoring/alerts/alerts-ai.yaml
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: funk-ai-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: funk.ai
|
||||
interval: 1m
|
||||
rules:
|
||||
|
||||
# --- llama-server ---
|
||||
- alert: LlamaServerGPUDown
|
||||
expr: up{job="llama-server-gpu"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "llama-server GPU (Qwen3-8B) inaccessible"
|
||||
description: "Le serveur LLM GPU sur gpu-01:1234 ne répond plus depuis 2 min. Inférence GPU unavailable."
|
||||
|
||||
- alert: LlamaServerSystemDown
|
||||
expr: up{job="llama-server-system"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "llama-server CPU system (Qwen3-1.7B) inaccessible"
|
||||
description: "Le serveur LLM CPU system sur gpu-01:1236 ne répond plus. Profil Hermes system degraded."
|
||||
|
||||
- alert: LlamaServerMonitorDown
|
||||
expr: up{job="llama-server-monitor"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "llama-server CPU monitor (Qwen3-1.7B) inaccessible"
|
||||
description: "Le serveur LLM CPU monitor sur gpu-01:1237 ne répond plus. Profil Hermes monitor degraded."
|
||||
|
||||
# --- GPU temperature (ROCm via textfile_collector) ---
|
||||
- alert: GPUTemperatureHigh
|
||||
expr: rocm_gpu_temperature_celsius > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "GPU RX 6700XT température élevée : {{ $value | printf \"%.0f\" }}°C"
|
||||
description: "Température GPU à {{ $value | printf \"%.1f\" }}°C depuis 5 min. Vérifier la ventilation."
|
||||
|
||||
- alert: GPUTemperatureCritical
|
||||
expr: rocm_gpu_temperature_celsius > 90
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "GPU RX 6700XT surchauffe : {{ $value | printf \"%.0f\" }}°C"
|
||||
description: "Température GPU critique à {{ $value | printf \"%.1f\" }}°C. Risque de throttling ou shutdown thermique."
|
||||
|
||||
# --- VRAM ---
|
||||
- alert: GPUVRAMAlmostFull
|
||||
expr: |
|
||||
(rocm_vram_used_bytes / rocm_vram_total_bytes) * 100 > 95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "VRAM GPU presque saturée"
|
||||
description: "VRAM utilisée à {{ $value | printf \"%.1f\" }}% ({{ $labels.gpu }}). Risque d'OOM sur les prochaines requêtes."
|
||||
|
||||
# --- Requêtes en attente (backpressure) ---
|
||||
- alert: LlamaServerHighQueueGPU
|
||||
expr: llamacpp:requests_deferred{job="llama-server-gpu"} > 5
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "File d'attente llama-server GPU élevée"
|
||||
description: "{{ $value | printf \"%.0f\" }} requêtes en attente sur le serveur GPU depuis 2 min."
|
||||
Loading…
Add table
Add a link
Reference in a new issue