mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-08 11:04:43 +02:00
18da44a89f
Dashboards (ConfigMaps, auto-importés par sidecar Grafana) : - dashboard-kubernetes : cluster overview, pods, PVCs, restarts - dashboard-infrastructure : compute/storage-01/gpu-01 CPU/RAM/disque/réseau + GPU ROCm - dashboard-ai : llama-server GPU/CPU tokens/s, requêtes, VRAM, température GPU Alertes (PrometheusRule) : - alerts-node : CPU>90%, RAM>90%, disque>80%, storage-01/gpu-01 down, load élevé - alerts-k8s : NodeNotReady, PodCrashLoop, PodPending, DeploymentUnavailable, PVCNotBound, ArgoCDOutOfSync - alerts-ai : llama-server GPU/system/monitor down, GPU temp>80°C, VRAM>95%, queue élevée Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
80 lines
3 KiB
YAML
80 lines
3 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: funk-ai-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
spec:
|
|
groups:
|
|
- name: funk.ai
|
|
interval: 1m
|
|
rules:
|
|
|
|
# --- llama-server ---
|
|
- alert: LlamaServerGPUDown
|
|
expr: up{job="llama-server-gpu"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "llama-server GPU (Qwen3-8B) inaccessible"
|
|
description: "Le serveur LLM GPU sur gpu-01:1234 ne répond plus depuis 2 min. Inférence GPU unavailable."
|
|
|
|
- alert: LlamaServerSystemDown
|
|
expr: up{job="llama-server-system"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "llama-server CPU system (Qwen3-1.7B) inaccessible"
|
|
description: "Le serveur LLM CPU system sur gpu-01:1236 ne répond plus. Profil Hermes system degraded."
|
|
|
|
- alert: LlamaServerMonitorDown
|
|
expr: up{job="llama-server-monitor"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "llama-server CPU monitor (Qwen3-1.7B) inaccessible"
|
|
description: "Le serveur LLM CPU monitor sur gpu-01:1237 ne répond plus. Profil Hermes monitor degraded."
|
|
|
|
# --- GPU temperature (ROCm via textfile_collector) ---
|
|
- alert: GPUTemperatureHigh
|
|
expr: rocm_gpu_temperature_celsius > 80
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "GPU RX 6700XT température élevée : {{ $value | printf \"%.0f\" }}°C"
|
|
description: "Température GPU à {{ $value | printf \"%.1f\" }}°C depuis 5 min. Vérifier la ventilation."
|
|
|
|
- alert: GPUTemperatureCritical
|
|
expr: rocm_gpu_temperature_celsius > 90
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "GPU RX 6700XT surchauffe : {{ $value | printf \"%.0f\" }}°C"
|
|
description: "Température GPU critique à {{ $value | printf \"%.1f\" }}°C. Risque de throttling ou shutdown thermique."
|
|
|
|
# --- VRAM ---
|
|
- alert: GPUVRAMAlmostFull
|
|
expr: |
|
|
(rocm_vram_used_bytes / rocm_vram_total_bytes) * 100 > 95
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "VRAM GPU presque saturée"
|
|
description: "VRAM utilisée à {{ $value | printf \"%.1f\" }}% ({{ $labels.gpu }}). Risque d'OOM sur les prochaines requêtes."
|
|
|
|
# --- Requêtes en attente (backpressure) ---
|
|
- alert: LlamaServerHighQueueGPU
|
|
expr: llamacpp:requests_deferred{job="llama-server-gpu"} > 5
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "File d'attente llama-server GPU élevée"
|
|
description: "{{ $value | printf \"%.0f\" }} requêtes en attente sur le serveur GPU depuis 2 min."
|