mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-08 11:04:43 +02:00
feat: monitoring — dashboards Grafana + alertes Prometheus
Dashboards (ConfigMaps, auto-importés par sidecar Grafana) : - dashboard-kubernetes : cluster overview, pods, PVCs, restarts - dashboard-infrastructure : compute/storage-01/gpu-01 CPU/RAM/disque/réseau + GPU ROCm - dashboard-ai : llama-server GPU/CPU tokens/s, requêtes, VRAM, température GPU Alertes (PrometheusRule) : - alerts-node : CPU>90%, RAM>90%, disque>80%, storage-01/gpu-01 down, load élevé - alerts-k8s : NodeNotReady, PodCrashLoop, PodPending, DeploymentUnavailable, PVCNotBound, ArgoCDOutOfSync - alerts-ai : llama-server GPU/system/monitor down, GPU temp>80°C, VRAM>95%, queue élevée Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
alkatrazz
parent
83c10feae7
commit
18da44a89f
6 changed files with 2304 additions and 0 deletions
80
k8s/infra/monitoring/alerts/alerts-ai.yaml
Normal file
80
k8s/infra/monitoring/alerts/alerts-ai.yaml
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: funk-ai-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: funk.ai
|
||||
interval: 1m
|
||||
rules:
|
||||
|
||||
# --- llama-server ---
|
||||
- alert: LlamaServerGPUDown
|
||||
expr: up{job="llama-server-gpu"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "llama-server GPU (Qwen3-8B) inaccessible"
|
||||
description: "Le serveur LLM GPU sur gpu-01:1234 ne répond plus depuis 2 min. Inférence GPU unavailable."
|
||||
|
||||
- alert: LlamaServerSystemDown
|
||||
expr: up{job="llama-server-system"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "llama-server CPU system (Qwen3-1.7B) inaccessible"
|
||||
description: "Le serveur LLM CPU system sur gpu-01:1236 ne répond plus. Profil Hermes system degraded."
|
||||
|
||||
- alert: LlamaServerMonitorDown
|
||||
expr: up{job="llama-server-monitor"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "llama-server CPU monitor (Qwen3-1.7B) inaccessible"
|
||||
description: "Le serveur LLM CPU monitor sur gpu-01:1237 ne répond plus. Profil Hermes monitor degraded."
|
||||
|
||||
# --- GPU temperature (ROCm via textfile_collector) ---
|
||||
- alert: GPUTemperatureHigh
|
||||
expr: rocm_gpu_temperature_celsius > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "GPU RX 6700XT température élevée : {{ $value | printf \"%.0f\" }}°C"
|
||||
description: "Température GPU à {{ $value | printf \"%.1f\" }}°C depuis 5 min. Vérifier la ventilation."
|
||||
|
||||
- alert: GPUTemperatureCritical
|
||||
expr: rocm_gpu_temperature_celsius > 90
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "GPU RX 6700XT surchauffe : {{ $value | printf \"%.0f\" }}°C"
|
||||
description: "Température GPU critique à {{ $value | printf \"%.1f\" }}°C. Risque de throttling ou shutdown thermique."
|
||||
|
||||
# --- VRAM ---
|
||||
- alert: GPUVRAMAlmostFull
|
||||
expr: |
|
||||
(rocm_vram_used_bytes / rocm_vram_total_bytes) * 100 > 95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "VRAM GPU presque saturée"
|
||||
description: "VRAM utilisée à {{ $value | printf \"%.1f\" }}% ({{ $labels.gpu }}). Risque d'OOM sur les prochaines requêtes."
|
||||
|
||||
# --- Requêtes en attente (backpressure) ---
|
||||
- alert: LlamaServerHighQueueGPU
|
||||
expr: llamacpp:requests_deferred{job="llama-server-gpu"} > 5
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "File d'attente llama-server GPU élevée"
|
||||
description: "{{ $value | printf \"%.0f\" }} requêtes en attente sur le serveur GPU depuis 2 min."
|
||||
88
k8s/infra/monitoring/alerts/alerts-k8s.yaml
Normal file
88
k8s/infra/monitoring/alerts/alerts-k8s.yaml
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: funk-k8s-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: funk.kubernetes
|
||||
interval: 1m
|
||||
rules:
|
||||
|
||||
# --- Nœuds ---
|
||||
- alert: KubeNodeNotReady
|
||||
expr: |
|
||||
kube_node_status_condition{condition="Ready",status="true"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Nœud k8s NotReady : {{ $labels.node }}"
|
||||
description: "Le nœud {{ $labels.node }} est NotReady depuis 5 min. Les workloads peuvent être impactés."
|
||||
|
||||
# --- Pods ---
|
||||
- alert: PodCrashLooping
|
||||
expr: |
|
||||
increase(kube_pod_container_status_restarts_total[15m]) > 3
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Pod en CrashLoop : {{ $labels.namespace }}/{{ $labels.pod }}"
|
||||
description: "{{ $labels.container }} a redémarré {{ $value | printf \"%.0f\" }} fois en 15 min."
|
||||
|
||||
- alert: PodFailedLong
|
||||
expr: |
|
||||
kube_pod_status_phase{phase=~"Failed|Unknown"} == 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Pod en échec : {{ $labels.namespace }}/{{ $labels.pod }}"
|
||||
description: "Le pod est en phase {{ $labels.phase }} depuis 10 min."
|
||||
|
||||
- alert: PodPendingLong
|
||||
expr: |
|
||||
kube_pod_status_phase{phase="Pending"} == 1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Pod Pending depuis 15 min : {{ $labels.namespace }}/{{ $labels.pod }}"
|
||||
description: "Vérifier les ressources disponibles, PVCs, et les événements du pod."
|
||||
|
||||
# --- Deployments ---
|
||||
- alert: DeploymentUnavailable
|
||||
expr: |
|
||||
kube_deployment_status_replicas_available
|
||||
< kube_deployment_spec_replicas
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Deployment dégradé : {{ $labels.namespace }}/{{ $labels.deployment }}"
|
||||
description: "{{ $labels.deployment }} : {{ $value }} réplicas disponibles sur {{ $labels.replicas }} attendus."
|
||||
|
||||
# --- Stockage ---
|
||||
- alert: PVCNotBound
|
||||
expr: |
|
||||
kube_persistentvolumeclaim_status_phase{phase!="Bound"} == 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PVC non lié : {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }}"
|
||||
description: "La PVC est en phase {{ $labels.phase }} depuis 10 min. Vérifier le NFS provisioner."
|
||||
|
||||
# --- ArgoCD ---
|
||||
- alert: ArgoCDAppOutOfSync
|
||||
expr: |
|
||||
count(argocd_app_info{sync_status!="Synced"}) > 0
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Application(s) ArgoCD OutOfSync depuis 30 min"
|
||||
description: "{{ $value }} app(s) ne sont pas synchronisées. Vérifier http://argocd.lab.local"
|
||||
106
k8s/infra/monitoring/alerts/alerts-node.yaml
Normal file
106
k8s/infra/monitoring/alerts/alerts-node.yaml
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: funk-node-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: funk.nodes
|
||||
interval: 1m
|
||||
rules:
|
||||
|
||||
# --- CPU ---
|
||||
- alert: NodeHighCPU
|
||||
expr: |
|
||||
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "CPU élevé sur {{ $labels.instance }}"
|
||||
description: "CPU à {{ $value | printf \"%.1f\" }}% depuis 5 min."
|
||||
|
||||
- alert: NodeCriticalCPU
|
||||
expr: |
|
||||
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 98
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "CPU critique sur {{ $labels.instance }}"
|
||||
description: "CPU à {{ $value | printf \"%.1f\" }}% depuis 2 min."
|
||||
|
||||
# --- RAM ---
|
||||
- alert: NodeHighMemory
|
||||
expr: |
|
||||
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Mémoire élevée sur {{ $labels.instance }}"
|
||||
description: "RAM utilisée à {{ $value | printf \"%.1f\" }}% depuis 5 min."
|
||||
|
||||
- alert: NodeCriticalMemory
|
||||
expr: |
|
||||
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 97
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Mémoire critique sur {{ $labels.instance }}"
|
||||
description: "RAM utilisée à {{ $value | printf \"%.1f\" }}%. Risque OOM."
|
||||
|
||||
# --- Disque ---
|
||||
- alert: NodeDiskSpaceLow
|
||||
expr: |
|
||||
(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"}
|
||||
/ node_filesystem_size_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"}) * 100 > 80
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Disque plein à {{ $value | printf \"%.0f\" }}% sur {{ $labels.instance }}:{{ $labels.mountpoint }}"
|
||||
description: "Point de montage {{ $labels.mountpoint }} à {{ $value | printf \"%.1f\" }}%."
|
||||
|
||||
- alert: NodeDiskSpaceCritical
|
||||
expr: |
|
||||
(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"}
|
||||
/ node_filesystem_size_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"}) * 100 > 93
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Disque presque plein sur {{ $labels.instance }}:{{ $labels.mountpoint }}"
|
||||
description: "{{ $labels.mountpoint }} à {{ $value | printf \"%.1f\" }}% — intervention requise."
|
||||
|
||||
# --- Hors cluster : storage-01 et gpu-01 ---
|
||||
- alert: Storage01Down
|
||||
expr: up{job="storage-01"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "storage-01 ne répond plus"
|
||||
description: "node_exporter de storage-01 inaccessible depuis 2 min. NFS, PostgreSQL, LiteLLM probablement down."
|
||||
|
||||
- alert: Gpu01Down
|
||||
expr: up{job="gpu-01-node"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "gpu-01 ne répond plus"
|
||||
description: "node_exporter de gpu-01 inaccessible depuis 2 min. LLM inference unavailable."
|
||||
|
||||
# --- Load average ---
|
||||
- alert: NodeHighLoad
|
||||
expr: node_load5 > 8
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Load average élevé sur {{ $labels.instance }}"
|
||||
description: "Load5 = {{ $value | printf \"%.1f\" }} depuis 10 min."
|
||||
Loading…
Add table
Add a link
Reference in a new issue