From 18da44a89fb89eaa4ff64419f81190cb2dc0fda3 Mon Sep 17 00:00:00 2001 From: alkatrazz Date: Wed, 13 May 2026 22:47:49 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20monitoring=20=E2=80=94=20dashboards=20G?= =?UTF-8?q?rafana=20+=20alertes=20Prometheus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dashboards (ConfigMaps, auto-importés par sidecar Grafana) : - dashboard-kubernetes : cluster overview, pods, PVCs, restarts - dashboard-infrastructure : compute/storage-01/gpu-01 CPU/RAM/disque/réseau + GPU ROCm - dashboard-ai : llama-server GPU/CPU tokens/s, requêtes, VRAM, température GPU Alertes (PrometheusRule) : - alerts-node : CPU>90%, RAM>90%, disque>80%, storage-01/gpu-01 down, load élevé - alerts-k8s : NodeNotReady, PodCrashLoop, PodPending, DeploymentUnavailable, PVCNotBound, ArgoCDOutOfSync - alerts-ai : llama-server GPU/system/monitor down, GPU temp>80°C, VRAM>95%, queue élevée Co-Authored-By: Claude Sonnet 4.6 --- k8s/infra/monitoring/alerts/alerts-ai.yaml | 80 ++ k8s/infra/monitoring/alerts/alerts-k8s.yaml | 88 ++ k8s/infra/monitoring/alerts/alerts-node.yaml | 106 +++ .../monitoring/dashboards/dashboard-ai.yaml | 753 ++++++++++++++++ .../dashboards/dashboard-infrastructure.yaml | 801 ++++++++++++++++++ .../dashboards/dashboard-kubernetes.yaml | 476 +++++++++++ 6 files changed, 2304 insertions(+) create mode 100644 k8s/infra/monitoring/alerts/alerts-ai.yaml create mode 100644 k8s/infra/monitoring/alerts/alerts-k8s.yaml create mode 100644 k8s/infra/monitoring/alerts/alerts-node.yaml create mode 100644 k8s/infra/monitoring/dashboards/dashboard-ai.yaml create mode 100644 k8s/infra/monitoring/dashboards/dashboard-infrastructure.yaml create mode 100644 k8s/infra/monitoring/dashboards/dashboard-kubernetes.yaml diff --git a/k8s/infra/monitoring/alerts/alerts-ai.yaml b/k8s/infra/monitoring/alerts/alerts-ai.yaml new file mode 100644 index 0000000..5f3530b --- /dev/null +++ b/k8s/infra/monitoring/alerts/alerts-ai.yaml @@ -0,0 +1,80 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: funk-ai-alerts + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + groups: + - name: funk.ai + interval: 1m + rules: + + # --- llama-server --- + - alert: LlamaServerGPUDown + expr: up{job="llama-server-gpu"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "llama-server GPU (Qwen3-8B) inaccessible" + description: "Le serveur LLM GPU sur gpu-01:1234 ne répond plus depuis 2 min. Inférence GPU unavailable." + + - alert: LlamaServerSystemDown + expr: up{job="llama-server-system"} == 0 + for: 2m + labels: + severity: warning + annotations: + summary: "llama-server CPU system (Qwen3-1.7B) inaccessible" + description: "Le serveur LLM CPU system sur gpu-01:1236 ne répond plus. Profil Hermes system degraded." + + - alert: LlamaServerMonitorDown + expr: up{job="llama-server-monitor"} == 0 + for: 2m + labels: + severity: warning + annotations: + summary: "llama-server CPU monitor (Qwen3-1.7B) inaccessible" + description: "Le serveur LLM CPU monitor sur gpu-01:1237 ne répond plus. Profil Hermes monitor degraded." + + # --- GPU temperature (ROCm via textfile_collector) --- + - alert: GPUTemperatureHigh + expr: rocm_gpu_temperature_celsius > 80 + for: 5m + labels: + severity: warning + annotations: + summary: "GPU RX 6700XT température élevée : {{ $value | printf \"%.0f\" }}°C" + description: "Température GPU à {{ $value | printf \"%.1f\" }}°C depuis 5 min. Vérifier la ventilation." + + - alert: GPUTemperatureCritical + expr: rocm_gpu_temperature_celsius > 90 + for: 2m + labels: + severity: critical + annotations: + summary: "GPU RX 6700XT surchauffe : {{ $value | printf \"%.0f\" }}°C" + description: "Température GPU critique à {{ $value | printf \"%.1f\" }}°C. Risque de throttling ou shutdown thermique." + + # --- VRAM --- + - alert: GPUVRAMAlmostFull + expr: | + (rocm_vram_used_bytes / rocm_vram_total_bytes) * 100 > 95 + for: 5m + labels: + severity: warning + annotations: + summary: "VRAM GPU presque saturée" + description: "VRAM utilisée à {{ $value | printf \"%.1f\" }}% ({{ $labels.gpu }}). Risque d'OOM sur les prochaines requêtes." + + # --- Requêtes en attente (backpressure) --- + - alert: LlamaServerHighQueueGPU + expr: llamacpp:requests_deferred{job="llama-server-gpu"} > 5 + for: 2m + labels: + severity: warning + annotations: + summary: "File d'attente llama-server GPU élevée" + description: "{{ $value | printf \"%.0f\" }} requêtes en attente sur le serveur GPU depuis 2 min." diff --git a/k8s/infra/monitoring/alerts/alerts-k8s.yaml b/k8s/infra/monitoring/alerts/alerts-k8s.yaml new file mode 100644 index 0000000..8b9e893 --- /dev/null +++ b/k8s/infra/monitoring/alerts/alerts-k8s.yaml @@ -0,0 +1,88 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: funk-k8s-alerts + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + groups: + - name: funk.kubernetes + interval: 1m + rules: + + # --- Nœuds --- + - alert: KubeNodeNotReady + expr: | + kube_node_status_condition{condition="Ready",status="true"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Nœud k8s NotReady : {{ $labels.node }}" + description: "Le nœud {{ $labels.node }} est NotReady depuis 5 min. Les workloads peuvent être impactés." + + # --- Pods --- + - alert: PodCrashLooping + expr: | + increase(kube_pod_container_status_restarts_total[15m]) > 3 + for: 5m + labels: + severity: warning + annotations: + summary: "Pod en CrashLoop : {{ $labels.namespace }}/{{ $labels.pod }}" + description: "{{ $labels.container }} a redémarré {{ $value | printf \"%.0f\" }} fois en 15 min." + + - alert: PodFailedLong + expr: | + kube_pod_status_phase{phase=~"Failed|Unknown"} == 1 + for: 10m + labels: + severity: warning + annotations: + summary: "Pod en échec : {{ $labels.namespace }}/{{ $labels.pod }}" + description: "Le pod est en phase {{ $labels.phase }} depuis 10 min." + + - alert: PodPendingLong + expr: | + kube_pod_status_phase{phase="Pending"} == 1 + for: 15m + labels: + severity: warning + annotations: + summary: "Pod Pending depuis 15 min : {{ $labels.namespace }}/{{ $labels.pod }}" + description: "Vérifier les ressources disponibles, PVCs, et les événements du pod." + + # --- Deployments --- + - alert: DeploymentUnavailable + expr: | + kube_deployment_status_replicas_available + < kube_deployment_spec_replicas + for: 5m + labels: + severity: warning + annotations: + summary: "Deployment dégradé : {{ $labels.namespace }}/{{ $labels.deployment }}" + description: "{{ $labels.deployment }} : {{ $value }} réplicas disponibles sur {{ $labels.replicas }} attendus." + + # --- Stockage --- + - alert: PVCNotBound + expr: | + kube_persistentvolumeclaim_status_phase{phase!="Bound"} == 1 + for: 10m + labels: + severity: warning + annotations: + summary: "PVC non lié : {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }}" + description: "La PVC est en phase {{ $labels.phase }} depuis 10 min. Vérifier le NFS provisioner." + + # --- ArgoCD --- + - alert: ArgoCDAppOutOfSync + expr: | + count(argocd_app_info{sync_status!="Synced"}) > 0 + for: 30m + labels: + severity: warning + annotations: + summary: "Application(s) ArgoCD OutOfSync depuis 30 min" + description: "{{ $value }} app(s) ne sont pas synchronisées. Vérifier http://argocd.lab.local" diff --git a/k8s/infra/monitoring/alerts/alerts-node.yaml b/k8s/infra/monitoring/alerts/alerts-node.yaml new file mode 100644 index 0000000..203d91f --- /dev/null +++ b/k8s/infra/monitoring/alerts/alerts-node.yaml @@ -0,0 +1,106 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: funk-node-alerts + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + groups: + - name: funk.nodes + interval: 1m + rules: + + # --- CPU --- + - alert: NodeHighCPU + expr: | + 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90 + for: 5m + labels: + severity: warning + annotations: + summary: "CPU élevé sur {{ $labels.instance }}" + description: "CPU à {{ $value | printf \"%.1f\" }}% depuis 5 min." + + - alert: NodeCriticalCPU + expr: | + 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 98 + for: 2m + labels: + severity: critical + annotations: + summary: "CPU critique sur {{ $labels.instance }}" + description: "CPU à {{ $value | printf \"%.1f\" }}% depuis 2 min." + + # --- RAM --- + - alert: NodeHighMemory + expr: | + (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90 + for: 5m + labels: + severity: warning + annotations: + summary: "Mémoire élevée sur {{ $labels.instance }}" + description: "RAM utilisée à {{ $value | printf \"%.1f\" }}% depuis 5 min." + + - alert: NodeCriticalMemory + expr: | + (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 97 + for: 2m + labels: + severity: critical + annotations: + summary: "Mémoire critique sur {{ $labels.instance }}" + description: "RAM utilisée à {{ $value | printf \"%.1f\" }}%. Risque OOM." + + # --- Disque --- + - alert: NodeDiskSpaceLow + expr: | + (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"} + / node_filesystem_size_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"}) * 100 > 80 + for: 10m + labels: + severity: warning + annotations: + summary: "Disque plein à {{ $value | printf \"%.0f\" }}% sur {{ $labels.instance }}:{{ $labels.mountpoint }}" + description: "Point de montage {{ $labels.mountpoint }} à {{ $value | printf \"%.1f\" }}%." + + - alert: NodeDiskSpaceCritical + expr: | + (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"} + / node_filesystem_size_bytes{fstype!~"tmpfs|squashfs|overlay",mountpoint!~"/boot.*"}) * 100 > 93 + for: 5m + labels: + severity: critical + annotations: + summary: "Disque presque plein sur {{ $labels.instance }}:{{ $labels.mountpoint }}" + description: "{{ $labels.mountpoint }} à {{ $value | printf \"%.1f\" }}% — intervention requise." + + # --- Hors cluster : storage-01 et gpu-01 --- + - alert: Storage01Down + expr: up{job="storage-01"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "storage-01 ne répond plus" + description: "node_exporter de storage-01 inaccessible depuis 2 min. NFS, PostgreSQL, LiteLLM probablement down." + + - alert: Gpu01Down + expr: up{job="gpu-01-node"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "gpu-01 ne répond plus" + description: "node_exporter de gpu-01 inaccessible depuis 2 min. LLM inference unavailable." + + # --- Load average --- + - alert: NodeHighLoad + expr: node_load5 > 8 + for: 10m + labels: + severity: warning + annotations: + summary: "Load average élevé sur {{ $labels.instance }}" + description: "Load5 = {{ $value | printf \"%.1f\" }} depuis 10 min." diff --git a/k8s/infra/monitoring/dashboards/dashboard-ai.yaml b/k8s/infra/monitoring/dashboards/dashboard-ai.yaml new file mode 100644 index 0000000..2fc8302 --- /dev/null +++ b/k8s/infra/monitoring/dashboards/dashboard-ai.yaml @@ -0,0 +1,753 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-ai + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + ai.json: | + { + "__inputs": [], + "__requires": [], + "annotations": {"list": []}, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + "id": 1, + "title": "GPU — Qwen3-8B (llama-server GPU, port 1234)", + "type": "row" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "blue", "value": null} + ] + }, + "unit": "short", + "decimals": 1, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 1}, + "id": 2, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area" + }, + "title": "Génération tokens/s (GPU)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-gpu\"}", + "legendFormat": "gen t/s", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "blue", "value": null} + ] + }, + "unit": "short", + "decimals": 1, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 1}, + "id": 3, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area" + }, + "title": "Prompt tokens/s (GPU)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-gpu\"}", + "legendFormat": "prompt t/s", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 2}, + {"color": "red", "value": 5} + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 1}, + "id": 4, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "Requêtes en cours (GPU)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:requests_processing{job=\"llama-server-gpu\"}", + "legendFormat": "processing", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 1}, + {"color": "red", "value": 3} + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 12, "y": 1}, + "id": 5, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "Requêtes en attente (GPU)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:requests_deferred{job=\"llama-server-gpu\"}", + "legendFormat": "deferred", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "short", + "min": 0, + "decimals": 1 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 5}, + "id": 11, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "Tokens/s — génération + prompt (GPU Qwen3-8B)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-gpu\"}", + "legendFormat": "génération t/s", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-gpu\"}", + "legendFormat": "prompt t/s", + "refId": "B" + } + ], + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 5}, + "id": 12, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "Tokens prédits (rate 5m) — GPU", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rate(llamacpp:tokens_predicted_total{job=\"llama-server-gpu\"}[5m])", + "legendFormat": "tokens/s (5m rate)", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rate(llamacpp:prompt_tokens_total{job=\"llama-server-gpu\"}[5m])", + "legendFormat": "prompt tokens/s (5m rate)", + "refId": "B" + } + ], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 13}, + "id": 20, + "title": "CPU — system + monitor (Qwen3-1.7B, ports 1236 / 1237)", + "type": "row" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [{"color": "blue", "value": null}] + }, + "unit": "short", + "decimals": 1, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 14}, + "id": 21, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area" + }, + "title": "Génération t/s — system (CPU)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-system\"}", + "legendFormat": "gen t/s system", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [{"color": "blue", "value": null}] + }, + "unit": "short", + "decimals": 1, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 14}, + "id": 22, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area" + }, + "title": "Génération t/s — monitor (CPU)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-monitor\"}", + "legendFormat": "gen t/s monitor", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [{"color": "blue", "value": null}] + }, + "unit": "short", + "decimals": 1, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 14}, + "id": 23, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area" + }, + "title": "Prompt t/s — system (CPU)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-system\"}", + "legendFormat": "prompt t/s system", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [{"color": "blue", "value": null}] + }, + "unit": "short", + "decimals": 1, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 12, "y": 14}, + "id": 24, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area" + }, + "title": "Prompt t/s — monitor (CPU)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-monitor\"}", + "legendFormat": "prompt t/s monitor", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 2}, + {"color": "red", "value": 5} + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 16, "y": 14}, + "id": 25, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "Requêtes en cours — system + monitor", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:requests_processing{job=~\"llama-server-system|llama-server-monitor\"}", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 1}, + {"color": "red", "value": 3} + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 20, "y": 14}, + "id": 26, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "Requêtes en attente — system + monitor", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:requests_deferred{job=~\"llama-server-system|llama-server-monitor\"}", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "short", + "min": 0, + "decimals": 1 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 18}, + "id": 27, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "Tokens/s — system + monitor (CPU Qwen3-1.7B)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:predicted_tokens_seconds{job=~\"llama-server-system|llama-server-monitor\"}", + "legendFormat": "gen {{job}}", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "llamacpp:prompt_tokens_seconds{job=~\"llama-server-system|llama-server-monitor\"}", + "legendFormat": "prompt {{job}}", + "refId": "B" + } + ], + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 18}, + "id": 28, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "Tokens prédits (rate 5m) — system + monitor", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rate(llamacpp:tokens_predicted_total{job=~\"llama-server-system|llama-server-monitor\"}[5m])", + "legendFormat": "tokens/s {{job}}", + "refId": "A" + } + ], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 26}, + "id": 40, + "title": "GPU Health (RX 6700XT — gfx1031)", + "type": "row" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 85} + ] + }, + "unit": "celsius", + "min": 0, + "max": 110, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 6, "x": 0, "y": 27}, + "id": 41, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "title": "GPU Temperature (°C)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}", + "legendFormat": "GPU Temp", + "refId": "A" + } + ], + "type": "gauge" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 10737418240}, + {"color": "red", "value": 11877376000} + ] + }, + "unit": "bytes", + "min": 0, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 6, "x": 6, "y": 27}, + "id": 42, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "title": "VRAM Usage (GB used / 12.9 GB total)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"}", + "legendFormat": "VRAM used", + "refId": "A" + } + ], + "type": "gauge" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 6, "x": 12, "y": 27}, + "id": 43, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "title": "GPU Utilization %", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rocm_gpu_utilization_percent{job=\"gpu-01-node\",gpu=\"0\"}", + "legendFormat": "GPU Util %", + "refId": "A" + } + ], + "type": "gauge" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "min": 0 + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "GPU Temp (°C)"}, + "properties": [ + {"id": "unit", "value": "celsius"}, + {"id": "custom.axisPlacement", "value": "left"} + ] + }, + { + "matcher": {"id": "byName", "options": "VRAM used (GB)"}, + "properties": [ + {"id": "unit", "value": "bytes"}, + {"id": "custom.axisPlacement", "value": "right"} + ] + } + ] + }, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 35}, + "id": 44, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "GPU Temp + VRAM usage — gpu-01 (gfx1031)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}", + "legendFormat": "GPU Temp (°C)", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"}", + "legendFormat": "VRAM used (GB)", + "refId": "B" + } + ], + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["funk"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "type": "datasource" + } + ] + }, + "time": {"from": "now-3h", "to": "now"}, + "timepicker": {}, + "timezone": "Europe/Paris", + "title": "Funk — IA / Hermes", + "uid": "funk-ai01", + "version": 1 + } diff --git a/k8s/infra/monitoring/dashboards/dashboard-infrastructure.yaml b/k8s/infra/monitoring/dashboards/dashboard-infrastructure.yaml new file mode 100644 index 0000000..1c3d15b --- /dev/null +++ b/k8s/infra/monitoring/dashboards/dashboard-infrastructure.yaml @@ -0,0 +1,801 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-infrastructure + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + infrastructure.json: | + { + "__inputs": [], + "__requires": [], + "annotations": {"list": []}, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + "id": 1, + "title": "Compute nodes (compute-01 / 02 / 03)", + "type": "row" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 1}, + "id": 2, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "CPU % (avg compute)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "avg(100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}[5m])) * 100))", + "legendFormat": "CPU avg", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 1}, + "id": 3, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "RAM % (avg compute)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "avg(100 - (node_memory_MemAvailable_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} / node_memory_MemTotal_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} * 100))", + "legendFormat": "RAM avg", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 2}, + {"color": "red", "value": 4} + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 1}, + "id": 4, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "Load1 (avg compute)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "avg(node_load1{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"})", + "legendFormat": "Load1", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 5}, + "id": 11, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "CPU % — compute-01/02/03", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}[5m])) * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 5}, + "id": 12, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "RAM % — compute-01/02/03", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "100 - (node_memory_MemAvailable_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} / node_memory_MemTotal_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 5, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "Bps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 5}, + "id": 13, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "Network I/O — compute nodes", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "sum by (instance) (irate(node_network_receive_bytes_total{instance=~\"192\\.168\\.10\\.(11|12|13):9100\",device!~\"lo|veth.*|flannel.*|cni.*\"}[5m]))", + "legendFormat": "rx {{instance}}", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "sum by (instance) (irate(node_network_transmit_bytes_total{instance=~\"192\\.168\\.10\\.(11|12|13):9100\",device!~\"lo|veth.*|flannel.*|cni.*\"}[5m]))", + "legendFormat": "tx {{instance}}", + "refId": "B" + } + ], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 13}, + "id": 20, + "title": "storage-01 (gateway — RAID5 — NFS)", + "type": "row" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 14}, + "id": 21, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "CPU % — storage-01", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"storage-01\"}[5m])) * 100)", + "legendFormat": "CPU", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 14}, + "id": 22, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "RAM % — storage-01", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "100 - (node_memory_MemAvailable_bytes{job=\"storage-01\"} / node_memory_MemTotal_bytes{job=\"storage-01\"} * 100)", + "legendFormat": "RAM", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 14}, + "id": 23, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "Disk /srv usage % — storage-01", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "100 - (node_filesystem_avail_bytes{job=\"storage-01\",mountpoint=\"/srv\",fstype!=\"tmpfs\"} / node_filesystem_size_bytes{job=\"storage-01\",mountpoint=\"/srv\",fstype!=\"tmpfs\"} * 100)", + "legendFormat": "/srv", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 6, "x": 0, "y": 18}, + "id": 24, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "CPU — storage-01", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"storage-01\"}[5m])) * 100)", + "legendFormat": "CPU %", + "refId": "A" + } + ], + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 6, "x": 6, "y": 18}, + "id": 25, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "RAM — storage-01", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "100 - (node_memory_MemAvailable_bytes{job=\"storage-01\"} / node_memory_MemTotal_bytes{job=\"storage-01\"} * 100)", + "legendFormat": "RAM %", + "refId": "A" + } + ], + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 5, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "Bps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 6, "x": 12, "y": 18}, + "id": 26, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "Disk I/O — storage-01", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "sum(irate(node_disk_read_bytes_total{job=\"storage-01\"}[5m]))", + "legendFormat": "read", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "sum(irate(node_disk_written_bytes_total{job=\"storage-01\"}[5m]))", + "legendFormat": "write", + "refId": "B" + } + ], + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 5, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "Bps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 6, "x": 18, "y": 18}, + "id": 27, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "Network I/O — storage-01", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "sum(irate(node_network_receive_bytes_total{job=\"storage-01\",device!~\"lo|veth.*\"}[5m]))", + "legendFormat": "rx", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "sum(irate(node_network_transmit_bytes_total{job=\"storage-01\",device!~\"lo|veth.*\"}[5m]))", + "legendFormat": "tx", + "refId": "B" + } + ], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 26}, + "id": 30, + "title": "gpu-01 (RX 6700XT — ROCm — llama-server)", + "type": "row" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 27}, + "id": 31, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "CPU % — gpu-01", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"gpu-01-node\"}[5m])) * 100)", + "legendFormat": "CPU", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 27}, + "id": 32, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "RAM % — gpu-01", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "100 - (node_memory_MemAvailable_bytes{job=\"gpu-01-node\"} / node_memory_MemTotal_bytes{job=\"gpu-01-node\"} * 100)", + "legendFormat": "RAM", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 85} + ] + }, + "unit": "celsius", + "min": 0, + "max": 110, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 6, "x": 0, "y": 31}, + "id": 33, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "title": "GPU Temperature (°C)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}", + "legendFormat": "GPU Temp", + "refId": "A" + } + ], + "type": "gauge" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 80}, + {"color": "red", "value": 92} + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 6, "x": 6, "y": 31}, + "id": 34, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "title": "VRAM Usage %", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"} / rocm_vram_total_bytes{job=\"gpu-01-node\",gpu=\"0\"} * 100", + "legendFormat": "VRAM %", + "refId": "A" + } + ], + "type": "gauge" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 70}, + {"color": "red", "value": 90} + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 6, "x": 12, "y": 31}, + "id": 35, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "title": "GPU Utilization %", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rocm_gpu_utilization_percent{job=\"gpu-01-node\",gpu=\"0\"}", + "legendFormat": "GPU Util", + "refId": "A" + } + ], + "type": "gauge" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "min": 0 + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "GPU Temp (°C)"}, + "properties": [{"id": "unit", "value": "celsius"}, {"id": "custom.axisPlacement", "value": "left"}] + }, + { + "matcher": {"id": "byName", "options": "VRAM %"}, + "properties": [{"id": "unit", "value": "percent"}, {"id": "custom.axisPlacement", "value": "right"}, {"id": "max", "value": 100}] + } + ] + }, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 39}, + "id": 36, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "GPU Temp + VRAM % — gpu-01", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}", + "legendFormat": "GPU Temp (°C)", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"} / rocm_vram_total_bytes{job=\"gpu-01-node\",gpu=\"0\"} * 100", + "legendFormat": "VRAM %", + "refId": "B" + } + ], + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["funk"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "type": "datasource" + } + ] + }, + "time": {"from": "now-3h", "to": "now"}, + "timepicker": {}, + "timezone": "Europe/Paris", + "title": "Funk — Infrastructure", + "uid": "funk-inf1", + "version": 1 + } diff --git a/k8s/infra/monitoring/dashboards/dashboard-kubernetes.yaml b/k8s/infra/monitoring/dashboards/dashboard-kubernetes.yaml new file mode 100644 index 0000000..b41ac55 --- /dev/null +++ b/k8s/infra/monitoring/dashboards/dashboard-kubernetes.yaml @@ -0,0 +1,476 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-kubernetes + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + kubernetes.json: | + { + "__inputs": [], + "__requires": [], + "annotations": {"list": []}, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + "id": 1, + "title": "Cluster", + "type": "row" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 1}, + "id": 2, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "Nodes Ready", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "count(kube_node_status_condition{condition=\"Ready\",status=\"true\"} == 1)", + "legendFormat": "Ready", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "blue", "value": null} + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 1}, + "id": 3, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "Pods Running", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "count(kube_pod_status_phase{phase=\"Running\"} == 1)", + "legendFormat": "Running", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 1}, + {"color": "red", "value": 3} + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 1}, + "id": 4, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "Pods Failed / Pending", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "count(kube_pod_status_phase{phase=~\"Failed|Pending\"} == 1) or vector(0)", + "legendFormat": "Failed/Pending", + "refId": "A" + } + ], + "type": "stat" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 12, "y": 1}, + "id": 5, + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none" + }, + "title": "PVCs Bound", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "count(kube_persistentvolumeclaim_status_phase{phase=\"Bound\"} == 1)", + "legendFormat": "Bound", + "refId": "A" + } + ], + "type": "stat" + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5}, + "id": 10, + "title": "Ressources par nœud", + "type": "row" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, + "id": 11, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "CPU % — compute nodes", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}[5m])) * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false + }, + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, + "id": 12, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "RAM % — compute nodes", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "100 - (avg by (instance) (node_memory_MemAvailable_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}) / avg by (instance) (node_memory_MemTotal_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}) * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 14}, + "id": 20, + "title": "Workloads", + "type": "row" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 3}, + {"color": "red", "value": 10} + ] + }, + "custom": { + "align": "auto", + "displayMode": "auto" + } + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "Restarts"}, + "properties": [ + {"id": "custom.displayMode", "value": "color-background"}, + {"id": "thresholds", "value": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 3}, + {"color": "red", "value": 10} + ] + }} + ] + } + ] + }, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 15}, + "id": 21, + "options": { + "showHeader": true, + "footer": {"show": false} + }, + "title": "Pod Restarts (top 10)", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "topk(10, sum by (pod, namespace) (kube_pod_container_status_restarts_total))", + "legendFormat": "", + "refId": "A", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "renameByName": { + "namespace": "Namespace", + "pod": "Pod", + "Value": "Restarts" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": [{"desc": true, "displayName": "Restarts"}] + } + } + ], + "type": "table" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "drawStyle": "line", + "spanNulls": false, + "stacking": {"mode": "normal", "group": "A"} + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 15}, + "id": 22, + "options": { + "tooltip": {"mode": "multi"}, + "legend": {"displayMode": "list", "placement": "bottom"} + }, + "title": "Pod count by namespace", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "count by (namespace) (kube_pod_status_phase{phase=\"Running\"} == 1)", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 23}, + "id": 30, + "title": "Stockage", + "type": "row" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "thresholds": { + "mode": "absolute", + "steps": [{"color": "blue", "value": null}] + }, + "custom": { + "align": "auto", + "displayMode": "auto" + } + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "Phase"}, + "properties": [ + {"id": "custom.displayMode", "value": "color-background"}, + {"id": "mappings", "value": [ + {"options": {"Bound": {"color": "green", "index": 0}}, "type": "value"}, + {"options": {"Pending": {"color": "yellow", "index": 1}}, "type": "value"}, + {"options": {"Lost": {"color": "red", "index": 2}}, "type": "value"} + ]} + ] + }, + { + "matcher": {"id": "byName", "options": "Capacity"}, + "properties": [{"id": "unit", "value": "bytes"}] + } + ] + }, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 24}, + "id": 31, + "options": { + "showHeader": true, + "footer": {"show": false} + }, + "title": "PVC — name / namespace / phase / capacity", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "kube_persistentvolumeclaim_status_phase", + "legendFormat": "", + "refId": "A", + "instant": true, + "format": "table" + }, + { + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "expr": "kube_persistentvolumeclaim_resource_requests_storage_bytes", + "legendFormat": "", + "refId": "B", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": ["persistentvolumeclaim", "namespace", "phase", "Value #A", "Value #B"] + } + } + }, + { + "id": "organize", + "options": { + "renameByName": { + "persistentvolumeclaim": "PVC", + "namespace": "Namespace", + "phase": "Phase", + "Value #A": "Status", + "Value #B": "Capacity" + } + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["funk"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "type": "datasource" + } + ] + }, + "time": {"from": "now-3h", "to": "now"}, + "timepicker": {}, + "timezone": "Europe/Paris", + "title": "Funk — Kubernetes", + "uid": "funk-k8s1", + "version": 1 + }