mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-15 04:24:43 +02:00
feat: monitoring — dashboards Grafana + alertes Prometheus
Dashboards (ConfigMaps, auto-importés par sidecar Grafana) : - dashboard-kubernetes : cluster overview, pods, PVCs, restarts - dashboard-infrastructure : compute/storage-01/gpu-01 CPU/RAM/disque/réseau + GPU ROCm - dashboard-ai : llama-server GPU/CPU tokens/s, requêtes, VRAM, température GPU Alertes (PrometheusRule) : - alerts-node : CPU>90%, RAM>90%, disque>80%, storage-01/gpu-01 down, load élevé - alerts-k8s : NodeNotReady, PodCrashLoop, PodPending, DeploymentUnavailable, PVCNotBound, ArgoCDOutOfSync - alerts-ai : llama-server GPU/system/monitor down, GPU temp>80°C, VRAM>95%, queue élevée Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
alkatrazz
parent
83c10feae7
commit
18da44a89f
6 changed files with 2304 additions and 0 deletions
753
k8s/infra/monitoring/dashboards/dashboard-ai.yaml
Normal file
753
k8s/infra/monitoring/dashboards/dashboard-ai.yaml
Normal file
|
|
@ -0,0 +1,753 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: grafana-dashboard-ai
|
||||
namespace: monitoring
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
ai.json: |
|
||||
{
|
||||
"__inputs": [],
|
||||
"__requires": [],
|
||||
"annotations": {"list": []},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
|
||||
"id": 1,
|
||||
"title": "GPU — Qwen3-8B (llama-server GPU, port 1234)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "blue", "value": null}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 1},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Génération tokens/s (GPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "gen t/s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "blue", "value": null}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 1},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Prompt tokens/s (GPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "prompt t/s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 2},
|
||||
{"color": "red", "value": 5}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 1},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Requêtes en cours (GPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:requests_processing{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "processing",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 1},
|
||||
{"color": "red", "value": 3}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 12, "y": 1},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Requêtes en attente (GPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:requests_deferred{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "deferred",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short",
|
||||
"min": 0,
|
||||
"decimals": 1
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 5},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Tokens/s — génération + prompt (GPU Qwen3-8B)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "génération t/s",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "prompt t/s",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 5},
|
||||
"id": 12,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Tokens prédits (rate 5m) — GPU",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rate(llamacpp:tokens_predicted_total{job=\"llama-server-gpu\"}[5m])",
|
||||
"legendFormat": "tokens/s (5m rate)",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rate(llamacpp:prompt_tokens_total{job=\"llama-server-gpu\"}[5m])",
|
||||
"legendFormat": "prompt tokens/s (5m rate)",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 13},
|
||||
"id": 20,
|
||||
"title": "CPU — system + monitor (Qwen3-1.7B, ports 1236 / 1237)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{"color": "blue", "value": null}]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 14},
|
||||
"id": 21,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Génération t/s — system (CPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-system\"}",
|
||||
"legendFormat": "gen t/s system",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{"color": "blue", "value": null}]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 14},
|
||||
"id": 22,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Génération t/s — monitor (CPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-monitor\"}",
|
||||
"legendFormat": "gen t/s monitor",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{"color": "blue", "value": null}]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 14},
|
||||
"id": 23,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Prompt t/s — system (CPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-system\"}",
|
||||
"legendFormat": "prompt t/s system",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{"color": "blue", "value": null}]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 12, "y": 14},
|
||||
"id": 24,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Prompt t/s — monitor (CPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-monitor\"}",
|
||||
"legendFormat": "prompt t/s monitor",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 2},
|
||||
{"color": "red", "value": 5}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 16, "y": 14},
|
||||
"id": 25,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Requêtes en cours — system + monitor",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:requests_processing{job=~\"llama-server-system|llama-server-monitor\"}",
|
||||
"legendFormat": "{{job}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 1},
|
||||
{"color": "red", "value": 3}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 20, "y": 14},
|
||||
"id": 26,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Requêtes en attente — system + monitor",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:requests_deferred{job=~\"llama-server-system|llama-server-monitor\"}",
|
||||
"legendFormat": "{{job}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short",
|
||||
"min": 0,
|
||||
"decimals": 1
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 18},
|
||||
"id": 27,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Tokens/s — system + monitor (CPU Qwen3-1.7B)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:predicted_tokens_seconds{job=~\"llama-server-system|llama-server-monitor\"}",
|
||||
"legendFormat": "gen {{job}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:prompt_tokens_seconds{job=~\"llama-server-system|llama-server-monitor\"}",
|
||||
"legendFormat": "prompt {{job}}",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 18},
|
||||
"id": 28,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Tokens prédits (rate 5m) — system + monitor",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rate(llamacpp:tokens_predicted_total{job=~\"llama-server-system|llama-server-monitor\"}[5m])",
|
||||
"legendFormat": "tokens/s {{job}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 26},
|
||||
"id": 40,
|
||||
"title": "GPU Health (RX 6700XT — gfx1031)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 85}
|
||||
]
|
||||
},
|
||||
"unit": "celsius",
|
||||
"min": 0,
|
||||
"max": 110,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 27},
|
||||
"id": 41,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"title": "GPU Temperature (°C)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "GPU Temp",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 10737418240},
|
||||
{"color": "red", "value": 11877376000}
|
||||
]
|
||||
},
|
||||
"unit": "bytes",
|
||||
"min": 0,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 27},
|
||||
"id": 42,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"title": "VRAM Usage (GB used / 12.9 GB total)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "VRAM used",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 12, "y": 27},
|
||||
"id": 43,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"title": "GPU Utilization %",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_gpu_utilization_percent{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "GPU Util %",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"min": 0
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "GPU Temp (°C)"},
|
||||
"properties": [
|
||||
{"id": "unit", "value": "celsius"},
|
||||
{"id": "custom.axisPlacement", "value": "left"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "VRAM used (GB)"},
|
||||
"properties": [
|
||||
{"id": "unit", "value": "bytes"},
|
||||
{"id": "custom.axisPlacement", "value": "right"}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 35},
|
||||
"id": 44,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "GPU Temp + VRAM usage — gpu-01 (gfx1031)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "GPU Temp (°C)",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "VRAM used (GB)",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 38,
|
||||
"tags": ["funk"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"type": "datasource"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {"from": "now-3h", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "Europe/Paris",
|
||||
"title": "Funk — IA / Hermes",
|
||||
"uid": "funk-ai01",
|
||||
"version": 1
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue