mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-08 10:04:42 +02:00
18da44a89f
Dashboards (ConfigMaps, auto-importés par sidecar Grafana) : - dashboard-kubernetes : cluster overview, pods, PVCs, restarts - dashboard-infrastructure : compute/storage-01/gpu-01 CPU/RAM/disque/réseau + GPU ROCm - dashboard-ai : llama-server GPU/CPU tokens/s, requêtes, VRAM, température GPU Alertes (PrometheusRule) : - alerts-node : CPU>90%, RAM>90%, disque>80%, storage-01/gpu-01 down, load élevé - alerts-k8s : NodeNotReady, PodCrashLoop, PodPending, DeploymentUnavailable, PVCNotBound, ArgoCDOutOfSync - alerts-ai : llama-server GPU/system/monitor down, GPU temp>80°C, VRAM>95%, queue élevée Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
753 lines
25 KiB
YAML
753 lines
25 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: grafana-dashboard-ai
|
|
namespace: monitoring
|
|
labels:
|
|
grafana_dashboard: "1"
|
|
data:
|
|
ai.json: |
|
|
{
|
|
"__inputs": [],
|
|
"__requires": [],
|
|
"annotations": {"list": []},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [],
|
|
"panels": [
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
|
|
"id": 1,
|
|
"title": "GPU — Qwen3-8B (llama-server GPU, port 1234)",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "blue", "value": null}
|
|
]
|
|
},
|
|
"unit": "short",
|
|
"decimals": 1,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 1},
|
|
"id": 2,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "area"
|
|
},
|
|
"title": "Génération tokens/s (GPU)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-gpu\"}",
|
|
"legendFormat": "gen t/s",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "blue", "value": null}
|
|
]
|
|
},
|
|
"unit": "short",
|
|
"decimals": 1,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 1},
|
|
"id": 3,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "area"
|
|
},
|
|
"title": "Prompt tokens/s (GPU)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-gpu\"}",
|
|
"legendFormat": "prompt t/s",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 2},
|
|
{"color": "red", "value": 5}
|
|
]
|
|
},
|
|
"unit": "short",
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 1},
|
|
"id": 4,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "Requêtes en cours (GPU)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:requests_processing{job=\"llama-server-gpu\"}",
|
|
"legendFormat": "processing",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 1},
|
|
{"color": "red", "value": 3}
|
|
]
|
|
},
|
|
"unit": "short",
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 12, "y": 1},
|
|
"id": 5,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "Requêtes en attente (GPU)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:requests_deferred{job=\"llama-server-gpu\"}",
|
|
"legendFormat": "deferred",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 2,
|
|
"fillOpacity": 10,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"unit": "short",
|
|
"min": 0,
|
|
"decimals": 1
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 5},
|
|
"id": 11,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "Tokens/s — génération + prompt (GPU Qwen3-8B)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-gpu\"}",
|
|
"legendFormat": "génération t/s",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-gpu\"}",
|
|
"legendFormat": "prompt t/s",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"unit": "short",
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 5},
|
|
"id": 12,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "Tokens prédits (rate 5m) — GPU",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rate(llamacpp:tokens_predicted_total{job=\"llama-server-gpu\"}[5m])",
|
|
"legendFormat": "tokens/s (5m rate)",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rate(llamacpp:prompt_tokens_total{job=\"llama-server-gpu\"}[5m])",
|
|
"legendFormat": "prompt tokens/s (5m rate)",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 13},
|
|
"id": 20,
|
|
"title": "CPU — system + monitor (Qwen3-1.7B, ports 1236 / 1237)",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [{"color": "blue", "value": null}]
|
|
},
|
|
"unit": "short",
|
|
"decimals": 1,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 14},
|
|
"id": 21,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "area"
|
|
},
|
|
"title": "Génération t/s — system (CPU)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-system\"}",
|
|
"legendFormat": "gen t/s system",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [{"color": "blue", "value": null}]
|
|
},
|
|
"unit": "short",
|
|
"decimals": 1,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 14},
|
|
"id": 22,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "area"
|
|
},
|
|
"title": "Génération t/s — monitor (CPU)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-monitor\"}",
|
|
"legendFormat": "gen t/s monitor",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [{"color": "blue", "value": null}]
|
|
},
|
|
"unit": "short",
|
|
"decimals": 1,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 14},
|
|
"id": 23,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "area"
|
|
},
|
|
"title": "Prompt t/s — system (CPU)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-system\"}",
|
|
"legendFormat": "prompt t/s system",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [{"color": "blue", "value": null}]
|
|
},
|
|
"unit": "short",
|
|
"decimals": 1,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 12, "y": 14},
|
|
"id": 24,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "area"
|
|
},
|
|
"title": "Prompt t/s — monitor (CPU)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-monitor\"}",
|
|
"legendFormat": "prompt t/s monitor",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 2},
|
|
{"color": "red", "value": 5}
|
|
]
|
|
},
|
|
"unit": "short",
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 16, "y": 14},
|
|
"id": 25,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "Requêtes en cours — system + monitor",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:requests_processing{job=~\"llama-server-system|llama-server-monitor\"}",
|
|
"legendFormat": "{{job}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 1},
|
|
{"color": "red", "value": 3}
|
|
]
|
|
},
|
|
"unit": "short",
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 20, "y": 14},
|
|
"id": 26,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "Requêtes en attente — system + monitor",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:requests_deferred{job=~\"llama-server-system|llama-server-monitor\"}",
|
|
"legendFormat": "{{job}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 2,
|
|
"fillOpacity": 10,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"unit": "short",
|
|
"min": 0,
|
|
"decimals": 1
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 18},
|
|
"id": 27,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "Tokens/s — system + monitor (CPU Qwen3-1.7B)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:predicted_tokens_seconds{job=~\"llama-server-system|llama-server-monitor\"}",
|
|
"legendFormat": "gen {{job}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "llamacpp:prompt_tokens_seconds{job=~\"llama-server-system|llama-server-monitor\"}",
|
|
"legendFormat": "prompt {{job}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"unit": "short",
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 18},
|
|
"id": 28,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "Tokens prédits (rate 5m) — system + monitor",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rate(llamacpp:tokens_predicted_total{job=~\"llama-server-system|llama-server-monitor\"}[5m])",
|
|
"legendFormat": "tokens/s {{job}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 26},
|
|
"id": 40,
|
|
"title": "GPU Health (RX 6700XT — gfx1031)",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 70},
|
|
{"color": "red", "value": 85}
|
|
]
|
|
},
|
|
"unit": "celsius",
|
|
"min": 0,
|
|
"max": 110,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 27},
|
|
"id": 41,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"showThresholdLabels": false,
|
|
"showThresholdMarkers": true
|
|
},
|
|
"title": "GPU Temperature (°C)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}",
|
|
"legendFormat": "GPU Temp",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "gauge"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 10737418240},
|
|
{"color": "red", "value": 11877376000}
|
|
]
|
|
},
|
|
"unit": "bytes",
|
|
"min": 0,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 27},
|
|
"id": 42,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"showThresholdLabels": false,
|
|
"showThresholdMarkers": true
|
|
},
|
|
"title": "VRAM Usage (GB used / 12.9 GB total)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"}",
|
|
"legendFormat": "VRAM used",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "gauge"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 70},
|
|
{"color": "red", "value": 90}
|
|
]
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 6, "x": 12, "y": 27},
|
|
"id": 43,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"showThresholdLabels": false,
|
|
"showThresholdMarkers": true
|
|
},
|
|
"title": "GPU Utilization %",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rocm_gpu_utilization_percent{job=\"gpu-01-node\",gpu=\"0\"}",
|
|
"legendFormat": "GPU Util %",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "gauge"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {"id": "byName", "options": "GPU Temp (°C)"},
|
|
"properties": [
|
|
{"id": "unit", "value": "celsius"},
|
|
{"id": "custom.axisPlacement", "value": "left"}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {"id": "byName", "options": "VRAM used (GB)"},
|
|
"properties": [
|
|
{"id": "unit", "value": "bytes"},
|
|
{"id": "custom.axisPlacement", "value": "right"}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 35},
|
|
"id": 44,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "GPU Temp + VRAM usage — gpu-01 (gfx1031)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}",
|
|
"legendFormat": "GPU Temp (°C)",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"}",
|
|
"legendFormat": "VRAM used (GB)",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"refresh": "30s",
|
|
"schemaVersion": 38,
|
|
"tags": ["funk"],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {},
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "Datasource",
|
|
"name": "datasource",
|
|
"options": [],
|
|
"query": "prometheus",
|
|
"refresh": 1,
|
|
"type": "datasource"
|
|
}
|
|
]
|
|
},
|
|
"time": {"from": "now-3h", "to": "now"},
|
|
"timepicker": {},
|
|
"timezone": "Europe/Paris",
|
|
"title": "Funk — IA / Hermes",
|
|
"uid": "funk-ai01",
|
|
"version": 1
|
|
}
|