mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-08 16:54:43 +02:00
18da44a89f
Dashboards (ConfigMaps, auto-importés par sidecar Grafana) : - dashboard-kubernetes : cluster overview, pods, PVCs, restarts - dashboard-infrastructure : compute/storage-01/gpu-01 CPU/RAM/disque/réseau + GPU ROCm - dashboard-ai : llama-server GPU/CPU tokens/s, requêtes, VRAM, température GPU Alertes (PrometheusRule) : - alerts-node : CPU>90%, RAM>90%, disque>80%, storage-01/gpu-01 down, load élevé - alerts-k8s : NodeNotReady, PodCrashLoop, PodPending, DeploymentUnavailable, PVCNotBound, ArgoCDOutOfSync - alerts-ai : llama-server GPU/system/monitor down, GPU temp>80°C, VRAM>95%, queue élevée Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
801 lines
27 KiB
YAML
801 lines
27 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: grafana-dashboard-infrastructure
|
|
namespace: monitoring
|
|
labels:
|
|
grafana_dashboard: "1"
|
|
data:
|
|
infrastructure.json: |
|
|
{
|
|
"__inputs": [],
|
|
"__requires": [],
|
|
"annotations": {"list": []},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [],
|
|
"panels": [
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
|
|
"id": 1,
|
|
"title": "Compute nodes (compute-01 / 02 / 03)",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 70},
|
|
{"color": "red", "value": 90}
|
|
]
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 1},
|
|
"id": 2,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "CPU % (avg compute)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "avg(100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}[5m])) * 100))",
|
|
"legendFormat": "CPU avg",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 70},
|
|
{"color": "red", "value": 90}
|
|
]
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 1},
|
|
"id": 3,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "RAM % (avg compute)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "avg(100 - (node_memory_MemAvailable_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} / node_memory_MemTotal_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} * 100))",
|
|
"legendFormat": "RAM avg",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 2},
|
|
{"color": "red", "value": 4}
|
|
]
|
|
},
|
|
"unit": "short",
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 1},
|
|
"id": 4,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "Load1 (avg compute)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "avg(node_load1{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"})",
|
|
"legendFormat": "Load1",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 5},
|
|
"id": 11,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "CPU % — compute-01/02/03",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}[5m])) * 100)",
|
|
"legendFormat": "{{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 5},
|
|
"id": 12,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "RAM % — compute-01/02/03",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "100 - (node_memory_MemAvailable_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} / node_memory_MemTotal_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} * 100)",
|
|
"legendFormat": "{{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 1,
|
|
"fillOpacity": 5,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"unit": "Bps",
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 5},
|
|
"id": 13,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "Network I/O — compute nodes",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "sum by (instance) (irate(node_network_receive_bytes_total{instance=~\"192\\.168\\.10\\.(11|12|13):9100\",device!~\"lo|veth.*|flannel.*|cni.*\"}[5m]))",
|
|
"legendFormat": "rx {{instance}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "sum by (instance) (irate(node_network_transmit_bytes_total{instance=~\"192\\.168\\.10\\.(11|12|13):9100\",device!~\"lo|veth.*|flannel.*|cni.*\"}[5m]))",
|
|
"legendFormat": "tx {{instance}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 13},
|
|
"id": 20,
|
|
"title": "storage-01 (gateway — RAID5 — NFS)",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 70},
|
|
{"color": "red", "value": 90}
|
|
]
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 14},
|
|
"id": 21,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "CPU % — storage-01",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"storage-01\"}[5m])) * 100)",
|
|
"legendFormat": "CPU",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 70},
|
|
{"color": "red", "value": 90}
|
|
]
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 14},
|
|
"id": 22,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "RAM % — storage-01",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "100 - (node_memory_MemAvailable_bytes{job=\"storage-01\"} / node_memory_MemTotal_bytes{job=\"storage-01\"} * 100)",
|
|
"legendFormat": "RAM",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 70},
|
|
{"color": "red", "value": 90}
|
|
]
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 14},
|
|
"id": 23,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "Disk /srv usage % — storage-01",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "100 - (node_filesystem_avail_bytes{job=\"storage-01\",mountpoint=\"/srv\",fstype!=\"tmpfs\"} / node_filesystem_size_bytes{job=\"storage-01\",mountpoint=\"/srv\",fstype!=\"tmpfs\"} * 100)",
|
|
"legendFormat": "/srv",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 18},
|
|
"id": 24,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "CPU — storage-01",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"storage-01\"}[5m])) * 100)",
|
|
"legendFormat": "CPU %",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 18},
|
|
"id": 25,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "RAM — storage-01",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "100 - (node_memory_MemAvailable_bytes{job=\"storage-01\"} / node_memory_MemTotal_bytes{job=\"storage-01\"} * 100)",
|
|
"legendFormat": "RAM %",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 1,
|
|
"fillOpacity": 5,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"unit": "Bps",
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 6, "x": 12, "y": 18},
|
|
"id": 26,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "Disk I/O — storage-01",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "sum(irate(node_disk_read_bytes_total{job=\"storage-01\"}[5m]))",
|
|
"legendFormat": "read",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "sum(irate(node_disk_written_bytes_total{job=\"storage-01\"}[5m]))",
|
|
"legendFormat": "write",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 1,
|
|
"fillOpacity": 5,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"unit": "Bps",
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 6, "x": 18, "y": 18},
|
|
"id": 27,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "Network I/O — storage-01",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "sum(irate(node_network_receive_bytes_total{job=\"storage-01\",device!~\"lo|veth.*\"}[5m]))",
|
|
"legendFormat": "rx",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "sum(irate(node_network_transmit_bytes_total{job=\"storage-01\",device!~\"lo|veth.*\"}[5m]))",
|
|
"legendFormat": "tx",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 26},
|
|
"id": 30,
|
|
"title": "gpu-01 (RX 6700XT — ROCm — llama-server)",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 70},
|
|
{"color": "red", "value": 90}
|
|
]
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 27},
|
|
"id": 31,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "CPU % — gpu-01",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"gpu-01-node\"}[5m])) * 100)",
|
|
"legendFormat": "CPU",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 70},
|
|
{"color": "red", "value": 90}
|
|
]
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 27},
|
|
"id": 32,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none"
|
|
},
|
|
"title": "RAM % — gpu-01",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "100 - (node_memory_MemAvailable_bytes{job=\"gpu-01-node\"} / node_memory_MemTotal_bytes{job=\"gpu-01-node\"} * 100)",
|
|
"legendFormat": "RAM",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 70},
|
|
{"color": "red", "value": 85}
|
|
]
|
|
},
|
|
"unit": "celsius",
|
|
"min": 0,
|
|
"max": 110,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 31},
|
|
"id": 33,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"showThresholdLabels": false,
|
|
"showThresholdMarkers": true
|
|
},
|
|
"title": "GPU Temperature (°C)",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}",
|
|
"legendFormat": "GPU Temp",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "gauge"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 80},
|
|
{"color": "red", "value": 92}
|
|
]
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 31},
|
|
"id": 34,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"showThresholdLabels": false,
|
|
"showThresholdMarkers": true
|
|
},
|
|
"title": "VRAM Usage %",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"} / rocm_vram_total_bytes{job=\"gpu-01-node\",gpu=\"0\"} * 100",
|
|
"legendFormat": "VRAM %",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "gauge"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": null},
|
|
{"color": "yellow", "value": 70},
|
|
{"color": "red", "value": 90}
|
|
]
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 6, "x": 12, "y": 31},
|
|
"id": 35,
|
|
"options": {
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
|
"orientation": "auto",
|
|
"showThresholdLabels": false,
|
|
"showThresholdMarkers": true
|
|
},
|
|
"title": "GPU Utilization %",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rocm_gpu_utilization_percent{job=\"gpu-01-node\",gpu=\"0\"}",
|
|
"legendFormat": "GPU Util",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"type": "gauge"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"drawStyle": "line",
|
|
"spanNulls": false
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {"id": "byName", "options": "GPU Temp (°C)"},
|
|
"properties": [{"id": "unit", "value": "celsius"}, {"id": "custom.axisPlacement", "value": "left"}]
|
|
},
|
|
{
|
|
"matcher": {"id": "byName", "options": "VRAM %"},
|
|
"properties": [{"id": "unit", "value": "percent"}, {"id": "custom.axisPlacement", "value": "right"}, {"id": "max", "value": 100}]
|
|
}
|
|
]
|
|
},
|
|
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 39},
|
|
"id": 36,
|
|
"options": {
|
|
"tooltip": {"mode": "multi"},
|
|
"legend": {"displayMode": "list", "placement": "bottom"}
|
|
},
|
|
"title": "GPU Temp + VRAM % — gpu-01",
|
|
"targets": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}",
|
|
"legendFormat": "GPU Temp (°C)",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
|
"expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"} / rocm_vram_total_bytes{job=\"gpu-01-node\",gpu=\"0\"} * 100",
|
|
"legendFormat": "VRAM %",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"refresh": "30s",
|
|
"schemaVersion": 38,
|
|
"tags": ["funk"],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {},
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "Datasource",
|
|
"name": "datasource",
|
|
"options": [],
|
|
"query": "prometheus",
|
|
"refresh": 1,
|
|
"type": "datasource"
|
|
}
|
|
]
|
|
},
|
|
"time": {"from": "now-3h", "to": "now"},
|
|
"timepicker": {},
|
|
"timezone": "Europe/Paris",
|
|
"title": "Funk — Infrastructure",
|
|
"uid": "funk-inf1",
|
|
"version": 1
|
|
}
|