mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-08 11:04:43 +02:00
feat: monitoring — dashboards Grafana + alertes Prometheus
Dashboards (ConfigMaps, auto-importés par sidecar Grafana) : - dashboard-kubernetes : cluster overview, pods, PVCs, restarts - dashboard-infrastructure : compute/storage-01/gpu-01 CPU/RAM/disque/réseau + GPU ROCm - dashboard-ai : llama-server GPU/CPU tokens/s, requêtes, VRAM, température GPU Alertes (PrometheusRule) : - alerts-node : CPU>90%, RAM>90%, disque>80%, storage-01/gpu-01 down, load élevé - alerts-k8s : NodeNotReady, PodCrashLoop, PodPending, DeploymentUnavailable, PVCNotBound, ArgoCDOutOfSync - alerts-ai : llama-server GPU/system/monitor down, GPU temp>80°C, VRAM>95%, queue élevée Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
alkatrazz
parent
83c10feae7
commit
18da44a89f
6 changed files with 2304 additions and 0 deletions
753
k8s/infra/monitoring/dashboards/dashboard-ai.yaml
Normal file
753
k8s/infra/monitoring/dashboards/dashboard-ai.yaml
Normal file
|
|
@ -0,0 +1,753 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: grafana-dashboard-ai
|
||||
namespace: monitoring
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
ai.json: |
|
||||
{
|
||||
"__inputs": [],
|
||||
"__requires": [],
|
||||
"annotations": {"list": []},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
|
||||
"id": 1,
|
||||
"title": "GPU — Qwen3-8B (llama-server GPU, port 1234)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "blue", "value": null}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 1},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Génération tokens/s (GPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "gen t/s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "blue", "value": null}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 1},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Prompt tokens/s (GPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "prompt t/s",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 2},
|
||||
{"color": "red", "value": 5}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 1},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Requêtes en cours (GPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:requests_processing{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "processing",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 1},
|
||||
{"color": "red", "value": 3}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 12, "y": 1},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Requêtes en attente (GPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:requests_deferred{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "deferred",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short",
|
||||
"min": 0,
|
||||
"decimals": 1
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 5},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Tokens/s — génération + prompt (GPU Qwen3-8B)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "génération t/s",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-gpu\"}",
|
||||
"legendFormat": "prompt t/s",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 5},
|
||||
"id": 12,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Tokens prédits (rate 5m) — GPU",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rate(llamacpp:tokens_predicted_total{job=\"llama-server-gpu\"}[5m])",
|
||||
"legendFormat": "tokens/s (5m rate)",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rate(llamacpp:prompt_tokens_total{job=\"llama-server-gpu\"}[5m])",
|
||||
"legendFormat": "prompt tokens/s (5m rate)",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 13},
|
||||
"id": 20,
|
||||
"title": "CPU — system + monitor (Qwen3-1.7B, ports 1236 / 1237)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{"color": "blue", "value": null}]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 14},
|
||||
"id": 21,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Génération t/s — system (CPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-system\"}",
|
||||
"legendFormat": "gen t/s system",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{"color": "blue", "value": null}]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 14},
|
||||
"id": 22,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Génération t/s — monitor (CPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:predicted_tokens_seconds{job=\"llama-server-monitor\"}",
|
||||
"legendFormat": "gen t/s monitor",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{"color": "blue", "value": null}]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 14},
|
||||
"id": 23,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Prompt t/s — system (CPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-system\"}",
|
||||
"legendFormat": "prompt t/s system",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{"color": "blue", "value": null}]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 12, "y": 14},
|
||||
"id": 24,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Prompt t/s — monitor (CPU)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:prompt_tokens_seconds{job=\"llama-server-monitor\"}",
|
||||
"legendFormat": "prompt t/s monitor",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 2},
|
||||
{"color": "red", "value": 5}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 16, "y": 14},
|
||||
"id": 25,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Requêtes en cours — system + monitor",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:requests_processing{job=~\"llama-server-system|llama-server-monitor\"}",
|
||||
"legendFormat": "{{job}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 1},
|
||||
{"color": "red", "value": 3}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 20, "y": 14},
|
||||
"id": 26,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Requêtes en attente — system + monitor",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:requests_deferred{job=~\"llama-server-system|llama-server-monitor\"}",
|
||||
"legendFormat": "{{job}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short",
|
||||
"min": 0,
|
||||
"decimals": 1
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 18},
|
||||
"id": 27,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Tokens/s — system + monitor (CPU Qwen3-1.7B)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:predicted_tokens_seconds{job=~\"llama-server-system|llama-server-monitor\"}",
|
||||
"legendFormat": "gen {{job}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "llamacpp:prompt_tokens_seconds{job=~\"llama-server-system|llama-server-monitor\"}",
|
||||
"legendFormat": "prompt {{job}}",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 18},
|
||||
"id": 28,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Tokens prédits (rate 5m) — system + monitor",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rate(llamacpp:tokens_predicted_total{job=~\"llama-server-system|llama-server-monitor\"}[5m])",
|
||||
"legendFormat": "tokens/s {{job}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 26},
|
||||
"id": 40,
|
||||
"title": "GPU Health (RX 6700XT — gfx1031)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 85}
|
||||
]
|
||||
},
|
||||
"unit": "celsius",
|
||||
"min": 0,
|
||||
"max": 110,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 27},
|
||||
"id": 41,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"title": "GPU Temperature (°C)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "GPU Temp",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 10737418240},
|
||||
{"color": "red", "value": 11877376000}
|
||||
]
|
||||
},
|
||||
"unit": "bytes",
|
||||
"min": 0,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 27},
|
||||
"id": 42,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"title": "VRAM Usage (GB used / 12.9 GB total)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "VRAM used",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 12, "y": 27},
|
||||
"id": 43,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"title": "GPU Utilization %",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_gpu_utilization_percent{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "GPU Util %",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"min": 0
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "GPU Temp (°C)"},
|
||||
"properties": [
|
||||
{"id": "unit", "value": "celsius"},
|
||||
{"id": "custom.axisPlacement", "value": "left"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "VRAM used (GB)"},
|
||||
"properties": [
|
||||
{"id": "unit", "value": "bytes"},
|
||||
{"id": "custom.axisPlacement", "value": "right"}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 35},
|
||||
"id": 44,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "GPU Temp + VRAM usage — gpu-01 (gfx1031)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "GPU Temp (°C)",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "VRAM used (GB)",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 38,
|
||||
"tags": ["funk"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"type": "datasource"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {"from": "now-3h", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "Europe/Paris",
|
||||
"title": "Funk — IA / Hermes",
|
||||
"uid": "funk-ai01",
|
||||
"version": 1
|
||||
}
|
||||
801
k8s/infra/monitoring/dashboards/dashboard-infrastructure.yaml
Normal file
801
k8s/infra/monitoring/dashboards/dashboard-infrastructure.yaml
Normal file
|
|
@ -0,0 +1,801 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: grafana-dashboard-infrastructure
|
||||
namespace: monitoring
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
infrastructure.json: |
|
||||
{
|
||||
"__inputs": [],
|
||||
"__requires": [],
|
||||
"annotations": {"list": []},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
|
||||
"id": 1,
|
||||
"title": "Compute nodes (compute-01 / 02 / 03)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 1},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "CPU % (avg compute)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "avg(100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}[5m])) * 100))",
|
||||
"legendFormat": "CPU avg",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 1},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "RAM % (avg compute)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "avg(100 - (node_memory_MemAvailable_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} / node_memory_MemTotal_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} * 100))",
|
||||
"legendFormat": "RAM avg",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 2},
|
||||
{"color": "red", "value": 4}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 1},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Load1 (avg compute)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "avg(node_load1{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"})",
|
||||
"legendFormat": "Load1",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 5},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "CPU % — compute-01/02/03",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}[5m])) * 100)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 5},
|
||||
"id": 12,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "RAM % — compute-01/02/03",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (node_memory_MemAvailable_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} / node_memory_MemTotal_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} * 100)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 5,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "Bps",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 5},
|
||||
"id": 13,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Network I/O — compute nodes",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "sum by (instance) (irate(node_network_receive_bytes_total{instance=~\"192\\.168\\.10\\.(11|12|13):9100\",device!~\"lo|veth.*|flannel.*|cni.*\"}[5m]))",
|
||||
"legendFormat": "rx {{instance}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "sum by (instance) (irate(node_network_transmit_bytes_total{instance=~\"192\\.168\\.10\\.(11|12|13):9100\",device!~\"lo|veth.*|flannel.*|cni.*\"}[5m]))",
|
||||
"legendFormat": "tx {{instance}}",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 13},
|
||||
"id": 20,
|
||||
"title": "storage-01 (gateway — RAID5 — NFS)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 14},
|
||||
"id": 21,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "CPU % — storage-01",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"storage-01\"}[5m])) * 100)",
|
||||
"legendFormat": "CPU",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 14},
|
||||
"id": 22,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "RAM % — storage-01",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (node_memory_MemAvailable_bytes{job=\"storage-01\"} / node_memory_MemTotal_bytes{job=\"storage-01\"} * 100)",
|
||||
"legendFormat": "RAM",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 14},
|
||||
"id": 23,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Disk /srv usage % — storage-01",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (node_filesystem_avail_bytes{job=\"storage-01\",mountpoint=\"/srv\",fstype!=\"tmpfs\"} / node_filesystem_size_bytes{job=\"storage-01\",mountpoint=\"/srv\",fstype!=\"tmpfs\"} * 100)",
|
||||
"legendFormat": "/srv",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 18},
|
||||
"id": 24,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "CPU — storage-01",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"storage-01\"}[5m])) * 100)",
|
||||
"legendFormat": "CPU %",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 18},
|
||||
"id": 25,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "RAM — storage-01",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (node_memory_MemAvailable_bytes{job=\"storage-01\"} / node_memory_MemTotal_bytes{job=\"storage-01\"} * 100)",
|
||||
"legendFormat": "RAM %",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 5,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "Bps",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 12, "y": 18},
|
||||
"id": 26,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Disk I/O — storage-01",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "sum(irate(node_disk_read_bytes_total{job=\"storage-01\"}[5m]))",
|
||||
"legendFormat": "read",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "sum(irate(node_disk_written_bytes_total{job=\"storage-01\"}[5m]))",
|
||||
"legendFormat": "write",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 5,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "Bps",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 18, "y": 18},
|
||||
"id": 27,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Network I/O — storage-01",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "sum(irate(node_network_receive_bytes_total{job=\"storage-01\",device!~\"lo|veth.*\"}[5m]))",
|
||||
"legendFormat": "rx",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "sum(irate(node_network_transmit_bytes_total{job=\"storage-01\",device!~\"lo|veth.*\"}[5m]))",
|
||||
"legendFormat": "tx",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 26},
|
||||
"id": 30,
|
||||
"title": "gpu-01 (RX 6700XT — ROCm — llama-server)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 27},
|
||||
"id": 31,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "CPU % — gpu-01",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"gpu-01-node\"}[5m])) * 100)",
|
||||
"legendFormat": "CPU",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 27},
|
||||
"id": 32,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "RAM % — gpu-01",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (node_memory_MemAvailable_bytes{job=\"gpu-01-node\"} / node_memory_MemTotal_bytes{job=\"gpu-01-node\"} * 100)",
|
||||
"legendFormat": "RAM",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 85}
|
||||
]
|
||||
},
|
||||
"unit": "celsius",
|
||||
"min": 0,
|
||||
"max": 110,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 31},
|
||||
"id": 33,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"title": "GPU Temperature (°C)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "GPU Temp",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 80},
|
||||
{"color": "red", "value": 92}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 31},
|
||||
"id": 34,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"title": "VRAM Usage %",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"} / rocm_vram_total_bytes{job=\"gpu-01-node\",gpu=\"0\"} * 100",
|
||||
"legendFormat": "VRAM %",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 12, "y": 31},
|
||||
"id": 35,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"title": "GPU Utilization %",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_gpu_utilization_percent{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "GPU Util",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"min": 0
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "GPU Temp (°C)"},
|
||||
"properties": [{"id": "unit", "value": "celsius"}, {"id": "custom.axisPlacement", "value": "left"}]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "VRAM %"},
|
||||
"properties": [{"id": "unit", "value": "percent"}, {"id": "custom.axisPlacement", "value": "right"}, {"id": "max", "value": 100}]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 39},
|
||||
"id": 36,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "GPU Temp + VRAM % — gpu-01",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}",
|
||||
"legendFormat": "GPU Temp (°C)",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"} / rocm_vram_total_bytes{job=\"gpu-01-node\",gpu=\"0\"} * 100",
|
||||
"legendFormat": "VRAM %",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 38,
|
||||
"tags": ["funk"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"type": "datasource"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {"from": "now-3h", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "Europe/Paris",
|
||||
"title": "Funk — Infrastructure",
|
||||
"uid": "funk-inf1",
|
||||
"version": 1
|
||||
}
|
||||
476
k8s/infra/monitoring/dashboards/dashboard-kubernetes.yaml
Normal file
476
k8s/infra/monitoring/dashboards/dashboard-kubernetes.yaml
Normal file
|
|
@ -0,0 +1,476 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: grafana-dashboard-kubernetes
|
||||
namespace: monitoring
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
kubernetes.json: |
|
||||
{
|
||||
"__inputs": [],
|
||||
"__requires": [],
|
||||
"annotations": {"list": []},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
|
||||
"id": 1,
|
||||
"title": "Cluster",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "red", "value": null},
|
||||
{"color": "green", "value": 1}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 1},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Nodes Ready",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "count(kube_node_status_condition{condition=\"Ready\",status=\"true\"} == 1)",
|
||||
"legendFormat": "Ready",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "blue", "value": null}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 1},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Pods Running",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "count(kube_pod_status_phase{phase=\"Running\"} == 1)",
|
||||
"legendFormat": "Running",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 1},
|
||||
{"color": "red", "value": 3}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 1},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Pods Failed / Pending",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "count(kube_pod_status_phase{phase=~\"Failed|Pending\"} == 1) or vector(0)",
|
||||
"legendFormat": "Failed/Pending",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "red", "value": null},
|
||||
{"color": "green", "value": 1}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 12, "y": 1},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "PVCs Bound",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "count(kube_persistentvolumeclaim_status_phase{phase=\"Bound\"} == 1)",
|
||||
"legendFormat": "Bound",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 5},
|
||||
"id": 10,
|
||||
"title": "Ressources par nœud",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "CPU % — compute nodes",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}[5m])) * 100)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
|
||||
"id": 12,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "RAM % — compute nodes",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (avg by (instance) (node_memory_MemAvailable_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}) / avg by (instance) (node_memory_MemTotal_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}) * 100)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 14},
|
||||
"id": 20,
|
||||
"title": "Workloads",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 3},
|
||||
{"color": "red", "value": 10}
|
||||
]
|
||||
},
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"displayMode": "auto"
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "Restarts"},
|
||||
"properties": [
|
||||
{"id": "custom.displayMode", "value": "color-background"},
|
||||
{"id": "thresholds", "value": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 3},
|
||||
{"color": "red", "value": 10}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 15},
|
||||
"id": 21,
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"footer": {"show": false}
|
||||
},
|
||||
"title": "Pod Restarts (top 10)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "topk(10, sum by (pod, namespace) (kube_pod_container_status_restarts_total))",
|
||||
"legendFormat": "",
|
||||
"refId": "A",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"renameByName": {
|
||||
"namespace": "Namespace",
|
||||
"pod": "Pod",
|
||||
"Value": "Restarts"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "sortBy",
|
||||
"options": {
|
||||
"fields": [{"desc": true, "displayName": "Restarts"}]
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false,
|
||||
"stacking": {"mode": "normal", "group": "A"}
|
||||
},
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 15},
|
||||
"id": 22,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Pod count by namespace",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "count by (namespace) (kube_pod_status_phase{phase=\"Running\"} == 1)",
|
||||
"legendFormat": "{{namespace}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 23},
|
||||
"id": 30,
|
||||
"title": "Stockage",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{"color": "blue", "value": null}]
|
||||
},
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"displayMode": "auto"
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "Phase"},
|
||||
"properties": [
|
||||
{"id": "custom.displayMode", "value": "color-background"},
|
||||
{"id": "mappings", "value": [
|
||||
{"options": {"Bound": {"color": "green", "index": 0}}, "type": "value"},
|
||||
{"options": {"Pending": {"color": "yellow", "index": 1}}, "type": "value"},
|
||||
{"options": {"Lost": {"color": "red", "index": 2}}, "type": "value"}
|
||||
]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "Capacity"},
|
||||
"properties": [{"id": "unit", "value": "bytes"}]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 24},
|
||||
"id": 31,
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"footer": {"show": false}
|
||||
},
|
||||
"title": "PVC — name / namespace / phase / capacity",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "kube_persistentvolumeclaim_status_phase",
|
||||
"legendFormat": "",
|
||||
"refId": "A",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "kube_persistentvolumeclaim_resource_requests_storage_bytes",
|
||||
"legendFormat": "",
|
||||
"refId": "B",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"id": "filterFieldsByName",
|
||||
"options": {
|
||||
"include": {
|
||||
"names": ["persistentvolumeclaim", "namespace", "phase", "Value #A", "Value #B"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"renameByName": {
|
||||
"persistentvolumeclaim": "PVC",
|
||||
"namespace": "Namespace",
|
||||
"phase": "Phase",
|
||||
"Value #A": "Status",
|
||||
"Value #B": "Capacity"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 38,
|
||||
"tags": ["funk"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"type": "datasource"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {"from": "now-3h", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "Europe/Paris",
|
||||
"title": "Funk — Kubernetes",
|
||||
"uid": "funk-k8s1",
|
||||
"version": 1
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue