mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-15 06:14:42 +02:00
feat: monitoring — dashboards Grafana + alertes Prometheus
Dashboards (ConfigMaps, auto-importés par sidecar Grafana) : - dashboard-kubernetes : cluster overview, pods, PVCs, restarts - dashboard-infrastructure : compute/storage-01/gpu-01 CPU/RAM/disque/réseau + GPU ROCm - dashboard-ai : llama-server GPU/CPU tokens/s, requêtes, VRAM, température GPU Alertes (PrometheusRule) : - alerts-node : CPU>90%, RAM>90%, disque>80%, storage-01/gpu-01 down, load élevé - alerts-k8s : NodeNotReady, PodCrashLoop, PodPending, DeploymentUnavailable, PVCNotBound, ArgoCDOutOfSync - alerts-ai : llama-server GPU/system/monitor down, GPU temp>80°C, VRAM>95%, queue élevée Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
alkatrazz
parent
83c10feae7
commit
18da44a89f
6 changed files with 2304 additions and 0 deletions
476
k8s/infra/monitoring/dashboards/dashboard-kubernetes.yaml
Normal file
476
k8s/infra/monitoring/dashboards/dashboard-kubernetes.yaml
Normal file
|
|
@ -0,0 +1,476 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: grafana-dashboard-kubernetes
|
||||
namespace: monitoring
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
kubernetes.json: |
|
||||
{
|
||||
"__inputs": [],
|
||||
"__requires": [],
|
||||
"annotations": {"list": []},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
|
||||
"id": 1,
|
||||
"title": "Cluster",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "red", "value": null},
|
||||
{"color": "green", "value": 1}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 1},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Nodes Ready",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "count(kube_node_status_condition{condition=\"Ready\",status=\"true\"} == 1)",
|
||||
"legendFormat": "Ready",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "blue", "value": null}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 1},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Pods Running",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "count(kube_pod_status_phase{phase=\"Running\"} == 1)",
|
||||
"legendFormat": "Running",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 1},
|
||||
{"color": "red", "value": 3}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 1},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "Pods Failed / Pending",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "count(kube_pod_status_phase{phase=~\"Failed|Pending\"} == 1) or vector(0)",
|
||||
"legendFormat": "Failed/Pending",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "red", "value": null},
|
||||
{"color": "green", "value": 1}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 12, "y": 1},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"title": "PVCs Bound",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "count(kube_persistentvolumeclaim_status_phase{phase=\"Bound\"} == 1)",
|
||||
"legendFormat": "Bound",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 5},
|
||||
"id": 10,
|
||||
"title": "Ressources par nœud",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "CPU % — compute nodes",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}[5m])) * 100)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
|
||||
"id": 12,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "RAM % — compute nodes",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "100 - (avg by (instance) (node_memory_MemAvailable_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}) / avg by (instance) (node_memory_MemTotal_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}) * 100)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 14},
|
||||
"id": 20,
|
||||
"title": "Workloads",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 3},
|
||||
{"color": "red", "value": 10}
|
||||
]
|
||||
},
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"displayMode": "auto"
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "Restarts"},
|
||||
"properties": [
|
||||
{"id": "custom.displayMode", "value": "color-background"},
|
||||
{"id": "thresholds", "value": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 3},
|
||||
{"color": "red", "value": 10}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 15},
|
||||
"id": 21,
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"footer": {"show": false}
|
||||
},
|
||||
"title": "Pod Restarts (top 10)",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "topk(10, sum by (pod, namespace) (kube_pod_container_status_restarts_total))",
|
||||
"legendFormat": "",
|
||||
"refId": "A",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"renameByName": {
|
||||
"namespace": "Namespace",
|
||||
"pod": "Pod",
|
||||
"Value": "Restarts"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "sortBy",
|
||||
"options": {
|
||||
"fields": [{"desc": true, "displayName": "Restarts"}]
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"drawStyle": "line",
|
||||
"spanNulls": false,
|
||||
"stacking": {"mode": "normal", "group": "A"}
|
||||
},
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 15},
|
||||
"id": 22,
|
||||
"options": {
|
||||
"tooltip": {"mode": "multi"},
|
||||
"legend": {"displayMode": "list", "placement": "bottom"}
|
||||
},
|
||||
"title": "Pod count by namespace",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "count by (namespace) (kube_pod_status_phase{phase=\"Running\"} == 1)",
|
||||
"legendFormat": "{{namespace}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 23},
|
||||
"id": 30,
|
||||
"title": "Stockage",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{"color": "blue", "value": null}]
|
||||
},
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"displayMode": "auto"
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "Phase"},
|
||||
"properties": [
|
||||
{"id": "custom.displayMode", "value": "color-background"},
|
||||
{"id": "mappings", "value": [
|
||||
{"options": {"Bound": {"color": "green", "index": 0}}, "type": "value"},
|
||||
{"options": {"Pending": {"color": "yellow", "index": 1}}, "type": "value"},
|
||||
{"options": {"Lost": {"color": "red", "index": 2}}, "type": "value"}
|
||||
]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "Capacity"},
|
||||
"properties": [{"id": "unit", "value": "bytes"}]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 24},
|
||||
"id": 31,
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"footer": {"show": false}
|
||||
},
|
||||
"title": "PVC — name / namespace / phase / capacity",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "kube_persistentvolumeclaim_status_phase",
|
||||
"legendFormat": "",
|
||||
"refId": "A",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"expr": "kube_persistentvolumeclaim_resource_requests_storage_bytes",
|
||||
"legendFormat": "",
|
||||
"refId": "B",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"id": "filterFieldsByName",
|
||||
"options": {
|
||||
"include": {
|
||||
"names": ["persistentvolumeclaim", "namespace", "phase", "Value #A", "Value #B"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"renameByName": {
|
||||
"persistentvolumeclaim": "PVC",
|
||||
"namespace": "Namespace",
|
||||
"phase": "Phase",
|
||||
"Value #A": "Status",
|
||||
"Value #B": "Capacity"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 38,
|
||||
"tags": ["funk"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"type": "datasource"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {"from": "now-3h", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "Europe/Paris",
|
||||
"title": "Funk — Kubernetes",
|
||||
"uid": "funk-k8s1",
|
||||
"version": 1
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue