apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboard-infrastructure namespace: monitoring labels: grafana_dashboard: "1" data: infrastructure.json: | { "__inputs": [], "__requires": [], "annotations": {"list": []}, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "panels": [ { "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, "id": 1, "title": "Compute nodes (compute-01 / 02 / 03)", "type": "row" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 0, "y": 1}, "id": 2, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "CPU % (avg compute)", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "avg(100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",job=\"node-exporter\"}[5m])) * 100))", "legendFormat": "CPU avg", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 4, "y": 1}, "id": 3, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "RAM % (avg compute)", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "avg(100 - (node_memory_MemAvailable_bytes{job=\"node-exporter\"} / node_memory_MemTotal_bytes{job=\"node-exporter\"} * 100))", "legendFormat": "RAM avg", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 2}, {"color": "red", "value": 4} ] }, "unit": "short", "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 8, "y": 1}, "id": 4, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "Load1 (avg compute)", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "avg(node_load1{job=\"node-exporter\"})", "legendFormat": "Load1", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "yellow", "value": null}, {"color": "green", "value": 43200}, {"color": "blue", "value": 604800} ] }, "unit": "dtdurations", "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 6, "x": 12, "y": 1}, "id": 40, "options": { "reduceOptions": {"calcs": ["min"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "Uptime min — compute", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "time() - node_boot_time_seconds{job=\"node-exporter\"}", "legendFormat": "{{instance}}", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 60}, {"color": "red", "value": 85} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 6, "x": 18, "y": 1}, "id": 41, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "Disk /var avg % — compute", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "avg(100 - (node_filesystem_avail_bytes{job=\"node-exporter\",mountpoint=\"/var\",fstype=\"xfs\"} / node_filesystem_size_bytes{job=\"node-exporter\",mountpoint=\"/var\",fstype=\"xfs\"} * 100))", "legendFormat": "/var avg", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 10, "drawStyle": "line", "spanNulls": false }, "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, "gridPos": {"h": 8, "w": 8, "x": 0, "y": 5}, "id": 11, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "CPU % — compute-01/02/03", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",job=\"node-exporter\"}[5m])) * 100)", "legendFormat": "{{instance}}", "refId": "A" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 10, "drawStyle": "line", "spanNulls": false }, "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, "gridPos": {"h": 8, "w": 8, "x": 8, "y": 5}, "id": 12, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "RAM % — compute-01/02/03", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (node_memory_MemAvailable_bytes{job=\"node-exporter\"} / node_memory_MemTotal_bytes{job=\"node-exporter\"} * 100)", "legendFormat": "{{instance}}", "refId": "A" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 5, "drawStyle": "line", "spanNulls": false }, "unit": "Bps", "min": 0 }, "overrides": [] }, "gridPos": {"h": 8, "w": 8, "x": 16, "y": 5}, "id": 13, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "Network I/O — compute nodes", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum by (instance) (irate(node_network_receive_bytes_total{job=\"node-exporter\",device!~\"lo|veth.*|flannel.*|cni.*\"}[5m]))", "legendFormat": "rx {{instance}}", "refId": "A" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum by (instance) (irate(node_network_transmit_bytes_total{job=\"node-exporter\",device!~\"lo|veth.*|flannel.*|cni.*\"}[5m]))", "legendFormat": "tx {{instance}}", "refId": "B" } ], "type": "timeseries" }, { "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 13}, "id": 20, "title": "storage-01 (gateway — RAID5 — NFS)", "type": "row" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 0, "y": 14}, "id": 21, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "CPU % — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"storage-01\"}[5m])) * 100)", "legendFormat": "CPU", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 4, "y": 14}, "id": 22, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "RAM % — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (node_memory_MemAvailable_bytes{job=\"storage-01\"} / node_memory_MemTotal_bytes{job=\"storage-01\"} * 100)", "legendFormat": "RAM", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 8, "y": 14}, "id": 23, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "Disk /srv/data % — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (node_filesystem_avail_bytes{job=\"storage-01\",mountpoint=\"/srv/data\",fstype!=\"tmpfs\"} / node_filesystem_size_bytes{job=\"storage-01\",mountpoint=\"/srv/data\",fstype!=\"tmpfs\"} * 100)", "legendFormat": "/srv/data", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [ { "type": "value", "options": { "1": {"text": "ACTIF", "color": "green", "index": 0}, "0": {"text": "DÉGRADÉ", "color": "red", "index": 1} } } ] }, "overrides": [] }, "gridPos": {"h": 4, "w": 3, "x": 12, "y": 14}, "id": 42, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "RAID5 état", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_md_state{job=\"storage-01\",device=\"md0\",state=\"active\"}", "legendFormat": "md0", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "red", "value": 1} ] }, "unit": "none", "mappings": [], "noValue": "0" }, "overrides": [] }, "gridPos": {"h": 4, "w": 3, "x": 15, "y": 14}, "id": 43, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "RAID5 disques HS", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_md_disks{job=\"storage-01\",device=\"md0\",state=\"failed\"}", "legendFormat": "failed", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "yellow", "value": null}, {"color": "green", "value": 43200}, {"color": "blue", "value": 604800} ] }, "unit": "dtdurations", "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 3, "x": 18, "y": 14}, "id": 44, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "Uptime — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "time() - node_boot_time_seconds{job=\"storage-01\"}", "legendFormat": "uptime", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 60}, {"color": "red", "value": 80} ] }, "unit": "celsius", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 3, "x": 21, "y": 14}, "id": 45, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "Temp CPU — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_hwmon_temp_celsius{job=\"storage-01\",chip=\"pci0000:00_0000:00:18_3\",sensor=\"temp1\"}", "legendFormat": "CPU", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 10, "drawStyle": "line", "spanNulls": false }, "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 0, "y": 18}, "id": 24, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "CPU — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"storage-01\"}[5m])) * 100)", "legendFormat": "CPU %", "refId": "A" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 10, "drawStyle": "line", "spanNulls": false }, "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 6, "y": 18}, "id": 25, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "RAM — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (node_memory_MemAvailable_bytes{job=\"storage-01\"} / node_memory_MemTotal_bytes{job=\"storage-01\"} * 100)", "legendFormat": "RAM %", "refId": "A" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 5, "drawStyle": "line", "spanNulls": false }, "unit": "Bps", "min": 0 }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 12, "y": 18}, "id": 26, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "Disk I/O — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum(irate(node_disk_read_bytes_total{job=\"storage-01\"}[5m]))", "legendFormat": "read", "refId": "A" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum(irate(node_disk_written_bytes_total{job=\"storage-01\"}[5m]))", "legendFormat": "write", "refId": "B" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 5, "drawStyle": "line", "spanNulls": false }, "unit": "Bps", "min": 0 }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 18, "y": 18}, "id": 27, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "Network I/O — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum(irate(node_network_receive_bytes_total{job=\"storage-01\",device!~\"lo|veth.*\"}[5m]))", "legendFormat": "rx", "refId": "A" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum(irate(node_network_transmit_bytes_total{job=\"storage-01\",device!~\"lo|veth.*\"}[5m]))", "legendFormat": "tx", "refId": "B" } ], "type": "timeseries" }, { "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 26}, "id": 30, "title": "gpu-01 (RX 6700XT — ROCm — llama-server)", "type": "row" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 0, "y": 27}, "id": 31, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "CPU % — gpu-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"gpu-01-node\"}[5m])) * 100)", "legendFormat": "CPU", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 4, "y": 27}, "id": 32, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "RAM % — gpu-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (node_memory_MemAvailable_bytes{job=\"gpu-01-node\"} / node_memory_MemTotal_bytes{job=\"gpu-01-node\"} * 100)", "legendFormat": "RAM", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "yellow", "value": null}, {"color": "green", "value": 43200}, {"color": "blue", "value": 604800} ] }, "unit": "dtdurations", "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 5, "x": 8, "y": 27}, "id": 46, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "Uptime — gpu-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "time() - node_boot_time_seconds{job=\"gpu-01-node\"}", "legendFormat": "uptime", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "celsius", "min": 0, "max": 110, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 5, "x": 13, "y": 27}, "id": 47, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "Temp CPU Tdie — gpu-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_hwmon_temp_celsius{job=\"gpu-01-node\",chip=\"pci0000:00_0000:00:18_3\",sensor=\"temp3\"}", "legendFormat": "Tdie", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 6, "x": 18, "y": 27}, "id": 48, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "Swap % — gpu-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "(1 - node_memory_SwapFree_bytes{job=\"gpu-01-node\"} / node_memory_SwapTotal_bytes{job=\"gpu-01-node\"}) * 100", "legendFormat": "Swap", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 85} ] }, "unit": "celsius", "min": 0, "max": 110, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 0, "y": 31}, "id": 33, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "GPU Temperature (°C)", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}", "legendFormat": "GPU Temp", "refId": "A" } ], "type": "gauge" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 80}, {"color": "red", "value": 92} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 6, "y": 31}, "id": 34, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "VRAM Usage %", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"} / rocm_vram_total_bytes{job=\"gpu-01-node\",gpu=\"0\"} * 100", "legendFormat": "VRAM %", "refId": "A" } ], "type": "gauge" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 12, "y": 31}, "id": 35, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "GPU Utilization %", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "rocm_gpu_utilization_percent{job=\"gpu-01-node\",gpu=\"0\"}", "legendFormat": "GPU Util", "refId": "A" } ], "type": "gauge" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 10, "drawStyle": "line", "spanNulls": false }, "min": 0 }, "overrides": [ { "matcher": {"id": "byName", "options": "GPU Temp (°C)"}, "properties": [{"id": "unit", "value": "celsius"}, {"id": "custom.axisPlacement", "value": "left"}] }, { "matcher": {"id": "byName", "options": "VRAM %"}, "properties": [{"id": "unit", "value": "percent"}, {"id": "custom.axisPlacement", "value": "right"}, {"id": "max", "value": 100}] } ] }, "gridPos": {"h": 8, "w": 24, "x": 0, "y": 39}, "id": 36, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "GPU Temp + VRAM % — gpu-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}", "legendFormat": "GPU Temp (°C)", "refId": "A" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"} / rocm_vram_total_bytes{job=\"gpu-01-node\",gpu=\"0\"} * 100", "legendFormat": "VRAM %", "refId": "B" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 8, "drawStyle": "line", "spanNulls": false }, "unit": "celsius", "min": 0, "max": 110 }, "overrides": [ { "matcher": {"id": "byName", "options": "edge"}, "properties": [{"id": "color", "value": {"fixedColor": "blue", "mode": "fixed"}}] }, { "matcher": {"id": "byName", "options": "hotspot"}, "properties": [{"id": "color", "value": {"fixedColor": "orange", "mode": "fixed"}}] }, { "matcher": {"id": "byName", "options": "VRAM"}, "properties": [{"id": "color", "value": {"fixedColor": "purple", "mode": "fixed"}}] } ] }, "gridPos": {"h": 8, "w": 12, "x": 0, "y": 47}, "id": 49, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "GPU Températures (edge / hotspot / VRAM) — gpu-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_hwmon_temp_celsius{job=\"gpu-01-node\",chip=\"0000:09:00_0_0000:0a:00_0\",sensor=\"temp1\"}", "legendFormat": "edge", "refId": "A" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_hwmon_temp_celsius{job=\"gpu-01-node\",chip=\"0000:09:00_0_0000:0a:00_0\",sensor=\"temp2\"}", "legendFormat": "hotspot", "refId": "B" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_hwmon_temp_celsius{job=\"gpu-01-node\",chip=\"0000:09:00_0_0000:0a:00_0\",sensor=\"temp3\"}", "legendFormat": "VRAM", "refId": "C" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 8, "drawStyle": "line", "spanNulls": false }, "unit": "celsius", "min": 0, "max": 110 }, "overrides": [] }, "gridPos": {"h": 8, "w": 12, "x": 12, "y": 47}, "id": 50, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "CPU Températures — gpu-01 (AMD Tdie)", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_hwmon_temp_celsius{job=\"gpu-01-node\",chip=\"pci0000:00_0000:00:18_3\",sensor=\"temp3\"}", "legendFormat": "Tdie", "refId": "A" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_hwmon_temp_celsius{job=\"gpu-01-node\",chip=\"pci0000:00_0000:00:18_3\",sensor=\"temp1\"}", "legendFormat": "Tctl", "refId": "B" } ], "type": "timeseries" }, { "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 55}, "id": 51, "title": "Services — état systemd", "type": "row" }, { "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 56}, "id": 70, "title": "storage-01", "type": "row" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [{"type": "value", "options": {"1": {"text": "dnsmasq", "color": "green", "index": 0}, "0": {"text": "ARRÊTÉ", "color": "red", "index": 1}}}] }, "overrides": [] }, "gridPos": {"h": 3, "w": 3, "x": 0, "y": 57}, "id": 52, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "colorMode": "background", "graphMode": "none"}, "title": "DNS (dnsmasq)", "targets": [{"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_systemd_unit_state{job=\"storage-01\",name=\"dnsmasq.service\",state=\"active\"}", "legendFormat": "", "refId": "A"}], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [{"type": "value", "options": {"1": {"text": "nfs-server", "color": "green", "index": 0}, "0": {"text": "ARRÊTÉ", "color": "red", "index": 1}}}] }, "overrides": [] }, "gridPos": {"h": 3, "w": 3, "x": 3, "y": 57}, "id": 53, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "colorMode": "background", "graphMode": "none"}, "title": "NFS server", "targets": [{"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_systemd_unit_state{job=\"storage-01\",name=\"nfs-server.service\",state=\"active\"}", "legendFormat": "", "refId": "A"}], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [{"type": "value", "options": {"1": {"text": "postgresql", "color": "green", "index": 0}, "0": {"text": "ARRÊTÉ", "color": "red", "index": 1}}}] }, "overrides": [] }, "gridPos": {"h": 3, "w": 3, "x": 6, "y": 57}, "id": 54, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "colorMode": "background", "graphMode": "none"}, "title": "PostgreSQL", "targets": [{"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_systemd_unit_state{job=\"storage-01\",name=\"postgresql-16.service\",state=\"active\"}", "legendFormat": "", "refId": "A"}], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [{"type": "value", "options": {"1": {"text": "qdrant", "color": "green", "index": 0}, "0": {"text": "ARRÊTÉ", "color": "red", "index": 1}}}] }, "overrides": [] }, "gridPos": {"h": 3, "w": 3, "x": 9, "y": 57}, "id": 55, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "colorMode": "background", "graphMode": "none"}, "title": "Qdrant", "targets": [{"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_systemd_unit_state{job=\"storage-01\",name=\"qdrant.service\",state=\"active\"}", "legendFormat": "", "refId": "A"}], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [{"type": "value", "options": {"1": {"text": "litellm", "color": "green", "index": 0}, "0": {"text": "ARRÊTÉ", "color": "red", "index": 1}}}] }, "overrides": [] }, "gridPos": {"h": 3, "w": 3, "x": 12, "y": 57}, "id": 56, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "colorMode": "background", "graphMode": "none"}, "title": "LiteLLM", "targets": [{"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_systemd_unit_state{job=\"storage-01\",name=\"litellm.service\",state=\"active\"}", "legendFormat": "", "refId": "A"}], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [{"type": "value", "options": {"1": {"text": "hermes-agent", "color": "green", "index": 0}, "0": {"text": "ARRÊTÉ", "color": "red", "index": 1}}}] }, "overrides": [] }, "gridPos": {"h": 3, "w": 3, "x": 15, "y": 57}, "id": 57, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "colorMode": "background", "graphMode": "none"}, "title": "Hermes Agent", "targets": [{"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_systemd_unit_state{job=\"storage-01\",name=\"hermes-agent.service\",state=\"active\"}", "legendFormat": "", "refId": "A"}], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [{"type": "value", "options": {"1": {"text": "nftables", "color": "green", "index": 0}, "0": {"text": "ARRÊTÉ", "color": "red", "index": 1}}}] }, "overrides": [] }, "gridPos": {"h": 3, "w": 3, "x": 18, "y": 57}, "id": 58, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "colorMode": "background", "graphMode": "none"}, "title": "nftables (firewall)", "targets": [{"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_systemd_unit_state{job=\"storage-01\",name=\"nftables.service\",state=\"active\"}", "legendFormat": "", "refId": "A"}], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [{"type": "value", "options": {"1": {"text": "chronyd", "color": "green", "index": 0}, "0": {"text": "ARRÊTÉ", "color": "red", "index": 1}}}] }, "overrides": [] }, "gridPos": {"h": 3, "w": 3, "x": 21, "y": 57}, "id": 59, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "colorMode": "background", "graphMode": "none"}, "title": "NTP (chronyd)", "targets": [{"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_systemd_unit_state{job=\"storage-01\",name=\"chronyd.service\",state=\"active\"}", "legendFormat": "", "refId": "A"}], "type": "stat" }, { "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 60}, "id": 71, "title": "gpu-01", "type": "row" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [{"type": "value", "options": {"1": {"text": "llama-server GPU", "color": "green", "index": 0}, "0": {"text": "ARRÊTÉ", "color": "red", "index": 1}}}] }, "overrides": [] }, "gridPos": {"h": 3, "w": 6, "x": 0, "y": 61}, "id": 60, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "colorMode": "background", "graphMode": "none"}, "title": "llama-server GPU (:1234)", "targets": [{"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_systemd_unit_state{job=\"gpu-01-node\",name=\"llama-server.service\",state=\"active\"}", "legendFormat": "", "refId": "A"}], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [{"type": "value", "options": {"1": {"text": "llama-embed nomic", "color": "green", "index": 0}, "0": {"text": "ARRÊTÉ", "color": "red", "index": 1}}}] }, "overrides": [] }, "gridPos": {"h": 3, "w": 6, "x": 6, "y": 61}, "id": 61, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "colorMode": "background", "graphMode": "none"}, "title": "llama-embed nomic (:1238)", "targets": [{"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_systemd_unit_state{job=\"gpu-01-node\",name=\"llama-embed.service\",state=\"active\"}", "legendFormat": "", "refId": "A"}], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "none", "mappings": [{"type": "value", "options": {"1": {"text": "chronyd", "color": "green", "index": 0}, "0": {"text": "ARRÊTÉ", "color": "red", "index": 1}}}] }, "overrides": [] }, "gridPos": {"h": 3, "w": 6, "x": 18, "y": 61}, "id": 63, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "colorMode": "background", "graphMode": "none"}, "title": "NTP (chronyd)", "targets": [{"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "node_systemd_unit_state{job=\"gpu-01-node\",name=\"chronyd.service\",state=\"active\"}", "legendFormat": "", "refId": "A"}], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": {"lineWidth": 1, "fillOpacity": 8, "drawStyle": "line", "spanNulls": false}, "unit": "Bps", "min": 0 }, "overrides": [] }, "gridPos": {"h": 7, "w": 12, "x": 0, "y": 64}, "id": 64, "options": {"tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"}}, "title": "NFS I/O — storage-01 (débit disque NFS)", "targets": [ {"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "irate(node_nfsd_disk_bytes_read_total{job=\"storage-01\"}[5m])", "legendFormat": "read", "refId": "A"}, {"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "irate(node_nfsd_disk_bytes_written_total{job=\"storage-01\"}[5m])", "legendFormat": "write", "refId": "B"} ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": {"lineWidth": 1, "fillOpacity": 8, "drawStyle": "line", "spanNulls": false}, "unit": "reqps", "min": 0 }, "overrides": [] }, "gridPos": {"h": 7, "w": 12, "x": 12, "y": 64}, "id": 65, "options": {"tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"}}, "title": "NFS requêtes/s par type — storage-01", "targets": [ {"datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "irate(node_nfsd_requests_total{job=\"storage-01\",method=~\"read|write|getattr|lookup\"}[5m])", "legendFormat": "{{method}}", "refId": "A"} ], "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 38, "tags": ["funk"], "templating": { "list": [ { "current": {}, "hide": 0, "includeAll": false, "label": "Datasource", "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "type": "datasource" } ] }, "time": {"from": "now-3h", "to": "now"}, "timepicker": {}, "timezone": "Europe/Paris", "title": "Funk — Infrastructure", "uid": "funk-inf1", "version": 1 }