apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboard-infrastructure namespace: monitoring labels: grafana_dashboard: "1" data: infrastructure.json: | { "__inputs": [], "__requires": [], "annotations": {"list": []}, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "panels": [ { "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, "id": 1, "title": "Compute nodes (compute-01 / 02 / 03)", "type": "row" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 0, "y": 1}, "id": 2, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "CPU % (avg compute)", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "avg(100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}[5m])) * 100))", "legendFormat": "CPU avg", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 4, "y": 1}, "id": 3, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "RAM % (avg compute)", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "avg(100 - (node_memory_MemAvailable_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} / node_memory_MemTotal_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} * 100))", "legendFormat": "RAM avg", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 2}, {"color": "red", "value": 4} ] }, "unit": "short", "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 8, "y": 1}, "id": 4, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "Load1 (avg compute)", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "avg(node_load1{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"})", "legendFormat": "Load1", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 10, "drawStyle": "line", "spanNulls": false }, "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, "gridPos": {"h": 8, "w": 8, "x": 0, "y": 5}, "id": 11, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "CPU % — compute-01/02/03", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\",instance=~\"192\\.168\\.10\\.(11|12|13):9100\"}[5m])) * 100)", "legendFormat": "{{instance}}", "refId": "A" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 10, "drawStyle": "line", "spanNulls": false }, "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, "gridPos": {"h": 8, "w": 8, "x": 8, "y": 5}, "id": 12, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "RAM % — compute-01/02/03", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (node_memory_MemAvailable_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} / node_memory_MemTotal_bytes{instance=~\"192\\.168\\.10\\.(11|12|13):9100\"} * 100)", "legendFormat": "{{instance}}", "refId": "A" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 5, "drawStyle": "line", "spanNulls": false }, "unit": "Bps", "min": 0 }, "overrides": [] }, "gridPos": {"h": 8, "w": 8, "x": 16, "y": 5}, "id": 13, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "Network I/O — compute nodes", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum by (instance) (irate(node_network_receive_bytes_total{instance=~\"192\\.168\\.10\\.(11|12|13):9100\",device!~\"lo|veth.*|flannel.*|cni.*\"}[5m]))", "legendFormat": "rx {{instance}}", "refId": "A" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum by (instance) (irate(node_network_transmit_bytes_total{instance=~\"192\\.168\\.10\\.(11|12|13):9100\",device!~\"lo|veth.*|flannel.*|cni.*\"}[5m]))", "legendFormat": "tx {{instance}}", "refId": "B" } ], "type": "timeseries" }, { "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 13}, "id": 20, "title": "storage-01 (gateway — RAID5 — NFS)", "type": "row" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 0, "y": 14}, "id": 21, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "CPU % — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"storage-01\"}[5m])) * 100)", "legendFormat": "CPU", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 4, "y": 14}, "id": 22, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "RAM % — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (node_memory_MemAvailable_bytes{job=\"storage-01\"} / node_memory_MemTotal_bytes{job=\"storage-01\"} * 100)", "legendFormat": "RAM", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 8, "y": 14}, "id": 23, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "Disk /srv usage % — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (node_filesystem_avail_bytes{job=\"storage-01\",mountpoint=\"/srv\",fstype!=\"tmpfs\"} / node_filesystem_size_bytes{job=\"storage-01\",mountpoint=\"/srv\",fstype!=\"tmpfs\"} * 100)", "legendFormat": "/srv", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 10, "drawStyle": "line", "spanNulls": false }, "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 0, "y": 18}, "id": 24, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "CPU — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"storage-01\"}[5m])) * 100)", "legendFormat": "CPU %", "refId": "A" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 10, "drawStyle": "line", "spanNulls": false }, "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 6, "y": 18}, "id": 25, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "RAM — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (node_memory_MemAvailable_bytes{job=\"storage-01\"} / node_memory_MemTotal_bytes{job=\"storage-01\"} * 100)", "legendFormat": "RAM %", "refId": "A" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 5, "drawStyle": "line", "spanNulls": false }, "unit": "Bps", "min": 0 }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 12, "y": 18}, "id": 26, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "Disk I/O — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum(irate(node_disk_read_bytes_total{job=\"storage-01\"}[5m]))", "legendFormat": "read", "refId": "A" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum(irate(node_disk_written_bytes_total{job=\"storage-01\"}[5m]))", "legendFormat": "write", "refId": "B" } ], "type": "timeseries" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 5, "drawStyle": "line", "spanNulls": false }, "unit": "Bps", "min": 0 }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 18, "y": 18}, "id": 27, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "Network I/O — storage-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum(irate(node_network_receive_bytes_total{job=\"storage-01\",device!~\"lo|veth.*\"}[5m]))", "legendFormat": "rx", "refId": "A" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "sum(irate(node_network_transmit_bytes_total{job=\"storage-01\",device!~\"lo|veth.*\"}[5m]))", "legendFormat": "tx", "refId": "B" } ], "type": "timeseries" }, { "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 26}, "id": 30, "title": "gpu-01 (RX 6700XT — ROCm — llama-server)", "type": "row" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 0, "y": 27}, "id": 31, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "CPU % — gpu-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\",job=\"gpu-01-node\"}[5m])) * 100)", "legendFormat": "CPU", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 4, "w": 4, "x": 4, "y": 27}, "id": 32, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none" }, "title": "RAM % — gpu-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "100 - (node_memory_MemAvailable_bytes{job=\"gpu-01-node\"} / node_memory_MemTotal_bytes{job=\"gpu-01-node\"} * 100)", "legendFormat": "RAM", "refId": "A" } ], "type": "stat" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 85} ] }, "unit": "celsius", "min": 0, "max": 110, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 0, "y": 31}, "id": 33, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "GPU Temperature (°C)", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}", "legendFormat": "GPU Temp", "refId": "A" } ], "type": "gauge" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 80}, {"color": "red", "value": 92} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 6, "y": 31}, "id": 34, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "VRAM Usage %", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"} / rocm_vram_total_bytes{job=\"gpu-01-node\",gpu=\"0\"} * 100", "legendFormat": "VRAM %", "refId": "A" } ], "type": "gauge" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "thresholds"}, "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 90} ] }, "unit": "percent", "min": 0, "max": 100, "mappings": [] }, "overrides": [] }, "gridPos": {"h": 8, "w": 6, "x": 12, "y": 31}, "id": 35, "options": { "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "GPU Utilization %", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "rocm_gpu_utilization_percent{job=\"gpu-01-node\",gpu=\"0\"}", "legendFormat": "GPU Util", "refId": "A" } ], "type": "gauge" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "fieldConfig": { "defaults": { "color": {"mode": "palette-classic"}, "custom": { "lineWidth": 1, "fillOpacity": 10, "drawStyle": "line", "spanNulls": false }, "min": 0 }, "overrides": [ { "matcher": {"id": "byName", "options": "GPU Temp (°C)"}, "properties": [{"id": "unit", "value": "celsius"}, {"id": "custom.axisPlacement", "value": "left"}] }, { "matcher": {"id": "byName", "options": "VRAM %"}, "properties": [{"id": "unit", "value": "percent"}, {"id": "custom.axisPlacement", "value": "right"}, {"id": "max", "value": 100}] } ] }, "gridPos": {"h": 8, "w": 24, "x": 0, "y": 39}, "id": 36, "options": { "tooltip": {"mode": "multi"}, "legend": {"displayMode": "list", "placement": "bottom"} }, "title": "GPU Temp + VRAM % — gpu-01", "targets": [ { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "rocm_gpu_temperature_celsius{job=\"gpu-01-node\",gpu=\"0\"}", "legendFormat": "GPU Temp (°C)", "refId": "A" }, { "datasource": {"type": "prometheus", "uid": "${datasource}"}, "expr": "rocm_vram_used_bytes{job=\"gpu-01-node\",gpu=\"0\"} / rocm_vram_total_bytes{job=\"gpu-01-node\",gpu=\"0\"} * 100", "legendFormat": "VRAM %", "refId": "B" } ], "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 38, "tags": ["funk"], "templating": { "list": [ { "current": {}, "hide": 0, "includeAll": false, "label": "Datasource", "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "type": "datasource" } ] }, "time": {"from": "now-3h", "to": "now"}, "timepicker": {}, "timezone": "Europe/Paris", "title": "Funk — Infrastructure", "uid": "funk-inf1", "version": 1 }