diff --git a/ansible/host_vars/gpu-01/vars.yml b/ansible/host_vars/gpu-01/vars.yml index 8acc216..986fada 100644 --- a/ansible/host_vars/gpu-01/vars.yml +++ b/ansible/host_vars/gpu-01/vars.yml @@ -6,6 +6,9 @@ llama_model_path: "/mnt/models/bartowski/Qwen3-8B-GGUF/Qwen3-8B-Q4_K_M.gguf" llama_model_alias: "qwen3-8b" llama_ctx_size: 32768 +# ROCm scraper — métriques GPU dans Prometheus +rocm_exporter_enabled: true + # Montages NFS (surcharge les defaults du rôle nfs_client) nfs_client_mounts: - local_path: /mnt/nfs diff --git a/ansible/roles/node_exporter/templates/node_exporter.service.j2 b/ansible/roles/node_exporter/templates/node_exporter.service.j2 index 95fc896..0d39f6a 100644 --- a/ansible/roles/node_exporter/templates/node_exporter.service.j2 +++ b/ansible/roles/node_exporter/templates/node_exporter.service.j2 @@ -9,7 +9,8 @@ ExecStart=/usr/local/bin/node_exporter \ --web.listen-address=":{{ node_exporter_port }}" \ --collector.textfile.directory={{ node_exporter_textfile_dir }} \ --collector.systemd \ - --collector.processes + --collector.processes \ + --collector.rapl Restart=on-failure RestartSec=5s diff --git a/ansible/roles/node_exporter/templates/rocm_scraper.sh.j2 b/ansible/roles/node_exporter/templates/rocm_scraper.sh.j2 index ab9b9b9..7752222 100644 --- a/ansible/roles/node_exporter/templates/rocm_scraper.sh.j2 +++ b/ansible/roles/node_exporter/templates/rocm_scraper.sh.j2 @@ -12,6 +12,10 @@ TMPFILE="${OUTPUT}.tmp" echo "# TYPE rocm_vram_used_bytes gauge" echo "# HELP rocm_vram_total_bytes VRAM total in bytes" echo "# TYPE rocm_vram_total_bytes gauge" + echo "# HELP rocm_gpu_power_watts GPU power draw in watts" + echo "# TYPE rocm_gpu_power_watts gauge" + echo "# HELP rocm_gpu_power_cap_watts GPU power cap in watts" + echo "# TYPE rocm_gpu_power_cap_watts gauge" for card in /sys/class/drm/card[0-9]; do [ -d "$card/device" ] || continue @@ -29,5 +33,11 @@ TMPFILE="${OUTPUT}.tmp" vram_total=$(cat "$card/device/mem_info_vram_total" 2>/dev/null) [ -n "$vram_total" ] && echo "rocm_vram_total_bytes{${labels}} ${vram_total}" + + power_raw=$(cat "$card/device/hwmon/hwmon"*/power1_average 2>/dev/null | head -1) + [ -n "$power_raw" ] && echo "rocm_gpu_power_watts{${labels}} $(echo "scale=2; ${power_raw}/1000000" | bc)" + + power_cap_raw=$(cat "$card/device/hwmon/hwmon"*/power1_cap 2>/dev/null | head -1) + [ -n "$power_cap_raw" ] && echo "rocm_gpu_power_cap_watts{${labels}} $(echo "scale=2; ${power_cap_raw}/1000000" | bc)" done } > "$TMPFILE" && mv "$TMPFILE" "$OUTPUT" diff --git a/k8s/infra/monitoring/values.yaml b/k8s/infra/monitoring/values.yaml index b129763..2d06485 100644 --- a/k8s/infra/monitoring/values.yaml +++ b/k8s/infra/monitoring/values.yaml @@ -162,6 +162,10 @@ nodeExporter: limits: memory: 64Mi +prometheus-node-exporter: + extraArgs: + - --collector.rapl + # Désactiver etcd/scheduler monitoring (Talos — non accessible) kubeEtcd: enabled: false