mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-08 11:04:43 +02:00
feat: monitoring — node_exporter + ROCm sysfs collector + alertmanager webhook
Ansible roles: - node_exporter : déploie node_exporter v1.9.1 sur storage-01 et gpu-01 avec textfile collector + services systemd. ROCm activé sur gpu-01 uniquement via group_vars. Collecte métriques GPU via sysfs (temp, VRAM, utilisation) — pas de dépendance rocm-smi (remplacé dans ROCm 7.x) - alertmanager_webhook : service Python sur storage-01 (:9093/webhook) reçoit alertes AlertManager et les route vers ask-agent monitor → Hermes Playbooks: node_exporter + alertmanager_webhook ajoutés à storage-01 et gpu-01 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
857351bcf9
commit
92a882aa94
15 changed files with 354 additions and 2 deletions
|
|
@ -0,0 +1,18 @@
|
|||
[Unit]
|
||||
Description=Prometheus Node Exporter
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
User={{ node_exporter_user }}
|
||||
Group={{ node_exporter_user }}
|
||||
ExecStart=/usr/local/bin/node_exporter \
|
||||
--web.listen-address=":{{ node_exporter_port }}" \
|
||||
--collector.textfile.directory={{ node_exporter_textfile_dir }} \
|
||||
--collector.systemd \
|
||||
--collector.processes
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
[Unit]
|
||||
Description=ROCm metrics scraper for Prometheus textfile collector
|
||||
After=multi-user.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/rocm_scraper.sh
|
||||
33
ansible/roles/node_exporter/templates/rocm_scraper.sh.j2
Normal file
33
ansible/roles/node_exporter/templates/rocm_scraper.sh.j2
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#!/bin/bash
|
||||
# Collecte les métriques GPU AMD via sysfs (pas de rocm-smi requis)
|
||||
OUTPUT="{{ node_exporter_textfile_dir }}/rocm.prom"
|
||||
TMPFILE="${OUTPUT}.tmp"
|
||||
|
||||
{
|
||||
echo "# HELP rocm_gpu_temperature_celsius GPU temperature in Celsius"
|
||||
echo "# TYPE rocm_gpu_temperature_celsius gauge"
|
||||
echo "# HELP rocm_gpu_utilization_percent GPU utilization percentage"
|
||||
echo "# TYPE rocm_gpu_utilization_percent gauge"
|
||||
echo "# HELP rocm_vram_used_bytes VRAM used in bytes"
|
||||
echo "# TYPE rocm_vram_used_bytes gauge"
|
||||
echo "# HELP rocm_vram_total_bytes VRAM total in bytes"
|
||||
echo "# TYPE rocm_vram_total_bytes gauge"
|
||||
|
||||
for card in /sys/class/drm/card[0-9]; do
|
||||
[ -d "$card/device" ] || continue
|
||||
idx=$(basename "$card" | tr -d 'card')
|
||||
labels="gpu=\"${idx}\",model=\"gfx1031\""
|
||||
|
||||
temp_raw=$(cat "$card/device/hwmon/hwmon"*/temp1_input 2>/dev/null | head -1)
|
||||
[ -n "$temp_raw" ] && echo "rocm_gpu_temperature_celsius{${labels}} $(echo "scale=1; ${temp_raw}/1000" | bc)"
|
||||
|
||||
util=$(cat "$card/device/gpu_busy_percent" 2>/dev/null)
|
||||
[ -n "$util" ] && echo "rocm_gpu_utilization_percent{${labels}} ${util}"
|
||||
|
||||
vram_used=$(cat "$card/device/mem_info_vram_used" 2>/dev/null)
|
||||
[ -n "$vram_used" ] && echo "rocm_vram_used_bytes{${labels}} ${vram_used}"
|
||||
|
||||
vram_total=$(cat "$card/device/mem_info_vram_total" 2>/dev/null)
|
||||
[ -n "$vram_total" ] && echo "rocm_vram_total_bytes{${labels}} ${vram_total}"
|
||||
done
|
||||
} > "$TMPFILE" && mv "$TMPFILE" "$OUTPUT"
|
||||
11
ansible/roles/node_exporter/templates/rocm_scraper.timer.j2
Normal file
11
ansible/roles/node_exporter/templates/rocm_scraper.timer.j2
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
[Unit]
|
||||
Description=ROCm metrics scraper timer
|
||||
Requires=rocm_scraper.service
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitActiveSec={{ rocm_scrape_interval }}s
|
||||
AccuracySec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Loading…
Add table
Add a link
Reference in a new issue