feat: monitoring — node_exporter + ROCm sysfs collector + alertmanager webhook

Ansible roles:
- node_exporter : déploie node_exporter v1.9.1 sur storage-01 et gpu-01
  avec textfile collector + services systemd. ROCm activé sur gpu-01 uniquement
  via group_vars. Collecte métriques GPU via sysfs (temp, VRAM, utilisation)
  — pas de dépendance rocm-smi (remplacé dans ROCm 7.x)
- alertmanager_webhook : service Python sur storage-01 (:9093/webhook)
  reçoit alertes AlertManager et les route vers ask-agent monitor → Hermes

Playbooks: node_exporter + alertmanager_webhook ajoutés à storage-01 et gpu-01

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

ansible/group_vars/gpu_hosts/vars.yml

@@ -1,2 +1,5 @@
 ---
 hsa_override_gfx_version: "10.3.0"
+
+# node_exporter — ROCm textfile collector activé sur gpu-01
+rocm_exporter_enabled: true

ansible/playbooks/gpu-01.yml

@@ -7,3 +7,4 @@
     - { role: rocm,          tags: [rocm] }
     - { role: nfs_client,    tags: [nfs_client] }
     - { role: llama_server,  tags: [llama_server] }
+    - { role: node_exporter, tags: [node_exporter] }

ansible/playbooks/storage-01.yml

@@ -10,5 +10,7 @@
     - { role: qdrant,       tags: [qdrant] }
     - { role: postgresql,   tags: [postgresql] }
     - { role: minio,        tags: [minio] }
-    - { role: litellm,      tags: [litellm] }
+    - { role: litellm,               tags: [litellm] }
-    - { role: hermes_agent, tags: [hermes_agent] }
+    - { role: hermes_agent,          tags: [hermes_agent] }
+    - { role: node_exporter,         tags: [node_exporter] }
+    - { role: alertmanager_webhook,  tags: [alertmanager_webhook] }

ansible/roles/alertmanager_webhook/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+webhook_port: 9093
+webhook_user: hermes
+webhook_install_dir: /opt/alertmanager-webhook
+litellm_url: "http://127.0.0.1:4000/v1"
+litellm_api_key: "lm-studio"

ansible/roles/alertmanager_webhook/files/webhook.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""
+AlertManager webhook → ask-agent monitor
+Reçoit les alertes Prometheus et les délègue au profil monitor de Hermes.
+"""
+import json
+import os
+import subprocess
+import logging
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from datetime import datetime
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
+log = logging.getLogger(__name__)
+
+PORT = int(os.environ.get("WEBHOOK_PORT", 9093))
+
+
+def format_alert(alert: dict) -> str:
+    name = alert.get("labels", {}).get("alertname", "unknown")
+    severity = alert.get("labels", {}).get("severity", "unknown")
+    instance = alert.get("labels", {}).get("instance", "unknown")
+    summary = alert.get("annotations", {}).get("summary", "")
+    description = alert.get("annotations", {}).get("description", "")
+    status = alert.get("status", "firing")
+    return f"[{status.upper()}] {name} — severity={severity} instance={instance}\n{summary}\n{description}".strip()
+
+
+class WebhookHandler(BaseHTTPRequestHandler):
+    def do_POST(self):
+        if self.path != "/webhook":
+            self.send_response(404)
+            self.end_headers()
+            return
+
+        length = int(self.headers.get("Content-Length", 0))
+        body = self.rfile.read(length)
+
+        try:
+            payload = json.loads(body)
+        except json.JSONDecodeError:
+            log.error("Invalid JSON payload")
+            self.send_response(400)
+            self.end_headers()
+            return
+
+        alerts = payload.get("alerts", [])
+        firing = [a for a in alerts if a.get("status") == "firing"]
+        resolved = [a for a in alerts if a.get("status") == "resolved"]
+
+        if not alerts:
+            self.send_response(200)
+            self.end_headers()
+            return
+
+        lines = [f"Alertes Prometheus reçues à {datetime.now().strftime('%H:%M:%S')} :"]
+        for a in firing:
+            lines.append(format_alert(a))
+        for a in resolved:
+            lines.append(format_alert(a))
+
+        message = "\n".join(lines) + "\n\nAnalyse ces alertes : sont-elles critiques ? Quelles actions recommandes-tu ?"
+
+        log.info(f"Forwarding {len(firing)} firing + {len(resolved)} resolved alerts to ask-agent monitor")
+
+        try:
+            result = subprocess.run(
+                ["/usr/local/bin/ask-agent", "monitor", message],
+                capture_output=True, text=True, timeout=120
+            )
+            log.info(f"ask-agent response: {result.stdout[:200]}")
+        except subprocess.TimeoutExpired:
+            log.error("ask-agent timed out")
+        except Exception as e:
+            log.error(f"ask-agent error: {e}")
+
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(b"ok")
+
+    def log_message(self, format, *args):
+        pass  # silence HTTP access logs
+
+
+if __name__ == "__main__":
+    log.info(f"AlertManager webhook listening on :{PORT}")
+    server = HTTPServer(("0.0.0.0", PORT), WebhookHandler)
+    server.serve_forever()

ansible/roles/alertmanager_webhook/handlers/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Restart alertmanager-webhook
+  ansible.builtin.systemd:
+    name: alertmanager-webhook
+    state: restarted
+    daemon_reload: true
+
+- name: Reload nftables
+  ansible.builtin.systemd:
+    name: nftables
+    state: reloaded

ansible/roles/alertmanager_webhook/tasks/main.yml

@@ -0,0 +1,40 @@
+---
+- name: Create webhook install directory
+  ansible.builtin.file:
+    path: "{{ webhook_install_dir }}"
+    state: directory
+    owner: "{{ webhook_user }}"
+    group: "{{ webhook_user }}"
+    mode: '0755'
+
+- name: Deploy webhook script
+  ansible.builtin.copy:
+    src: webhook.py
+    dest: "{{ webhook_install_dir }}/webhook.py"
+    owner: "{{ webhook_user }}"
+    group: "{{ webhook_user }}"
+    mode: '0755'
+  notify: Restart alertmanager-webhook
+
+- name: Deploy webhook systemd service
+  ansible.builtin.template:
+    src: alertmanager-webhook.service.j2
+    dest: /etc/systemd/system/alertmanager-webhook.service
+    mode: '0644'
+  notify: Restart alertmanager-webhook
+
+- name: Enable and start alertmanager-webhook
+  ansible.builtin.systemd:
+    name: alertmanager-webhook
+    enabled: true
+    state: started
+    daemon_reload: true
+
+- name: Open webhook port in nftables
+  ansible.builtin.lineinfile:
+    path: /etc/nftables.conf
+    insertafter: '# prometheus'
+    line: "    tcp dport {{ webhook_port }} ip saddr 192.168.10.0/24 accept comment \"alertmanager-webhook\""
+    state: present
+  notify: Reload nftables
+  ignore_errors: true

ansible/roles/alertmanager_webhook/templates/alertmanager-webhook.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=AlertManager webhook — Hermes monitor bridge
+After=network.target hermes-agent.service
+
+[Service]
+User={{ webhook_user }}
+Group={{ webhook_user }}
+Environment=WEBHOOK_PORT={{ webhook_port }}
+ExecStart=/usr/bin/python3 {{ webhook_install_dir }}/webhook.py
+Restart=on-failure
+RestartSec=10s
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=alertmanager-webhook
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/node_exporter/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+node_exporter_version: "1.9.1"
+node_exporter_port: 9100
+node_exporter_user: node_exporter
+node_exporter_install_dir: /opt/node_exporter
+node_exporter_textfile_dir: /var/lib/node_exporter/textfile_collector
+
+# ROCm scraper (gpu-01 uniquement)
+rocm_exporter_enabled: false
+rocm_exporter_port: 9101
+rocm_scrape_interval: 30

ansible/roles/node_exporter/handlers/main.yml

@@ -0,0 +1,18 @@
+---
+- name: Restart node_exporter
+  ansible.builtin.systemd:
+    name: node_exporter
+    state: restarted
+    daemon_reload: true
+
+- name: Reload nftables
+  ansible.builtin.systemd:
+    name: nftables
+    state: reloaded
+
+- name: Enable rocm_scraper timer
+  ansible.builtin.systemd:
+    name: rocm_scraper.timer
+    enabled: true
+    state: restarted
+    daemon_reload: true

ansible/roles/node_exporter/tasks/main.yml

@@ -0,0 +1,86 @@
+---
+- name: Create node_exporter user
+  ansible.builtin.user:
+    name: "{{ node_exporter_user }}"
+    shell: /sbin/nologin
+    create_home: false
+    system: true
+    state: present
+
+- name: Create textfile collector directory
+  ansible.builtin.file:
+    path: "{{ node_exporter_textfile_dir }}"
+    state: directory
+    owner: "{{ node_exporter_user }}"
+    group: "{{ node_exporter_user }}"
+    mode: '0755'
+
+- name: Download node_exporter
+  ansible.builtin.get_url:
+    url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz"
+    dest: "/tmp/node_exporter-{{ node_exporter_version }}.tar.gz"
+    mode: '0644'
+
+- name: Extract node_exporter
+  ansible.builtin.unarchive:
+    src: "/tmp/node_exporter-{{ node_exporter_version }}.tar.gz"
+    dest: /tmp/
+    remote_src: true
+    creates: "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64/node_exporter"
+
+- name: Install node_exporter binary
+  ansible.builtin.copy:
+    src: "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64/node_exporter"
+    dest: /usr/local/bin/node_exporter
+    owner: root
+    group: root
+    mode: '0755'
+    remote_src: true
+
+- name: Deploy node_exporter systemd service
+  ansible.builtin.template:
+    src: node_exporter.service.j2
+    dest: /etc/systemd/system/node_exporter.service
+    mode: '0644'
+  notify: Restart node_exporter
+
+- name: Enable and start node_exporter
+  ansible.builtin.systemd:
+    name: node_exporter
+    enabled: true
+    state: started
+    daemon_reload: true
+
+- name: Open node_exporter port in nftables
+  ansible.builtin.lineinfile:
+    path: /etc/nftables.conf
+    insertafter: '# prometheus'
+    line: "    tcp dport {{ node_exporter_port }} accept comment \"node_exporter\""
+    state: present
+  notify: Reload nftables
+  ignore_errors: true
+
+- name: Deploy ROCm scraper script
+  ansible.builtin.template:
+    src: rocm_scraper.sh.j2
+    dest: /usr/local/bin/rocm_scraper.sh
+    mode: '0755'
+  when: rocm_exporter_enabled
+
+- name: Deploy ROCm scraper service + timer
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: '0644'
+  loop:
+    - rocm_scraper.service
+    - rocm_scraper.timer
+  when: rocm_exporter_enabled
+
+- name: Enable ROCm scraper timer
+  ansible.builtin.systemd:
+    name: rocm_scraper.timer
+    enabled: true
+    state: started
+    daemon_reload: true
+  when: rocm_exporter_enabled

ansible/roles/node_exporter/templates/node_exporter.service.j2

@@ -0,0 +1,18 @@
+[Unit]
+Description=Prometheus Node Exporter
+After=network.target
+
+[Service]
+User={{ node_exporter_user }}
+Group={{ node_exporter_user }}
+ExecStart=/usr/local/bin/node_exporter \
+  --web.listen-address=":{{ node_exporter_port }}" \
+  --collector.textfile.directory={{ node_exporter_textfile_dir }} \
+  --collector.systemd \
+  --collector.processes
+
+Restart=on-failure
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/node_exporter/templates/rocm_scraper.service.j2

@@ -0,0 +1,7 @@
+[Unit]
+Description=ROCm metrics scraper for Prometheus textfile collector
+After=multi-user.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/rocm_scraper.sh

ansible/roles/node_exporter/templates/rocm_scraper.sh.j2

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Collecte les métriques GPU AMD via sysfs (pas de rocm-smi requis)
+OUTPUT="{{ node_exporter_textfile_dir }}/rocm.prom"
+TMPFILE="${OUTPUT}.tmp"
+
+{
+  echo "# HELP rocm_gpu_temperature_celsius GPU temperature in Celsius"
+  echo "# TYPE rocm_gpu_temperature_celsius gauge"
+  echo "# HELP rocm_gpu_utilization_percent GPU utilization percentage"
+  echo "# TYPE rocm_gpu_utilization_percent gauge"
+  echo "# HELP rocm_vram_used_bytes VRAM used in bytes"
+  echo "# TYPE rocm_vram_used_bytes gauge"
+  echo "# HELP rocm_vram_total_bytes VRAM total in bytes"
+  echo "# TYPE rocm_vram_total_bytes gauge"
+
+  for card in /sys/class/drm/card[0-9]; do
+    [ -d "$card/device" ] || continue
+    idx=$(basename "$card" | tr -d 'card')
+    labels="gpu=\"${idx}\",model=\"gfx1031\""
+
+    temp_raw=$(cat "$card/device/hwmon/hwmon"*/temp1_input 2>/dev/null | head -1)
+    [ -n "$temp_raw" ] && echo "rocm_gpu_temperature_celsius{${labels}} $(echo "scale=1; ${temp_raw}/1000" | bc)"
+
+    util=$(cat "$card/device/gpu_busy_percent" 2>/dev/null)
+    [ -n "$util" ] && echo "rocm_gpu_utilization_percent{${labels}} ${util}"
+
+    vram_used=$(cat "$card/device/mem_info_vram_used" 2>/dev/null)
+    [ -n "$vram_used" ] && echo "rocm_vram_used_bytes{${labels}} ${vram_used}"
+
+    vram_total=$(cat "$card/device/mem_info_vram_total" 2>/dev/null)
+    [ -n "$vram_total" ] && echo "rocm_vram_total_bytes{${labels}} ${vram_total}"
+  done
+} > "$TMPFILE" && mv "$TMPFILE" "$OUTPUT"

ansible/roles/node_exporter/templates/rocm_scraper.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=ROCm metrics scraper timer
+Requires=rocm_scraper.service
+
+[Timer]
+OnBootSec=30s
+OnUnitActiveSec={{ rocm_scrape_interval }}s
+AccuracySec=5s
+
+[Install]
+WantedBy=timers.target