From 92a882aa943528dc3d91d5aac4f2da8cba53df87 Mon Sep 17 00:00:00 2001 From: alkatrazz Date: Wed, 13 May 2026 21:01:03 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20monitoring=20=E2=80=94=20node=5Fexporte?= =?UTF-8?q?r=20+=20ROCm=20sysfs=20collector=20+=20alertmanager=20webhook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ansible roles: - node_exporter : déploie node_exporter v1.9.1 sur storage-01 et gpu-01 avec textfile collector + services systemd. ROCm activé sur gpu-01 uniquement via group_vars. Collecte métriques GPU via sysfs (temp, VRAM, utilisation) — pas de dépendance rocm-smi (remplacé dans ROCm 7.x) - alertmanager_webhook : service Python sur storage-01 (:9093/webhook) reçoit alertes AlertManager et les route vers ask-agent monitor → Hermes Playbooks: node_exporter + alertmanager_webhook ajoutés à storage-01 et gpu-01 Co-Authored-By: Claude Sonnet 4.6 --- ansible/group_vars/gpu_hosts/vars.yml | 3 + ansible/playbooks/gpu-01.yml | 1 + ansible/playbooks/storage-01.yml | 6 +- .../alertmanager_webhook/defaults/main.yml | 6 ++ .../alertmanager_webhook/files/webhook.py | 88 +++++++++++++++++++ .../alertmanager_webhook/handlers/main.yml | 11 +++ .../roles/alertmanager_webhook/tasks/main.yml | 40 +++++++++ .../templates/alertmanager-webhook.service.j2 | 17 ++++ ansible/roles/node_exporter/defaults/main.yml | 11 +++ ansible/roles/node_exporter/handlers/main.yml | 18 ++++ ansible/roles/node_exporter/tasks/main.yml | 86 ++++++++++++++++++ .../templates/node_exporter.service.j2 | 18 ++++ .../templates/rocm_scraper.service.j2 | 7 ++ .../templates/rocm_scraper.sh.j2 | 33 +++++++ .../templates/rocm_scraper.timer.j2 | 11 +++ 15 files changed, 354 insertions(+), 2 deletions(-) create mode 100644 ansible/roles/alertmanager_webhook/defaults/main.yml create mode 100644 ansible/roles/alertmanager_webhook/files/webhook.py create mode 100644 ansible/roles/alertmanager_webhook/handlers/main.yml create mode 100644 ansible/roles/alertmanager_webhook/tasks/main.yml create mode 100644 ansible/roles/alertmanager_webhook/templates/alertmanager-webhook.service.j2 create mode 100644 ansible/roles/node_exporter/defaults/main.yml create mode 100644 ansible/roles/node_exporter/handlers/main.yml create mode 100644 ansible/roles/node_exporter/tasks/main.yml create mode 100644 ansible/roles/node_exporter/templates/node_exporter.service.j2 create mode 100644 ansible/roles/node_exporter/templates/rocm_scraper.service.j2 create mode 100644 ansible/roles/node_exporter/templates/rocm_scraper.sh.j2 create mode 100644 ansible/roles/node_exporter/templates/rocm_scraper.timer.j2 diff --git a/ansible/group_vars/gpu_hosts/vars.yml b/ansible/group_vars/gpu_hosts/vars.yml index abb67f3..0c7d6dc 100644 --- a/ansible/group_vars/gpu_hosts/vars.yml +++ b/ansible/group_vars/gpu_hosts/vars.yml @@ -1,2 +1,5 @@ --- hsa_override_gfx_version: "10.3.0" + +# node_exporter — ROCm textfile collector activé sur gpu-01 +rocm_exporter_enabled: true diff --git a/ansible/playbooks/gpu-01.yml b/ansible/playbooks/gpu-01.yml index eb3cf23..bbafb01 100644 --- a/ansible/playbooks/gpu-01.yml +++ b/ansible/playbooks/gpu-01.yml @@ -7,3 +7,4 @@ - { role: rocm, tags: [rocm] } - { role: nfs_client, tags: [nfs_client] } - { role: llama_server, tags: [llama_server] } + - { role: node_exporter, tags: [node_exporter] } diff --git a/ansible/playbooks/storage-01.yml b/ansible/playbooks/storage-01.yml index 1d97376..7f3e685 100644 --- a/ansible/playbooks/storage-01.yml +++ b/ansible/playbooks/storage-01.yml @@ -10,5 +10,7 @@ - { role: qdrant, tags: [qdrant] } - { role: postgresql, tags: [postgresql] } - { role: minio, tags: [minio] } - - { role: litellm, tags: [litellm] } - - { role: hermes_agent, tags: [hermes_agent] } + - { role: litellm, tags: [litellm] } + - { role: hermes_agent, tags: [hermes_agent] } + - { role: node_exporter, tags: [node_exporter] } + - { role: alertmanager_webhook, tags: [alertmanager_webhook] } diff --git a/ansible/roles/alertmanager_webhook/defaults/main.yml b/ansible/roles/alertmanager_webhook/defaults/main.yml new file mode 100644 index 0000000..de8cc97 --- /dev/null +++ b/ansible/roles/alertmanager_webhook/defaults/main.yml @@ -0,0 +1,6 @@ +--- +webhook_port: 9093 +webhook_user: hermes +webhook_install_dir: /opt/alertmanager-webhook +litellm_url: "http://127.0.0.1:4000/v1" +litellm_api_key: "lm-studio" diff --git a/ansible/roles/alertmanager_webhook/files/webhook.py b/ansible/roles/alertmanager_webhook/files/webhook.py new file mode 100644 index 0000000..9ea010d --- /dev/null +++ b/ansible/roles/alertmanager_webhook/files/webhook.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +""" +AlertManager webhook → ask-agent monitor +Reçoit les alertes Prometheus et les délègue au profil monitor de Hermes. +""" +import json +import os +import subprocess +import logging +from http.server import HTTPServer, BaseHTTPRequestHandler +from datetime import datetime + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) + +PORT = int(os.environ.get("WEBHOOK_PORT", 9093)) + + +def format_alert(alert: dict) -> str: + name = alert.get("labels", {}).get("alertname", "unknown") + severity = alert.get("labels", {}).get("severity", "unknown") + instance = alert.get("labels", {}).get("instance", "unknown") + summary = alert.get("annotations", {}).get("summary", "") + description = alert.get("annotations", {}).get("description", "") + status = alert.get("status", "firing") + return f"[{status.upper()}] {name} — severity={severity} instance={instance}\n{summary}\n{description}".strip() + + +class WebhookHandler(BaseHTTPRequestHandler): + def do_POST(self): + if self.path != "/webhook": + self.send_response(404) + self.end_headers() + return + + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + + try: + payload = json.loads(body) + except json.JSONDecodeError: + log.error("Invalid JSON payload") + self.send_response(400) + self.end_headers() + return + + alerts = payload.get("alerts", []) + firing = [a for a in alerts if a.get("status") == "firing"] + resolved = [a for a in alerts if a.get("status") == "resolved"] + + if not alerts: + self.send_response(200) + self.end_headers() + return + + lines = [f"Alertes Prometheus reçues à {datetime.now().strftime('%H:%M:%S')} :"] + for a in firing: + lines.append(format_alert(a)) + for a in resolved: + lines.append(format_alert(a)) + + message = "\n".join(lines) + "\n\nAnalyse ces alertes : sont-elles critiques ? Quelles actions recommandes-tu ?" + + log.info(f"Forwarding {len(firing)} firing + {len(resolved)} resolved alerts to ask-agent monitor") + + try: + result = subprocess.run( + ["/usr/local/bin/ask-agent", "monitor", message], + capture_output=True, text=True, timeout=120 + ) + log.info(f"ask-agent response: {result.stdout[:200]}") + except subprocess.TimeoutExpired: + log.error("ask-agent timed out") + except Exception as e: + log.error(f"ask-agent error: {e}") + + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, format, *args): + pass # silence HTTP access logs + + +if __name__ == "__main__": + log.info(f"AlertManager webhook listening on :{PORT}") + server = HTTPServer(("0.0.0.0", PORT), WebhookHandler) + server.serve_forever() diff --git a/ansible/roles/alertmanager_webhook/handlers/main.yml b/ansible/roles/alertmanager_webhook/handlers/main.yml new file mode 100644 index 0000000..4c98df1 --- /dev/null +++ b/ansible/roles/alertmanager_webhook/handlers/main.yml @@ -0,0 +1,11 @@ +--- +- name: Restart alertmanager-webhook + ansible.builtin.systemd: + name: alertmanager-webhook + state: restarted + daemon_reload: true + +- name: Reload nftables + ansible.builtin.systemd: + name: nftables + state: reloaded diff --git a/ansible/roles/alertmanager_webhook/tasks/main.yml b/ansible/roles/alertmanager_webhook/tasks/main.yml new file mode 100644 index 0000000..c8a9c79 --- /dev/null +++ b/ansible/roles/alertmanager_webhook/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: Create webhook install directory + ansible.builtin.file: + path: "{{ webhook_install_dir }}" + state: directory + owner: "{{ webhook_user }}" + group: "{{ webhook_user }}" + mode: '0755' + +- name: Deploy webhook script + ansible.builtin.copy: + src: webhook.py + dest: "{{ webhook_install_dir }}/webhook.py" + owner: "{{ webhook_user }}" + group: "{{ webhook_user }}" + mode: '0755' + notify: Restart alertmanager-webhook + +- name: Deploy webhook systemd service + ansible.builtin.template: + src: alertmanager-webhook.service.j2 + dest: /etc/systemd/system/alertmanager-webhook.service + mode: '0644' + notify: Restart alertmanager-webhook + +- name: Enable and start alertmanager-webhook + ansible.builtin.systemd: + name: alertmanager-webhook + enabled: true + state: started + daemon_reload: true + +- name: Open webhook port in nftables + ansible.builtin.lineinfile: + path: /etc/nftables.conf + insertafter: '# prometheus' + line: " tcp dport {{ webhook_port }} ip saddr 192.168.10.0/24 accept comment \"alertmanager-webhook\"" + state: present + notify: Reload nftables + ignore_errors: true diff --git a/ansible/roles/alertmanager_webhook/templates/alertmanager-webhook.service.j2 b/ansible/roles/alertmanager_webhook/templates/alertmanager-webhook.service.j2 new file mode 100644 index 0000000..296cbf0 --- /dev/null +++ b/ansible/roles/alertmanager_webhook/templates/alertmanager-webhook.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=AlertManager webhook — Hermes monitor bridge +After=network.target hermes-agent.service + +[Service] +User={{ webhook_user }} +Group={{ webhook_user }} +Environment=WEBHOOK_PORT={{ webhook_port }} +ExecStart=/usr/bin/python3 {{ webhook_install_dir }}/webhook.py +Restart=on-failure +RestartSec=10s +StandardOutput=journal +StandardError=journal +SyslogIdentifier=alertmanager-webhook + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/node_exporter/defaults/main.yml b/ansible/roles/node_exporter/defaults/main.yml new file mode 100644 index 0000000..de90c75 --- /dev/null +++ b/ansible/roles/node_exporter/defaults/main.yml @@ -0,0 +1,11 @@ +--- +node_exporter_version: "1.9.1" +node_exporter_port: 9100 +node_exporter_user: node_exporter +node_exporter_install_dir: /opt/node_exporter +node_exporter_textfile_dir: /var/lib/node_exporter/textfile_collector + +# ROCm scraper (gpu-01 uniquement) +rocm_exporter_enabled: false +rocm_exporter_port: 9101 +rocm_scrape_interval: 30 diff --git a/ansible/roles/node_exporter/handlers/main.yml b/ansible/roles/node_exporter/handlers/main.yml new file mode 100644 index 0000000..dd59fb2 --- /dev/null +++ b/ansible/roles/node_exporter/handlers/main.yml @@ -0,0 +1,18 @@ +--- +- name: Restart node_exporter + ansible.builtin.systemd: + name: node_exporter + state: restarted + daemon_reload: true + +- name: Reload nftables + ansible.builtin.systemd: + name: nftables + state: reloaded + +- name: Enable rocm_scraper timer + ansible.builtin.systemd: + name: rocm_scraper.timer + enabled: true + state: restarted + daemon_reload: true diff --git a/ansible/roles/node_exporter/tasks/main.yml b/ansible/roles/node_exporter/tasks/main.yml new file mode 100644 index 0000000..6b17e81 --- /dev/null +++ b/ansible/roles/node_exporter/tasks/main.yml @@ -0,0 +1,86 @@ +--- +- name: Create node_exporter user + ansible.builtin.user: + name: "{{ node_exporter_user }}" + shell: /sbin/nologin + create_home: false + system: true + state: present + +- name: Create textfile collector directory + ansible.builtin.file: + path: "{{ node_exporter_textfile_dir }}" + state: directory + owner: "{{ node_exporter_user }}" + group: "{{ node_exporter_user }}" + mode: '0755' + +- name: Download node_exporter + ansible.builtin.get_url: + url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz" + dest: "/tmp/node_exporter-{{ node_exporter_version }}.tar.gz" + mode: '0644' + +- name: Extract node_exporter + ansible.builtin.unarchive: + src: "/tmp/node_exporter-{{ node_exporter_version }}.tar.gz" + dest: /tmp/ + remote_src: true + creates: "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64/node_exporter" + +- name: Install node_exporter binary + ansible.builtin.copy: + src: "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64/node_exporter" + dest: /usr/local/bin/node_exporter + owner: root + group: root + mode: '0755' + remote_src: true + +- name: Deploy node_exporter systemd service + ansible.builtin.template: + src: node_exporter.service.j2 + dest: /etc/systemd/system/node_exporter.service + mode: '0644' + notify: Restart node_exporter + +- name: Enable and start node_exporter + ansible.builtin.systemd: + name: node_exporter + enabled: true + state: started + daemon_reload: true + +- name: Open node_exporter port in nftables + ansible.builtin.lineinfile: + path: /etc/nftables.conf + insertafter: '# prometheus' + line: " tcp dport {{ node_exporter_port }} accept comment \"node_exporter\"" + state: present + notify: Reload nftables + ignore_errors: true + +- name: Deploy ROCm scraper script + ansible.builtin.template: + src: rocm_scraper.sh.j2 + dest: /usr/local/bin/rocm_scraper.sh + mode: '0755' + when: rocm_exporter_enabled + +- name: Deploy ROCm scraper service + timer + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + mode: '0644' + loop: + - rocm_scraper.service + - rocm_scraper.timer + when: rocm_exporter_enabled + +- name: Enable ROCm scraper timer + ansible.builtin.systemd: + name: rocm_scraper.timer + enabled: true + state: started + daemon_reload: true + when: rocm_exporter_enabled diff --git a/ansible/roles/node_exporter/templates/node_exporter.service.j2 b/ansible/roles/node_exporter/templates/node_exporter.service.j2 new file mode 100644 index 0000000..95fc896 --- /dev/null +++ b/ansible/roles/node_exporter/templates/node_exporter.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=Prometheus Node Exporter +After=network.target + +[Service] +User={{ node_exporter_user }} +Group={{ node_exporter_user }} +ExecStart=/usr/local/bin/node_exporter \ + --web.listen-address=":{{ node_exporter_port }}" \ + --collector.textfile.directory={{ node_exporter_textfile_dir }} \ + --collector.systemd \ + --collector.processes + +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/node_exporter/templates/rocm_scraper.service.j2 b/ansible/roles/node_exporter/templates/rocm_scraper.service.j2 new file mode 100644 index 0000000..70c965f --- /dev/null +++ b/ansible/roles/node_exporter/templates/rocm_scraper.service.j2 @@ -0,0 +1,7 @@ +[Unit] +Description=ROCm metrics scraper for Prometheus textfile collector +After=multi-user.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/rocm_scraper.sh diff --git a/ansible/roles/node_exporter/templates/rocm_scraper.sh.j2 b/ansible/roles/node_exporter/templates/rocm_scraper.sh.j2 new file mode 100644 index 0000000..ab9b9b9 --- /dev/null +++ b/ansible/roles/node_exporter/templates/rocm_scraper.sh.j2 @@ -0,0 +1,33 @@ +#!/bin/bash +# Collecte les métriques GPU AMD via sysfs (pas de rocm-smi requis) +OUTPUT="{{ node_exporter_textfile_dir }}/rocm.prom" +TMPFILE="${OUTPUT}.tmp" + +{ + echo "# HELP rocm_gpu_temperature_celsius GPU temperature in Celsius" + echo "# TYPE rocm_gpu_temperature_celsius gauge" + echo "# HELP rocm_gpu_utilization_percent GPU utilization percentage" + echo "# TYPE rocm_gpu_utilization_percent gauge" + echo "# HELP rocm_vram_used_bytes VRAM used in bytes" + echo "# TYPE rocm_vram_used_bytes gauge" + echo "# HELP rocm_vram_total_bytes VRAM total in bytes" + echo "# TYPE rocm_vram_total_bytes gauge" + + for card in /sys/class/drm/card[0-9]; do + [ -d "$card/device" ] || continue + idx=$(basename "$card" | tr -d 'card') + labels="gpu=\"${idx}\",model=\"gfx1031\"" + + temp_raw=$(cat "$card/device/hwmon/hwmon"*/temp1_input 2>/dev/null | head -1) + [ -n "$temp_raw" ] && echo "rocm_gpu_temperature_celsius{${labels}} $(echo "scale=1; ${temp_raw}/1000" | bc)" + + util=$(cat "$card/device/gpu_busy_percent" 2>/dev/null) + [ -n "$util" ] && echo "rocm_gpu_utilization_percent{${labels}} ${util}" + + vram_used=$(cat "$card/device/mem_info_vram_used" 2>/dev/null) + [ -n "$vram_used" ] && echo "rocm_vram_used_bytes{${labels}} ${vram_used}" + + vram_total=$(cat "$card/device/mem_info_vram_total" 2>/dev/null) + [ -n "$vram_total" ] && echo "rocm_vram_total_bytes{${labels}} ${vram_total}" + done +} > "$TMPFILE" && mv "$TMPFILE" "$OUTPUT" diff --git a/ansible/roles/node_exporter/templates/rocm_scraper.timer.j2 b/ansible/roles/node_exporter/templates/rocm_scraper.timer.j2 new file mode 100644 index 0000000..b61a85e --- /dev/null +++ b/ansible/roles/node_exporter/templates/rocm_scraper.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=ROCm metrics scraper timer +Requires=rocm_scraper.service + +[Timer] +OnBootSec=30s +OnUnitActiveSec={{ rocm_scrape_interval }}s +AccuracySec=5s + +[Install] +WantedBy=timers.target