mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-11 19:54:43 +02:00
feat: monitoring — node_exporter + ROCm sysfs collector + alertmanager webhook
Ansible roles: - node_exporter : déploie node_exporter v1.9.1 sur storage-01 et gpu-01 avec textfile collector + services systemd. ROCm activé sur gpu-01 uniquement via group_vars. Collecte métriques GPU via sysfs (temp, VRAM, utilisation) — pas de dépendance rocm-smi (remplacé dans ROCm 7.x) - alertmanager_webhook : service Python sur storage-01 (:9093/webhook) reçoit alertes AlertManager et les route vers ask-agent monitor → Hermes Playbooks: node_exporter + alertmanager_webhook ajoutés à storage-01 et gpu-01 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
857351bcf9
commit
92a882aa94
15 changed files with 354 additions and 2 deletions
88
ansible/roles/alertmanager_webhook/files/webhook.py
Normal file
88
ansible/roles/alertmanager_webhook/files/webhook.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
AlertManager webhook → ask-agent monitor
|
||||
Reçoit les alertes Prometheus et les délègue au profil monitor de Hermes.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import logging
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
from datetime import datetime
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
PORT = int(os.environ.get("WEBHOOK_PORT", 9093))
|
||||
|
||||
|
||||
def format_alert(alert: dict) -> str:
|
||||
name = alert.get("labels", {}).get("alertname", "unknown")
|
||||
severity = alert.get("labels", {}).get("severity", "unknown")
|
||||
instance = alert.get("labels", {}).get("instance", "unknown")
|
||||
summary = alert.get("annotations", {}).get("summary", "")
|
||||
description = alert.get("annotations", {}).get("description", "")
|
||||
status = alert.get("status", "firing")
|
||||
return f"[{status.upper()}] {name} — severity={severity} instance={instance}\n{summary}\n{description}".strip()
|
||||
|
||||
|
||||
class WebhookHandler(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
if self.path != "/webhook":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(length)
|
||||
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
log.error("Invalid JSON payload")
|
||||
self.send_response(400)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
alerts = payload.get("alerts", [])
|
||||
firing = [a for a in alerts if a.get("status") == "firing"]
|
||||
resolved = [a for a in alerts if a.get("status") == "resolved"]
|
||||
|
||||
if not alerts:
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
lines = [f"Alertes Prometheus reçues à {datetime.now().strftime('%H:%M:%S')} :"]
|
||||
for a in firing:
|
||||
lines.append(format_alert(a))
|
||||
for a in resolved:
|
||||
lines.append(format_alert(a))
|
||||
|
||||
message = "\n".join(lines) + "\n\nAnalyse ces alertes : sont-elles critiques ? Quelles actions recommandes-tu ?"
|
||||
|
||||
log.info(f"Forwarding {len(firing)} firing + {len(resolved)} resolved alerts to ask-agent monitor")
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["/usr/local/bin/ask-agent", "monitor", message],
|
||||
capture_output=True, text=True, timeout=120
|
||||
)
|
||||
log.info(f"ask-agent response: {result.stdout[:200]}")
|
||||
except subprocess.TimeoutExpired:
|
||||
log.error("ask-agent timed out")
|
||||
except Exception as e:
|
||||
log.error(f"ask-agent error: {e}")
|
||||
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(b"ok")
|
||||
|
||||
def log_message(self, format, *args):
|
||||
pass # silence HTTP access logs
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
log.info(f"AlertManager webhook listening on :{PORT}")
|
||||
server = HTTPServer(("0.0.0.0", PORT), WebhookHandler)
|
||||
server.serve_forever()
|
||||
Loading…
Add table
Add a link
Reference in a new issue