mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-08 19:44:41 +02:00
92a882aa94
Ansible roles: - node_exporter : déploie node_exporter v1.9.1 sur storage-01 et gpu-01 avec textfile collector + services systemd. ROCm activé sur gpu-01 uniquement via group_vars. Collecte métriques GPU via sysfs (temp, VRAM, utilisation) — pas de dépendance rocm-smi (remplacé dans ROCm 7.x) - alertmanager_webhook : service Python sur storage-01 (:9093/webhook) reçoit alertes AlertManager et les route vers ask-agent monitor → Hermes Playbooks: node_exporter + alertmanager_webhook ajoutés à storage-01 et gpu-01 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
88 lines
3 KiB
Python
88 lines
3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
AlertManager webhook → ask-agent monitor
|
|
Reçoit les alertes Prometheus et les délègue au profil monitor de Hermes.
|
|
"""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import logging
|
|
from http.server import HTTPServer, BaseHTTPRequestHandler
|
|
from datetime import datetime
|
|
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
|
log = logging.getLogger(__name__)
|
|
|
|
PORT = int(os.environ.get("WEBHOOK_PORT", 9093))
|
|
|
|
|
|
def format_alert(alert: dict) -> str:
|
|
name = alert.get("labels", {}).get("alertname", "unknown")
|
|
severity = alert.get("labels", {}).get("severity", "unknown")
|
|
instance = alert.get("labels", {}).get("instance", "unknown")
|
|
summary = alert.get("annotations", {}).get("summary", "")
|
|
description = alert.get("annotations", {}).get("description", "")
|
|
status = alert.get("status", "firing")
|
|
return f"[{status.upper()}] {name} — severity={severity} instance={instance}\n{summary}\n{description}".strip()
|
|
|
|
|
|
class WebhookHandler(BaseHTTPRequestHandler):
|
|
def do_POST(self):
|
|
if self.path != "/webhook":
|
|
self.send_response(404)
|
|
self.end_headers()
|
|
return
|
|
|
|
length = int(self.headers.get("Content-Length", 0))
|
|
body = self.rfile.read(length)
|
|
|
|
try:
|
|
payload = json.loads(body)
|
|
except json.JSONDecodeError:
|
|
log.error("Invalid JSON payload")
|
|
self.send_response(400)
|
|
self.end_headers()
|
|
return
|
|
|
|
alerts = payload.get("alerts", [])
|
|
firing = [a for a in alerts if a.get("status") == "firing"]
|
|
resolved = [a for a in alerts if a.get("status") == "resolved"]
|
|
|
|
if not alerts:
|
|
self.send_response(200)
|
|
self.end_headers()
|
|
return
|
|
|
|
lines = [f"Alertes Prometheus reçues à {datetime.now().strftime('%H:%M:%S')} :"]
|
|
for a in firing:
|
|
lines.append(format_alert(a))
|
|
for a in resolved:
|
|
lines.append(format_alert(a))
|
|
|
|
message = "\n".join(lines) + "\n\nAnalyse ces alertes : sont-elles critiques ? Quelles actions recommandes-tu ?"
|
|
|
|
log.info(f"Forwarding {len(firing)} firing + {len(resolved)} resolved alerts to ask-agent monitor")
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
["/usr/local/bin/ask-agent", "monitor", message],
|
|
capture_output=True, text=True, timeout=120
|
|
)
|
|
log.info(f"ask-agent response: {result.stdout[:200]}")
|
|
except subprocess.TimeoutExpired:
|
|
log.error("ask-agent timed out")
|
|
except Exception as e:
|
|
log.error(f"ask-agent error: {e}")
|
|
|
|
self.send_response(200)
|
|
self.end_headers()
|
|
self.wfile.write(b"ok")
|
|
|
|
def log_message(self, format, *args):
|
|
pass # silence HTTP access logs
|
|
|
|
|
|
if __name__ == "__main__":
|
|
log.info(f"AlertManager webhook listening on :{PORT}")
|
|
server = HTTPServer(("0.0.0.0", PORT), WebhookHandler)
|
|
server.serve_forever()
|