From 4a7a5ac9fc028a39c255302a62092f344311fd78 Mon Sep 17 00:00:00 2001 From: alkatrazz Date: Tue, 2 Jun 2026 23:53:53 +0200 Subject: [PATCH] =?UTF-8?q?feat(auto-improve):=20workflow=20Hermes=20d'ana?= =?UTF-8?q?lyse=20et=20am=C3=A9lioration=20de=20la=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipeline complet dans feature/hermes-auto-improve : - auto-improve.py : analyse admin/ avec hermes --profile funk-ai, applique les améliorations, committe dans la branche, ré-ingère dans Qdrant - trigger-server.py : HTTP 9095 (POST /trigger, /trigger/all, /trigger/subdir, /trigger/dry-run) pour déclenchement depuis n8n ou manuellement - Ansible role hermes_auto_improve : déploie scripts + service systemd sur s01 - n8n workflow "Hermes Auto-Improve" (ID: CX72UUOempE1YtNL) : schedule hebdo dimanche 09h + webhook manuel → analyse → résumé LLM → email Tout reste dans cette branche — rien ne touche main avant review. Co-Authored-By: Claude Sonnet 4.6 --- ansible/playbooks/storage-01.yml | 5 +- .../hermes_auto_improve/defaults/main.yml | 6 + .../hermes_auto_improve/handlers/main.yml | 11 + .../roles/hermes_auto_improve/tasks/main.yml | 43 ++++ .../templates/hermes-auto-improve.service.j2 | 17 ++ tools/hermes-auto-improve/auto-improve.py | 221 ++++++++++++++++++ tools/hermes-auto-improve/trigger-server.py | 128 ++++++++++ 7 files changed, 429 insertions(+), 2 deletions(-) create mode 100644 ansible/roles/hermes_auto_improve/defaults/main.yml create mode 100644 ansible/roles/hermes_auto_improve/handlers/main.yml create mode 100644 ansible/roles/hermes_auto_improve/tasks/main.yml create mode 100644 ansible/roles/hermes_auto_improve/templates/hermes-auto-improve.service.j2 create mode 100644 tools/hermes-auto-improve/auto-improve.py create mode 100644 tools/hermes-auto-improve/trigger-server.py diff --git a/ansible/playbooks/storage-01.yml b/ansible/playbooks/storage-01.yml index 5493981..4f46343 100644 --- a/ansible/playbooks/storage-01.yml +++ b/ansible/playbooks/storage-01.yml @@ -14,5 +14,6 @@ - { role: hermes_agent, tags: [hermes_agent] } - { role: rag, tags: [rag] } - { role: node_exporter, tags: [node_exporter] } - - { role: alertmanager_webhook, tags: [alertmanager_webhook] } - - { role: postfix_relay, tags: [postfix_relay] } + - { role: alertmanager_webhook, tags: [alertmanager_webhook] } + - { role: postfix_relay, tags: [postfix_relay] } + - { role: hermes_auto_improve, tags: [hermes_auto_improve] } diff --git a/ansible/roles/hermes_auto_improve/defaults/main.yml b/ansible/roles/hermes_auto_improve/defaults/main.yml new file mode 100644 index 0000000..bacc948 --- /dev/null +++ b/ansible/roles/hermes_auto_improve/defaults/main.yml @@ -0,0 +1,6 @@ +--- +hermes_auto_improve_port: 9095 +hermes_auto_improve_repo: /srv/data/lab/Funk-lab +hermes_auto_improve_branch: feature/hermes-auto-improve +hermes_auto_improve_state_dir: /var/lib/hermes-auto-improve +hermes_auto_improve_log: /var/log/hermes-auto-improve.log diff --git a/ansible/roles/hermes_auto_improve/handlers/main.yml b/ansible/roles/hermes_auto_improve/handlers/main.yml new file mode 100644 index 0000000..e8f1d3a --- /dev/null +++ b/ansible/roles/hermes_auto_improve/handlers/main.yml @@ -0,0 +1,11 @@ +--- +- name: Restart hermes-auto-improve-server + ansible.builtin.systemd: + name: hermes-auto-improve + state: restarted + daemon_reload: true + +- name: Reload nftables + ansible.builtin.service: + name: nftables + state: reloaded diff --git a/ansible/roles/hermes_auto_improve/tasks/main.yml b/ansible/roles/hermes_auto_improve/tasks/main.yml new file mode 100644 index 0000000..c9ffa38 --- /dev/null +++ b/ansible/roles/hermes_auto_improve/tasks/main.yml @@ -0,0 +1,43 @@ +--- +- name: Create state directory + ansible.builtin.file: + path: "{{ hermes_auto_improve_state_dir }}" + state: directory + mode: '0755' + +- name: Deploy auto-improve script + ansible.builtin.copy: + src: "{{ playbook_dir }}/../tools/hermes-auto-improve/auto-improve.py" + dest: /usr/local/bin/hermes-auto-improve + mode: '0755' + notify: Restart hermes-auto-improve-server + +- name: Deploy trigger server + ansible.builtin.copy: + src: "{{ playbook_dir }}/../tools/hermes-auto-improve/trigger-server.py" + dest: /usr/local/bin/hermes-auto-improve-server + mode: '0755' + notify: Restart hermes-auto-improve-server + +- name: Deploy systemd service + ansible.builtin.template: + src: hermes-auto-improve.service.j2 + dest: /etc/systemd/system/hermes-auto-improve.service + mode: '0644' + notify: Restart hermes-auto-improve-server + +- name: Enable and start service + ansible.builtin.systemd: + name: hermes-auto-improve + enabled: true + state: started + daemon_reload: true + +- name: Open port in nftables + ansible.builtin.lineinfile: + path: /etc/nftables.conf + insertafter: '# prometheus' + line: " tcp dport {{ hermes_auto_improve_port }} ip saddr { 10.42.0.0/16, 192.168.10.0/24 } accept comment \"hermes-auto-improve\"" + state: present + notify: Reload nftables + ignore_errors: true diff --git a/ansible/roles/hermes_auto_improve/templates/hermes-auto-improve.service.j2 b/ansible/roles/hermes_auto_improve/templates/hermes-auto-improve.service.j2 new file mode 100644 index 0000000..18e9cdb --- /dev/null +++ b/ansible/roles/hermes_auto_improve/templates/hermes-auto-improve.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Hermes Auto-Improve Trigger Server +After=network-online.target hermes-agent.service +Wants=network-online.target + +[Service] +ExecStart=/usr/bin/python3 /usr/local/bin/hermes-auto-improve-server +Restart=on-failure +RestartSec=15s +Environment=PYTHONUNBUFFERED=1 + +StandardOutput=journal +StandardError=journal +SyslogIdentifier=hermes-auto-improve + +[Install] +WantedBy=multi-user.target diff --git a/tools/hermes-auto-improve/auto-improve.py b/tools/hermes-auto-improve/auto-improve.py new file mode 100644 index 0000000..38ddd4d --- /dev/null +++ b/tools/hermes-auto-improve/auto-improve.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +""" +hermes-auto-improve — Analyse les docs admin/ avec Hermes. +Génère des rapports d'amélioration dans la feature branch, ré-ingère dans Qdrant. + +Usage : + python3 auto-improve.py [--all] [--subdir ia] [--dry-run] +""" + +import argparse +import json +import subprocess +import sys +from datetime import datetime +from pathlib import Path + +REPO_DIR = Path("/srv/data/lab/Funk-lab") +BRANCH = "feature/hermes-auto-improve" +ADMIN_DIR = REPO_DIR / "admin" +REPORTS_DIR = REPO_DIR / "tools/hermes-auto-improve/reports" +STATE_FILE = Path("/var/lib/hermes-auto-improve/state.json") +MAX_FILES = 6 # fichiers par run (sans --all) + +HERMES_CMD = [ + "sudo", "-i", "-u", "hermes", "bash", "-c", + 'HERMES_HOME=/srv/data/hermes hermes --profile funk-ai -z "{prompt}"' +] + + +def git(args: list, check=False) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", "-C", str(REPO_DIR)] + args, + capture_output=True, text=True, check=check + ) + + +def ask_hermes(prompt: str) -> str: + safe = prompt.replace("'", "'\\''").replace('"', '\\"') + cmd = [ + "sudo", "-i", "-u", "hermes", "bash", "-c", + f'HERMES_HOME=/srv/data/hermes hermes --profile funk-ai -z "{safe}"' + ] + result = subprocess.run(cmd, capture_output=True, text=True, timeout=180) + return result.stdout.strip() + + +def analyze_file(fpath: Path) -> dict: + rel = str(fpath.relative_to(REPO_DIR)) + content = fpath.read_text(encoding="utf-8") + + prompt = f"""Tu es l'agent d'analyse du homelab Funk. Analyse ce fichier de documentation : + +Fichier : {rel} +--- +{content[:4000]} +--- + +Réponds en JSON strict (pas de markdown) : +{{ + "status": "ok" | "à_améliorer", + "score": 1-10, + "problèmes": ["liste des problèmes concrets"], + "améliorations": ["liste des améliorations prioritaires"], + "contenu_amélioré": "contenu markdown complet si status=à_améliorer, sinon null" +}} + +Critères d'évaluation : +- Informations obsolètes ou incorrectes par rapport à l'état actuel du homelab +- Sections manquantes importantes pour un ops +- Commandes ou chemins érronés +- Documentation incomplète pour reproduire une procédure + +Si le document est à jour et complet, mettre status=ok et contenu_amélioré=null.""" + + raw = ask_hermes(prompt) + + # Extraire le JSON de la réponse Hermes (qui peut contenir du texte autour) + try: + start = raw.find("{") + end = raw.rfind("}") + 1 + if start >= 0 and end > start: + return json.loads(raw[start:end]) + except json.JSONDecodeError: + pass + + return { + "status": "erreur", + "score": 0, + "problèmes": ["Hermes n'a pas retourné de JSON valide"], + "améliorations": [], + "contenu_amélioré": None, + "raw_response": raw[:500] + } + + +def load_state() -> dict: + if STATE_FILE.exists(): + return json.loads(STATE_FILE.read_text()) + return {"processed": [], "last_run": None} + + +def save_state(state: dict): + STATE_FILE.parent.mkdir(parents=True, exist_ok=True) + STATE_FILE.write_text(json.dumps(state, indent=2, ensure_ascii=False)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--all", action="store_true", help="Traiter tous les fichiers") + parser.add_argument("--subdir", default=None, help="Sous-dossier admin/ à analyser (ex: ia, k8s)") + parser.add_argument("--dry-run", action="store_true", help="Analyser sans commit ni push") + parser.add_argument("--reset", action="store_true", help="Réinitialiser l'état (retraiter tous les fichiers)") + args = parser.parse_args() + + # ── Sync branche ──────────────────────────────────────────── + git(["fetch", "origin"]) + checkout = git(["checkout", BRANCH]) + if checkout.returncode != 0: + git(["checkout", "-b", BRANCH, f"origin/{BRANCH}"]) + git(["pull", "origin", BRANCH]) + + # ── Sélection des fichiers ─────────────────────────────────── + scan_root = ADMIN_DIR / args.subdir if args.subdir else ADMIN_DIR + all_files = sorted(scan_root.rglob("*.md")) + + state = load_state() + if args.reset: + state["processed"] = [] + + todo = [f for f in all_files if str(f) not in state["processed"]] + if not todo: + state["processed"] = [] + todo = all_files + + batch = todo if args.all else todo[:MAX_FILES] + + # ── Analyse ────────────────────────────────────────────────── + run_ts = datetime.now().strftime("%Y-%m-%d_%H-%M") + results = [] + + for fpath in batch: + rel = str(fpath.relative_to(REPO_DIR)) + print(f" → Analyse : {rel}", flush=True) + analysis = analyze_file(fpath) + analysis["file"] = rel + results.append(analysis) + + # Appliquer les améliorations si Hermes en a proposé + if analysis.get("status") == "à_améliorer" and analysis.get("contenu_amélioré"): + if not args.dry_run: + fpath.write_text(analysis["contenu_amélioré"], encoding="utf-8") + print(f" ✓ Améliorations appliquées (score: {analysis.get('score', '?')}/10)", flush=True) + elif analysis.get("status") == "ok": + print(f" ✓ OK (score: {analysis.get('score', '?')}/10)", flush=True) + else: + print(f" ⚠ {analysis.get('status', '?')}", flush=True) + + state["processed"].append(str(fpath)) + + save_state(state) + + # ── Rapport ────────────────────────────────────────────────── + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + report_file = REPORTS_DIR / f"{run_ts}.json" + report_file.write_text( + json.dumps({"run": run_ts, "results": results}, indent=2, ensure_ascii=False) + ) + + improved = [r["file"] for r in results if r.get("status") == "à_améliorer" and r.get("contenu_amélioré")] + + # ── Commit + push ──────────────────────────────────────────── + if not args.dry_run: + git(["add", str(report_file.relative_to(REPO_DIR))]) + for f in improved: + git(["add", f]) + + if git(["diff", "--cached", "--quiet"]).returncode != 0: + msg_files = f"\n\nFichiers améliorés :\n" + "\n".join(f" - {f}" for f in improved) if improved else "" + git(["commit", "-m", + f"docs(auto): analyse Hermes {run_ts}{msg_files}\n\n" + f"Co-Authored-By: Hermes "]) + git(["push", "origin", BRANCH]) + print(f"\n✓ Commit + push → {BRANCH}", flush=True) + + # ── Ré-ingestion Qdrant ────────────────────────────────── + if improved: + print("\n⏳ Ré-ingestion Qdrant...", flush=True) + ingest = subprocess.run( + ["/usr/local/bin/rag-ingest"], + capture_output=True, text=True, timeout=300 + ) + print(f" {ingest.stdout.strip()[-200:] if ingest.stdout else 'OK'}", flush=True) + + # ── Résumé JSON (lu par n8n) ───────────────────────────────── + summary = { + "run": run_ts, + "branch": BRANCH, + "analyzed": len(results), + "improved": len(improved), + "ok": len([r for r in results if r.get("status") == "ok"]), + "errors": len([r for r in results if r.get("status") == "erreur"]), + "files_improved": improved, + "dry_run": args.dry_run, + "details": [ + { + "file": r["file"], + "status": r.get("status"), + "score": r.get("score"), + "issues": r.get("problèmes", []), + "suggestions": r.get("améliorations", []) + } + for r in results + ] + } + + print("\n" + json.dumps(summary, indent=2, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/hermes-auto-improve/trigger-server.py b/tools/hermes-auto-improve/trigger-server.py new file mode 100644 index 0000000..ad7b806 --- /dev/null +++ b/tools/hermes-auto-improve/trigger-server.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +""" +Serveur HTTP minimal pour déclencher hermes-auto-improve depuis n8n. +Écoute sur 0.0.0.0:9095. + +POST /trigger → lance auto-improve.py (run standard, MAX_FILES fichiers) +POST /trigger/all → lance avec --all +POST /trigger/subdir → body JSON {"subdir": "ia"} → lance sur un sous-dossier +POST /trigger/dry-run → analyse sans commit +GET /status → dernier état (state.json) +GET /health → {"ok": true} +""" + +import json +import subprocess +import threading +from datetime import datetime +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +SCRIPT = "/usr/local/bin/hermes-auto-improve" +STATE_FILE = Path("/var/lib/hermes-auto-improve/state.json") +LOG_FILE = Path("/var/log/hermes-auto-improve.log") + +_lock = threading.Lock() +_running = False +_last_result = None + + +def run_script(extra_args: list = None) -> dict: + global _running, _last_result + with _lock: + if _running: + return {"error": "already_running"} + _running = True + + try: + cmd = [SCRIPT] + (extra_args or []) + LOG_FILE.parent.mkdir(parents=True, exist_ok=True) + with open(LOG_FILE, "a") as log: + log.write(f"\n=== {datetime.now().isoformat()} — {' '.join(cmd)} ===\n") + result = subprocess.run( + cmd, capture_output=True, text=True, timeout=600 + ) + log.write(result.stdout) + if result.stderr: + log.write(f"STDERR: {result.stderr}") + + # Extraire le JSON du dernier bloc de stdout + out = result.stdout + start = out.rfind("{") + end = out.rfind("}") + 1 + if start >= 0 and end > start: + _last_result = json.loads(out[start:end]) + else: + _last_result = {"error": "no_json", "stdout": out[-500:]} + return _last_result + + except subprocess.TimeoutExpired: + _last_result = {"error": "timeout"} + return _last_result + except Exception as e: + _last_result = {"error": str(e)} + return _last_result + finally: + _running = False + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + pass # silencer les logs HTTP par défaut + + def send_json(self, code: int, data: dict): + body = json.dumps(data, ensure_ascii=False, indent=2).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if self.path == "/health": + self.send_json(200, {"ok": True, "running": _running}) + elif self.path == "/status": + if STATE_FILE.exists(): + state = json.loads(STATE_FILE.read_text()) + self.send_json(200, {"state": state, "last_result": _last_result, "running": _running}) + else: + self.send_json(200, {"state": None, "last_result": _last_result, "running": _running}) + else: + self.send_json(404, {"error": "not_found"}) + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = json.loads(self.rfile.read(length)) if length > 0 else {} + + if self.path == "/trigger": + result = run_script() + self.send_json(200, result) + + elif self.path == "/trigger/all": + result = run_script(["--all"]) + self.send_json(200, result) + + elif self.path == "/trigger/subdir": + subdir = body.get("subdir") + if not subdir: + self.send_json(400, {"error": "missing 'subdir' in body"}) + return + result = run_script(["--subdir", subdir]) + self.send_json(200, result) + + elif self.path == "/trigger/dry-run": + result = run_script(["--dry-run"]) + self.send_json(200, result) + + elif self.path == "/trigger/reset": + result = run_script(["--reset"]) + self.send_json(200, result) + + else: + self.send_json(404, {"error": "not_found"}) + + +if __name__ == "__main__": + server = HTTPServer(("0.0.0.0", 9095), Handler) + print("hermes-auto-improve trigger server — port 9095", flush=True) + server.serve_forever()