akadmin/Funk-lab
1
0
Fork 0
mirror of https://github.com/Alkatrazz24/Funk-lab.git synced 2026-07-11 02:34:42 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor(hermes-doc-enrich): append-only — section État vérifié, jamais de réécriture

Browse source 
Le script n'écrase plus les docs existants. Pour chaque fichier admin/ :
- Inspecte le cluster en read-only (systemctl, kubectl, nft, etc.)
- Demande à Hermes d'écrire UNIQUEMENT une section "## État vérifié — DATE"
- Appende cette section au doc (ou remplace la précédente)
- Commit local dans hermes/daily-work, pas de push
- rag-ingest Qdrant à la fin du batch

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
ALI YESILKAYA 2026-06-05 12:12:18 +02:00 • committed by Alkatrazz
parent d79ebf7d36
commit 3780c90804
1 changed files with 133 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

240
tools/hermes-auto-improve/doc-rewrite.py
View file

@ -1,8 +1,9 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
hermes-doc-rewrite — Vérifie l'état réel du cluster (LECTURE SEULE) hermes-doc-enrich — Hermes inspecte le cluster (LECTURE SEULE) et ajoute
et réécrit les docs admin/ dans hermes/daily-work. une section "## État vérifié" à chaque doc admin/.
Commit local uniquement, pas de push. Jamais de suppression ni de réécriture — ajout uniquement.
Commit local dans hermes/daily-work + rag-ingest Qdrant à la fin.
""" """
import argparse import argparse
@ -14,84 +15,76 @@ from pathlib import Path
REPO_DIR = Path("/srv/data/lab/Funk-lab") REPO_DIR = Path("/srv/data/lab/Funk-lab")
BRANCH = "hermes/daily-work" BRANCH = "hermes/daily-work"
ADMIN_DIR = REPO_DIR / "admin" ADMIN_DIR = REPO_DIR / "admin"
STATE_FILE = Path("/var/lib/hermes-auto-improve/rewrite-state.json") RAG_DOCS = Path("/srv/data/rag/docs")
STATE_FILE = Path("/var/lib/hermes-auto-improve/enrich-state.json")
MAX_FILES = 3 MAX_FILES = 3
SECTION_MARKER = "## État vérifié"
# ── Commandes read-only par fichier (clé = substring du chemin relatif) ────── # ── Commandes read-only par fichier (clé = substring du chemin relatif) ──────
CONTEXT_MAP = { CONTEXT_MAP = {
"ia/litellm": [ "ia/litellm": [
"systemctl status litellm --no-pager -n 30", "systemctl status litellm --no-pager -n 5",
"cat /etc/litellm/config.yaml", "cat /etc/litellm/config.yaml",
], ],
"ia/hermes.md": [ "ia/hermes.md": [
"systemctl status hermes --no-pager -n 20", "systemctl status hermes --no-pager -n 5",
"cat /srv/data/hermes/config.yaml 2>/dev/null | head -60", "cat /srv/data/hermes/config.yaml 2>/dev/null | head -40",
"ls /opt/hermes/ 2>/dev/null",
], ],
"ia/hermes-souls": [ "ia/hermes-souls": [
"ls -la ~hermes/ 2>/dev/null",
"ls /srv/data/hermes/ 2>/dev/null", "ls /srv/data/hermes/ 2>/dev/null",
"cat /srv/data/hermes/config.yaml 2>/dev/null | grep -A5 'profile' | head -30", "cat /srv/data/hermes/config.yaml 2>/dev/null | grep -A3 'profile' | head -20",
], ],
"ia/hermes-voice": [ "ia/hermes-voice": [
"systemctl status hermes-voice 2>/dev/null --no-pager -n 10 || echo 'service hermes-voice absent'", "systemctl status hermes-voice 2>/dev/null --no-pager -n 5 || echo 'service hermes-voice absent'",
"ls /opt/hermes/ 2>/dev/null",
], ],
"ia/llama_server": [ "ia/llama_server": [
"systemctl cat llama-server 2>/dev/null | head -40", "systemctl cat llama-server 2>/dev/null | grep -E 'ExecStart|Environment|Restart' | head -10",
"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 gpu-01 'systemctl status llama-server --no-pager -n 15; systemctl status llama-server-cpu llama-server-cpu2 --no-pager -n 5 2>/dev/null' 2>/dev/null || echo 'gpu-01 inaccessible'", "ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 gpu-01 'systemctl status llama-server --no-pager -n 5' 2>/dev/null || echo 'gpu-01 inaccessible'",
], ],
"ia/rag": [ "ia/rag": [
"systemctl status qdrant --no-pager -n 15", "systemctl status qdrant --no-pager -n 5",
"ls /srv/data/rag/docs/ 2>/dev/null | wc -l", "ls /srv/data/rag/docs/ 2>/dev/null | wc -l",
"/usr/local/bin/rag-query 'hermes configuration' 2>&1 | head -20", "/usr/local/bin/rag-query 'hermes' 2>&1 | head -10",
], ],
"ia/rocm": [ "ia/rocm": [
"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 gpu-01 'uname -r; ls /opt/rocm/bin/ 2>/dev/null | head -5; rocm-smi 2>/dev/null | head -15' 2>/dev/null || echo 'gpu-01 inaccessible'", "ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 gpu-01 'uname -r; rocm-smi 2>/dev/null | head -10' 2>/dev/null || echo 'gpu-01 inaccessible'",
], ],
"ia/alertmanager-webhook": [ "ia/alertmanager-webhook": [
"systemctl status alertmanager-webhook --no-pager -n 15", "systemctl status alertmanager-webhook --no-pager -n 5",
"cat /etc/alertmanager-webhook/*.yaml 2>/dev/null | head -50", "cat /etc/alertmanager-webhook/*.yaml 2>/dev/null | head -30",
], ],
"infra/dnsmasq": [ "infra/dnsmasq": [
"systemctl status dnsmasq --no-pager -n 10", "systemctl status dnsmasq --no-pager -n 5",
"cat /etc/dnsmasq.d/funk.conf 2>/dev/null || ls /etc/dnsmasq.d/",
"dig @127.0.0.1 compute-01.lab.local +short 2>/dev/null", "dig @127.0.0.1 compute-01.lab.local +short 2>/dev/null",
"dig @127.0.0.1 openwebui.lab.local +short 2>/dev/null", "dig @127.0.0.1 openwebui.lab.local +short 2>/dev/null",
], ],
"infra/nfs": [ "infra/nfs": [
"systemctl status nfs-server --no-pager -n 10", "systemctl status nfs-server --no-pager -n 5",
"exportfs -v 2>/dev/null", "exportfs -v 2>/dev/null",
"showmount -e localhost 2>/dev/null",
], ],
"infra/reseau": [ "infra/reseau": [
"ip addr show | grep -E 'inet |^[0-9]+'", "ip addr show | grep 'inet '",
"ip route show", "nft list ruleset 2>/dev/null | grep -E 'chain|policy|iif|oif|dport' | head -30",
"nft list ruleset 2>/dev/null | head -80",
], ],
"infra/ssh": [ "infra/ssh": [
"sshd -T 2>/dev/null | grep -E 'permitrootlogin|passwordauth|pubkeyauth|^port '", "sshd -T 2>/dev/null | grep -E 'permitrootlogin|passwordauth|pubkeyauth|^port '",
"cat /etc/ssh/sshd_config.d/*.conf 2>/dev/null | head -30",
], ],
"infra/email": [ "infra/email": [
"systemctl status postfix --no-pager -n 10", "systemctl status postfix --no-pager -n 5",
"postconf -n 2>/dev/null | grep -vi 'pass'", "postconf -n 2>/dev/null | grep -vi 'pass' | grep -E 'relay|tls|sasl'",
], ],
"install/storage-01": [ "install/storage-01": [
"hostnamectl", "hostnamectl | grep -E 'Hostname|OS|Kernel'",
"df -h", "df -h | grep -v tmpfs",
"free -h", "free -h",
"lsblk -o NAME,SIZE,TYPE,MOUNTPOINT", "systemctl list-units --state=failed --no-pager 2>/dev/null | grep -v '^$'",
"systemctl list-units --state=failed --no-pager 2>/dev/null | head -10",
], ],
"install/gpu-01": [ "install/gpu-01": [
"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 gpu-01 'hostnamectl; df -h; free -h; uname -r; lsblk -o NAME,SIZE,TYPE,MOUNTPOINT' 2>/dev/null || echo 'gpu-01 inaccessible'", "ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 gpu-01 'hostnamectl | grep -E \"Hostname|OS|Kernel\"; df -h | grep -v tmpfs; systemctl list-units --state=failed --no-pager 2>/dev/null' 2>/dev/null || echo 'gpu-01 inaccessible'",
"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 gpu-01 'systemctl list-units --state=failed --no-pager 2>/dev/null' 2>/dev/null",
], ],
"install/kubernetes": [ "install/kubernetes": [
"kubectl get nodes -o wide 2>/dev/null", "kubectl get nodes -o wide 2>/dev/null",
"kubectl version 2>/dev/null | head -4", "kubectl get pods -A --field-selector=status.phase!=Running 2>/dev/null | head -10",
"kubectl get pods -A --field-selector=status.phase!=Running 2>/dev/null | head -20",
], ],
"k8s/argocd": [ "k8s/argocd": [
"kubectl get applications -n argocd -o wide 2>/dev/null", "kubectl get applications -n argocd -o wide 2>/dev/null",
@ -99,7 +92,6 @@ CONTEXT_MAP = {
], ],
"k8s/monitoring": [ "k8s/monitoring": [
"kubectl get pods -n monitoring 2>/dev/null", "kubectl get pods -n monitoring 2>/dev/null",
"kubectl get svc -n monitoring 2>/dev/null",
"kubectl get ingress -n monitoring 2>/dev/null", "kubectl get ingress -n monitoring 2>/dev/null",
], ],
"k8s/n8n": [ "k8s/n8n": [
@ -112,40 +104,35 @@ CONTEXT_MAP = {
], ],
"k8s/talos": [ "k8s/talos": [
"kubectl get nodes 2>/dev/null", "kubectl get nodes 2>/dev/null",
"talosctl version 2>/dev/null | head -5", "talosctl version 2>/dev/null | head -4",
], ],
"k8s/k9s": [ "k8s/k9s": [
"k9s version 2>/dev/null || echo 'k9s: check depuis storage-01'", "k9s version 2>/dev/null || echo 'k9s non installé sur storage-01'",
"kubectl config get-contexts 2>/dev/null", "kubectl config get-contexts 2>/dev/null",
], ],
"ops/ansible": [ "ops/ansible": [
"ansible --version 2>/dev/null | head -3", "ansible --version 2>/dev/null | head -3",
"ls /srv/data/lab/Funk-lab/ansible/roles/", "ls /srv/data/lab/Funk-lab/ansible/roles/",
"ls /srv/data/lab/Funk-lab/ansible/playbooks/",
], ],
"ops/cluster": [ "ops/cluster": [
"kubectl get nodes -o wide 2>/dev/null", "kubectl get nodes -o wide 2>/dev/null",
"systemctl list-units --state=failed --no-pager 2>/dev/null | head -10", "systemctl list-units --state=failed --no-pager 2>/dev/null | grep -v '^$' | head -10",
"df -h /srv/data 2>/dev/null",
], ],
"ops/systeme": [ "ops/systeme": [
"uname -r", "uname -r",
"hostnamectl",
"uptime", "uptime",
"df -h", "df -h | grep -v tmpfs",
"free -h", "free -h",
"systemctl list-units --state=failed --no-pager 2>/dev/null | head -10", "systemctl list-units --state=failed --no-pager 2>/dev/null | grep -v '^$' | head -10",
], ],
"README": [ "README": [
"ls /srv/data/lab/Funk-lab/admin/",
"systemctl is-active litellm hermes dnsmasq nfs-server qdrant postfix 2>/dev/null", "systemctl is-active litellm hermes dnsmasq nfs-server qdrant postfix 2>/dev/null",
], ],
} }
DEFAULT_COMMANDS = [ DEFAULT_COMMANDS = [
"date", "date",
"df -h /srv/data 2>/dev/null", "systemctl list-units --state=failed --no-pager 2>/dev/null | grep -v '^$' | head -5",
"systemctl list-units --state=failed --no-pager 2>/dev/null | head -5",
] ]
@ -176,7 +163,7 @@ def run_cmd(cmd):
cmd, shell=True, capture_output=True, text=True, timeout=30 cmd, shell=True, capture_output=True, text=True, timeout=30
) )
out = (r.stdout + r.stderr).strip() out = (r.stdout + r.stderr).strip()
return out[:2000] if out else "(aucune sortie)" return out[:1500] if out else "(aucune sortie)"
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
return "(timeout)" return "(timeout)"
except Exception as e: except Exception as e:
@ -210,48 +197,81 @@ def ask_hermes(prompt):
return r.stdout.strip() return r.stdout.strip()
def rewrite_doc(fpath): def build_verified_section(fpath, cluster_state, today):
rel = str(fpath.relative_to(REPO_DIR)) rel = str(fpath.relative_to(REPO_DIR))
content = fpath.read_text(encoding="utf-8", errors="replace") content = fpath.read_text(encoding="utf-8", errors="replace")
prompt = f"""Tu es l'agent de documentation du homelab Funk.
MISSION : écrire UNIQUEMENT une nouvelle section à ajouter à la fin de ce document.
CONTRAINTE ABSOLUE : ne touche pas au contenu existant — ajout uniquement.
Fichier concerné : {rel}
--- ÉTAT RÉEL DU CLUSTER (commandes read-only) ---
{cluster_state[:3000]}
--- FIN ---
Écris une section Markdown intitulée exactement :
## État vérifié — {today}
Cette section doit contenir :
- État des services concernés (actif/inactif/dégradé)
- Versions observées si pertinent
- Toute divergence notable entre la doc et l'état réel
- Tout élément opérationnel utile observé (ex: pods crashés, export NFS manquant, config inattendue)
RÈGLES :
- Sois concis (5-15 lignes max)
- Ne reproduis pas ce qui est déjà dans le document
- Si tout est conforme, dis-le en une phrase
- Réponds UNIQUEMENT avec le contenu Markdown de la section, rien d'autre"""
result = ask_hermes(prompt)
# S'assure que la section commence bien par le bon titre
if SECTION_MARKER not in result:
result = f"{SECTION_MARKER} — {today}\n\n{result}"
return result, content
def enrich_file(fpath, today):
rel = str(fpath.relative_to(REPO_DIR))
cmds = get_context_cmds(rel) cmds = get_context_cmds(rel)
print(f" Collecte état cluster ({len(cmds)} cmd)...", flush=True) print(f" Collecte état cluster ({len(cmds)} cmd)...", flush=True)
cluster_state = collect_cluster_state(cmds) cluster_state = collect_cluster_state(cmds)
prompt = f"""Tu es l'agent de documentation du homelab Funk. print(f" Hermes génère la section...", flush=True)
MISSION : réécrire ce fichier de documentation en te basant sur l'état RÉEL du système. new_section, original_content = build_verified_section(fpath, cluster_state, today)
CONTRAINTE ABSOLUE : lecture seule — tu documentes ce que tu vois, tu ne modifies rien sur le cluster.
Fichier : {rel} if not new_section or len(new_section) < 30:
--- CONTENU ACTUEL --- return False, f"section trop courte ({len(new_section)} chars)"
{content[:3000]}
--- FIN CONTENU ---
--- ÉTAT RÉEL (commandes read-only exécutées sur le cluster) --- # Retire l'ancienne section "État vérifié" si elle existe déjà
{cluster_state[:3000]} if SECTION_MARKER in original_content:
--- FIN ÉTAT --- # Coupe au premier SECTION_MARKER et remplace par la nouvelle section
idx = original_content.index(SECTION_MARKER)
base = original_content[:idx].rstrip()
else:
base = original_content.rstrip()
RÈGLES : updated = base + "\n\n---\n\n" + new_section.strip() + "\n"
1. Réécris le document en Markdown en corrigeant les informations inexactes ou obsolètes fpath.write_text(updated, encoding="utf-8")
2. Conserve la structure et le style du document original return True, None
3. Base-toi UNIQUEMENT sur ce que tu vois dans l'état réel ci-dessus — n'invente rien
4. Si une section est correcte, garde-la telle quelle
5. Ajoute les informations manquantes que tu observes dans l'état réel
6. Ne mentionne pas cette réécriture dans le contenu du document
Réponds UNIQUEMENT avec le contenu Markdown du fichier réécrit, sans JSON, sans introduction."""
print(f" Hermes réécrit le document...", flush=True) # ── Qdrant re-ingest ──────────────────────────────────────────────────────────
result = ask_hermes(prompt)
if not result or len(result) < 80: def rag_ingest():
return None, f"réponse trop courte ({len(result)} chars)" subprocess.run(
["rsync", "-a", "--delete", str(ADMIN_DIR) + "/", str(RAG_DOCS) + "/"],
stripped = result.lstrip() check=True
if stripped.startswith("{") or stripped.startswith("["): )
return None, "réponse JSON inattendue, ignorée" r = subprocess.run(
["/usr/local/bin/rag-ingest"],
return result, None capture_output=True, text=True, timeout=300
)
return (r.stdout + r.stderr).strip()[-300:]
# ── State ───────────────────────────────────────────────────────────────────── # ── State ─────────────────────────────────────────────────────────────────────
@ -259,7 +279,7 @@ Réponds UNIQUEMENT avec le contenu Markdown du fichier réécrit, sans JSON, sa
def load_state(): def load_state():
if STATE_FILE.exists(): if STATE_FILE.exists():
return json.loads(STATE_FILE.read_text()) return json.loads(STATE_FILE.read_text())
return {"rewritten": [], "errors": []} return {"enriched": [], "errors": []}
def save_state(state): def save_state(state):
@ -275,18 +295,21 @@ def main():
parser.add_argument("--reset", action="store_true", help="Réinitialiser l'état de progression") parser.add_argument("--reset", action="store_true", help="Réinitialiser l'état de progression")
parser.add_argument("--dry-run", action="store_true", help="Afficher le plan sans modifier les fichiers") parser.add_argument("--dry-run", action="store_true", help="Afficher le plan sans modifier les fichiers")
parser.add_argument("--subdir", default=None, help="Sous-dossier admin/ (ex: ia, k8s)") parser.add_argument("--subdir", default=None, help="Sous-dossier admin/ (ex: ia, k8s)")
parser.add_argument("--no-rag", action="store_true", help="Ne pas relancer rag-ingest à la fin")
args = parser.parse_args() args = parser.parse_args()
print(f"hermes-doc-rewrite — {datetime.now().strftime('%Y-%m-%d %H:%M')}", flush=True) today = datetime.now().strftime("%Y-%m-%d")
print(f"Branche cible : {BRANCH} (pas de push)", flush=True) now = datetime.now().strftime("%Y-%m-%d_%H-%M")
print(f"hermes-doc-enrich — {now}", flush=True)
print(f"Branche : {BRANCH} (local, pas de push)", flush=True)
if not args.dry_run: if not args.dry_run:
git_sync() git_sync()
state = load_state() state = load_state()
if args.reset: if args.reset:
state = {"rewritten": [], "errors": []} state = {"enriched": [], "errors": []}
save_state(state) save_state(state)
print("État réinitialisé.\n") print("État réinitialisé.\n")
@ -299,66 +322,69 @@ def main():
if args.all: if args.all:
batch = all_md batch = all_md
else: else:
todo = [f for f in all_md if str(f) not in state["rewritten"]] todo = [f for f in all_md if str(f) not in state["enriched"]]
if not todo: if not todo:
state["rewritten"] = [] state["enriched"] = []
save_state(state) save_state(state)
todo = all_md todo = all_md
print("Rotation : tous les fichiers retraités depuis le début.") print("Rotation complète — tous les fichiers repassés.")
batch = todo[:MAX_FILES] batch = todo[:MAX_FILES]
now = datetime.now().strftime("%Y-%m-%d_%H-%M") print(f"\n{len(batch)} fichiers à enrichir\n{'─'*50}", flush=True)
print(f"\n{len(batch)} fichiers à réécrire\n{'─'*50}", flush=True)
changed = [] changed = []
errors = []
for fpath in batch: for fpath in batch:
rel = str(fpath.relative_to(REPO_DIR)) rel = str(fpath.relative_to(REPO_DIR))
print(f"\n→ {rel}", flush=True) print(f"\n→ {rel}", flush=True)
if args.dry_run: if args.dry_run:
cmds = get_context_cmds(rel) cmds = get_context_cmds(rel)
print(f" [dry-run] {len(cmds)} commandes prévues :") print(f" [dry-run] {len(cmds)} commandes :")
for c in cmds: for c in cmds:
print(f" {c}") print(f" {c}")
continue continue
new_content, err = rewrite_doc(fpath) ok, err = enrich_file(fpath, today)
if not ok:
if err:
print(f" ✗ {err}", flush=True) print(f" ✗ {err}", flush=True)
errors.append(rel) state["errors"].append(rel)
continue continue
fpath.write_text(new_content, encoding="utf-8") state["enriched"].append(str(fpath))
state["rewritten"].append(str(fpath))
changed.append(rel) changed.append(rel)
print(f" ✓ Réécrit ({len(new_content)} chars)", flush=True) print(f" ✓ Section ajoutée", flush=True)
git(["add", rel]) git(["add", rel])
if git(["diff", "--cached", "--quiet"]).returncode != 0: if git(["diff", "--cached", "--quiet"]).returncode != 0:
git(["commit", "-m", git(["commit", "-m",
f"docs(hermes): révision {Path(rel).name} — état cluster {now}\n\n" f"docs(hermes): état vérifié {Path(rel).name} — {today}\n\n"
f"Basé sur inspection read-only du cluster.\n\n" f"Section '## État vérifié' ajoutée/mise à jour.\n"
f"Inspection read-only du cluster, aucune modification de service.\n\n"
"Co-Authored-By: Hermes <hermes@funk.lab>"]) "Co-Authored-By: Hermes <hermes@funk.lab>"])
print(f" ✓ Commit local — pas de push", flush=True) print(f" ✓ Commit local", flush=True)
else: else:
print(f" — Aucun changement détecté par git", flush=True) print(f" — Aucun changement détecté (section déjà à jour)", flush=True)
if not args.dry_run: if not args.dry_run:
save_state(state) save_state(state)
# ── Qdrant re-ingest ──────────────────────────────────────────────────────
if not args.dry_run and changed and not args.no_rag:
print(f"\n⏳ Qdrant re-ingest ({len(changed)} docs mis à jour)...", flush=True)
try:
out = rag_ingest()
print(f" ✓ {out}", flush=True)
except Exception as e:
print(f" ✗ rag-ingest échoué : {e}", flush=True)
print(f"\n{'─'*50}") print(f"\n{'─'*50}")
print(f"Réécritures réussies : {len(changed)}/{len(batch)}") print(f"Enrichissements : {len(changed)}/{len(batch)}")
if errors:
print(f"Erreurs ({len(errors)}) : {', '.join(errors)}")
if changed: if changed:
print("Fichiers mis à jour :") print("Fichiers mis à jour :")
for f in changed: for f in changed:
print(f" • {f}") print(f" • {f}")
print(f"\nBranche : {BRANCH} (local, pas de push)") print(f"\nBranche : {BRANCH} (local, pas de push)")
print("Pour pusher : git -C /srv/data/lab/Funk-lab push origin hermes/daily-work") print("Pour créer la PR : hermes-auto-improve --daily-pr")
if __name__ == "__main__": if __name__ == "__main__":