From a1826fe3398210f245251b5e32c66f2acc54d351 Mon Sep 17 00:00:00 2001 From: alkatrazz Date: Wed, 13 May 2026 02:11:21 +0200 Subject: [PATCH] =?UTF-8?q?docs:=20gestion=20cluster=20+=20script=20funk-c?= =?UTF-8?q?luster=20mis=20=C3=A0=20jour?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - admin/cluster.md : nouveau guide démarrage/arrêt (funk-start/stop/--k8s) - admin/README.md : lien vers cluster.md - admin/systeme.md : services llama-server mis à jour (GPU + 2 CPU) - ansible/roles/litellm/files/funk-cluster : script versionné (talosctl fix, storage-01 shutdown, --k8s optionnel, stop timeout) Co-Authored-By: Claude Sonnet 4.6 --- admin/README.md | 2 +- admin/cluster.md | 103 ++++++ admin/systeme.md | 4 +- ansible/roles/litellm/files/funk-cluster | 436 +++++++++++++++++++++++ 4 files changed, 543 insertions(+), 2 deletions(-) create mode 100644 admin/cluster.md create mode 100644 ansible/roles/litellm/files/funk-cluster diff --git a/admin/README.md b/admin/README.md index 44b4849..0d4e14a 100644 --- a/admin/README.md +++ b/admin/README.md @@ -26,4 +26,4 @@ | [talos.md](talos.md) | Talos Linux — administration quotidienne du cluster k8s | | [incidents.md](incidents.md) | Incidents passés — causes, résolutions, fixes permanents | | **Gestion cluster** | | -| `funk-start` / `funk-stop` / `funk-status` | Aliases locaux → `ssh storage-01 "sudo /usr/local/bin/funk-cluster [start\|stop\|status]"` | +| [cluster.md](cluster.md) | Démarrage / arrêt / status — `funk-start`, `funk-stop`, `funk-stop --k8s` | diff --git a/admin/cluster.md b/admin/cluster.md new file mode 100644 index 0000000..baf8b20 --- /dev/null +++ b/admin/cluster.md @@ -0,0 +1,103 @@ +# Gestion du cluster Funk + +Script central : `/usr/local/bin/funk-cluster` sur storage-01. +Aliases locaux configurés dans `~/.bashrc` du poste perso. + +--- + +## Commandes + +```bash +# Depuis le poste perso +funk-status # État complet (storage-01 + gpu-01 + k8s) +funk-start # Démarre gpu-01 (WOL) + tous les services +funk-start --k8s # Idem + démarre le cluster k8s (compute-01/02/03) +funk-stop # Arrête services + gpu-01 + storage-01 +funk-stop --k8s # Idem + éteint le cluster k8s avant +``` + +> `funk-stop` éteint storage-01 en dernier — la connexion SSH se coupe, c'est normal. + +--- + +## Ce que fait chaque commande + +### `funk-status` + +Vérifie dans l'ordre : +- Services storage-01 : postgresql-16, qdrant, litellm, hermes-agent +- Santé : PostgreSQL (connexion réelle), Qdrant (collections), LiteLLM (modèles), Hermes, NFS export +- gpu-01 : ping, NFS /mnt/models, llama-server GPU (:1234), embeddings, llama-server CPU (:1236/:1237) +- Cluster k8s : ping compute-01/02/03, kubectl get nodes Ready + +### `funk-start [--k8s]` + +1. Démarre postgresql-16, qdrant, litellm, hermes-agent sur storage-01 +2. Envoie un paquet Wake-on-LAN à gpu-01 si éteint, attend le ping + SSH +3. Attend le montage NFS `/mnt/models` sur gpu-01 +4. Attend le chargement du modèle GPU (jusqu'à 4 min) +5. Attend les modèles CPU (jusqu'à 2 min) +6. `--k8s` : envoie WOL aux 3 nœuds, attend ping, attend nœuds Ready (3 min max) + +### `funk-stop [--k8s]` + +1. Arrête hermes-agent et litellm sur storage-01 (timeout 10s + SIGKILL si besoin) +2. `--k8s` : `talosctl shutdown` sur compute-01/02/03, attend extinction +3. Éteint gpu-01 (`shutdown -h now` via SSH) +4. Arrête qdrant et postgresql-16 sur storage-01 +5. Éteint storage-01 (`shutdown -h now`) — connexion SSH coupée + +--- + +## Machines et MACs + +| Machine | IP | MAC | Rôle | +|---|---|---|---| +| storage-01 | 192.168.10.1 / 192.168.1.200 | — | Passerelle + services IA | +| gpu-01 | 192.168.10.20 | `a8:a1:59:72:a6:11` | LLM GPU + CPU | +| compute-01 | 192.168.10.11 | `6c:4b:90:82:8e:47` | k8s control-plane | +| compute-02 | 192.168.10.12 | `6c:4b:90:cf:7f:c5` | k8s worker | +| compute-03 | 192.168.10.13 | `6c:4b:90:b6:49:20` | k8s worker | + +--- + +## Aliases locaux (`~/.bashrc` poste perso) + +```bash +alias funk-status='ssh storage-01 "sudo /usr/local/bin/funk-cluster status"' +alias funk-start='ssh storage-01 "sudo /usr/local/bin/funk-cluster start"' +alias funk-stop='ssh storage-01 "sudo /usr/local/bin/funk-cluster stop"' +``` + +Pour passer `--k8s` : +```bash +ssh storage-01 "sudo /usr/local/bin/funk-cluster stop --k8s" +ssh storage-01 "sudo /usr/local/bin/funk-cluster start --k8s" +``` + +> **Note** : les aliases `funk-start`/`funk-stop` ne passent pas les arguments. +> Utiliser la commande SSH directe pour `--k8s`. + +--- + +## Déploiement du script + +Le script est versionné dans `ansible/roles/litellm/files/funk-cluster`. +Pour le redéployer sur storage-01 : + +```bash +scp ansible/roles/litellm/files/funk-cluster s01:/tmp/ +ssh s01 "sudo cp /tmp/funk-cluster /usr/local/bin/funk-cluster && sudo chmod +x /usr/local/bin/funk-cluster" +``` + +--- + +## Points d'attention + +| Sujet | Détail | +|---|---| +| Ordre d'arrêt | Hermes/LiteLLM → k8s (optionnel) → gpu-01 → Qdrant/PG → storage-01 | +| Ordre de démarrage | storage-01 (toujours déjà up) → gpu-01 → k8s (optionnel) | +| LiteLLM bloquant | `systemctl stop --timeout=10` + SIGKILL automatique si timeout | +| talosctl path | `/home/ansible/.config/talos/config` requis explicitement (sudo change HOME) | +| `--k8s` non passé par alias | Utiliser `ssh s01 "sudo funk-cluster stop --k8s"` directement | diff --git a/admin/systeme.md b/admin/systeme.md index 1cc5afa..4abc346 100644 --- a/admin/systeme.md +++ b/admin/systeme.md @@ -35,7 +35,9 @@ sudo systemctl list-units --type=service --state=running | storage-01 | `hermes-agent` | Agent IA (gateway + cron) | | gpu-01 | `chronyd` | NTP | | gpu-01 | `sshd` | SSH | -| gpu-01 | `llama-server` | Inférence LLM — Qwen2.5-7B via ROCm 7.x | +| gpu-01 | `llama-server` | Inférence GPU — Qwen3-8B via ROCm 7.x (port 1234) | +| gpu-01 | `llama-server-system` | Inférence CPU — Qwen3-1.7B (port 1236, 14 threads) | +| gpu-01 | `llama-server-monitor` | Inférence CPU — Qwen3-1.7B (port 1237, 14 threads) | --- diff --git a/ansible/roles/litellm/files/funk-cluster b/ansible/roles/litellm/files/funk-cluster new file mode 100644 index 0000000..82d870b --- /dev/null +++ b/ansible/roles/litellm/files/funk-cluster @@ -0,0 +1,436 @@ +#!/bin/bash +# Gestion du homelab Funk (storage-01 + gpu-01 + cluster k8s optionnel) +# Usage: funk-cluster [start|stop|status] [--k8s] + +GPU01_IP="192.168.10.20" +GPU01_MAC="a8:a1:59:72:a6:11" +GPU01_SSH="ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no gpu-01" + +K8S_IPS=(192.168.10.11 192.168.10.12 192.168.10.13) +K8S_MACS=("6c:4b:90:82:8e:47" "6c:4b:90:cf:7f:c5" "6c:4b:90:b6:49:20") +K8S_NAMES=(compute-01 compute-02 compute-03) +TALOSCTL="sudo -u ansible talosctl --talosconfig /home/ansible/.config/talos/config" +KUBECTL="sudo -u ansible kubectl" + +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +BOLD='\033[1m' +NC='\033[0m' + +ok() { echo -e " ${GREEN}✓${NC} $*"; } +fail() { echo -e " ${RED}✗${NC} $*"; } +warn() { echo -e " ${YELLOW}!${NC} $*"; } +info() { echo -e " → $*"; } + +CHECKS_OK=0 +CHECKS_FAIL=0 + +check_ok() { ok "$*"; CHECKS_OK=$((CHECKS_OK + 1)); } +check_fail() { fail "$*"; CHECKS_FAIL=$((CHECKS_FAIL + 1)); } + +wol() { + local mac="${1//:}" + python3 -c " +import socket +mac = '$mac' +payload = bytes.fromhex('ff' * 6 + mac * 16) +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) +s.sendto(payload, ('192.168.10.255', 9)) +s.close() +" +} + +check_service() { + local svc=$1 + if systemctl is-active --quiet "$svc" 2>/dev/null; then + check_ok "$svc" + else + check_fail "$svc (inactif)" + fi +} + +check_postgresql() { + if sudo -u postgres psql -c "SELECT 1" hermes &>/dev/null 2>&1; then + check_ok "PostgreSQL — base hermes accessible" + else + check_fail "PostgreSQL — base hermes inaccessible" + fi +} + +check_qdrant() { + local result + result=$(curl -sf --max-time 5 "http://127.0.0.1:6333/collections" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('result',{}).get('collections',[])))" 2>/dev/null) + if [ -n "$result" ]; then + check_ok "Qdrant — $result collection(s)" + else + check_fail "Qdrant non joignable (port 6333)" + fi +} + +check_litellm() { + local models + models=$(curl -sf --max-time 5 \ + -H "Authorization: Bearer lm-studio" \ + "http://127.0.0.1:4000/v1/models" \ + | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('data',[])))" 2>/dev/null) + if [ -n "$models" ]; then + check_ok "LiteLLM — $models modèle(s) enregistré(s)" + else + check_fail "LiteLLM non joignable (port 4000)" + fi +} + +check_hermes() { + if curl -sf --max-time 5 "http://127.0.0.1:8080/health" &>/dev/null; then + check_ok "Hermes HTTP (port 8080)" + elif systemctl is-active --quiet hermes-agent 2>/dev/null; then + check_ok "Hermes service actif (pas d'endpoint HTTP)" + else + check_fail "Hermes inactif" + fi +} + +check_nfs_export() { + if showmount -e localhost 2>/dev/null | grep -q "/srv/data/models"; then + check_ok "NFS export /srv/data/models OK" + else + check_fail "NFS export /srv/data/models manquant" + fi +} + +check_llama_server() { + local port=${1:-1234} label=${2:-"GPU"} + local model + model=$(curl -sf --max-time 5 "http://${GPU01_IP}:${port}/v1/models" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['data'][0]['id'])" 2>/dev/null) + if [ -n "$model" ]; then + check_ok "llama-server $label (:$port) — $model" + else + check_fail "llama-server $label (:$port) — non joignable" + fi +} + +check_embeddings() { + local dim + dim=$(curl -sf --max-time 15 -X POST "http://${GPU01_IP}:1234/v1/embeddings" \ + -H "Content-Type: application/json" \ + -d '{"model":"qwen3-8b","input":["test"]}' \ + | python3 -c "import sys,json; print(len(json.load(sys.stdin)['data'][0]['embedding']))" 2>/dev/null) + if [ -n "$dim" ]; then + check_ok "Embeddings OK — dimension : $dim" + else + check_fail "Embeddings non disponibles" + fi +} + +check_k8s() { + echo "" + echo "[ cluster k8s — compute-01/02/03 ]" + local up=0 + for i in 0 1 2; do + local ip="${K8S_IPS[$i]}" name="${K8S_NAMES[$i]}" + if ping -c1 -W2 "$ip" &>/dev/null; then + check_ok "$name ($ip) — ping OK" + up=$((up + 1)) + else + warn "$name ($ip) — éteint" + fi + done + if [ $up -gt 0 ]; then + local ready + ready=$($KUBECTL get nodes --no-headers 2>/dev/null | grep -c " Ready" || echo 0) + if [ "$ready" -gt 0 ]; then + check_ok "kubectl — $ready nœud(s) Ready" + else + warn "kubectl — nœuds non Ready (encore en démarrage ?)" + fi + fi +} + +wait_ping() { + local host=$1 timeout=${2:-120} elapsed=0 + while ! ping -c1 -W1 "$host" &>/dev/null; do + sleep 3; elapsed=$((elapsed + 3)) + [ $elapsed -ge $timeout ] && return 1 + echo -ne " → Attente ping $host... ${elapsed}s\r" + done + echo ""; return 0 +} + +wait_llama() { + local port=${1:-1234} timeout=${2:-240} elapsed=0 + while ! curl -sf --max-time 3 "http://${GPU01_IP}:${port}/v1/models" &>/dev/null; do + sleep 5; elapsed=$((elapsed + 5)) + [ $elapsed -ge $timeout ] && return 1 + echo -ne " → Attente llama-server :${port}... ${elapsed}s\r" + done + echo ""; return 0 +} + +stop_service() { + local svc=$1 + systemctl stop --timeout=10 "$svc" 2>/dev/null + systemctl kill --signal=SIGKILL "$svc" 2>/dev/null + ok "$svc arrêté" +} + +print_score() { + local total=$((CHECKS_OK + CHECKS_FAIL)) + echo "" + if [ $CHECKS_FAIL -eq 0 ]; then + echo -e " ${GREEN}${BOLD}[$CHECKS_OK/$total] Tous les checks OK${NC}" + else + echo -e " ${YELLOW}${BOLD}[$CHECKS_OK/$total] — $CHECKS_FAIL check(s) en échec${NC}" + fi +} + +# ============================================================ +# STATUS +# ============================================================ +cmd_status() { + CHECKS_OK=0; CHECKS_FAIL=0 + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " FUNK — État du homelab" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + echo "" + echo "[ storage-01 — services ]" + check_service postgresql-16 + check_service qdrant + check_service litellm + check_service hermes-agent + + echo "" + echo "[ storage-01 — santé ]" + check_postgresql + check_qdrant + check_litellm + check_hermes + check_nfs_export + + echo "" + echo "[ gpu-01 — 192.168.10.20 ]" + if ping -c1 -W2 "$GPU01_IP" &>/dev/null; then + check_ok "ping OK" + if $GPU01_SSH "mountpoint -q /mnt/models" 2>/dev/null; then + check_ok "NFS /mnt/models monté" + else + check_fail "NFS /mnt/models non monté" + fi + check_llama_server 1234 "GPU (Qwen3-8B)" + check_embeddings + check_llama_server 1236 "CPU1 (Qwen3-1.7B system)" + check_llama_server 1237 "CPU2 (Qwen3-1.7B monitor)" + else + check_fail "gpu-01 non joignable (éteint ?)" + fi + + check_k8s + print_score + echo "" +} + +# ============================================================ +# START +# ============================================================ +cmd_start() { + local with_k8s=$1 + CHECKS_OK=0; CHECKS_FAIL=0 + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " FUNK — Démarrage${with_k8s:+ (+ cluster k8s)}" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + echo "" + echo "[ storage-01 — démarrage des services ]" + for svc in postgresql-16 qdrant litellm hermes-agent; do + systemctl start "$svc" 2>/dev/null; sleep 1 + systemctl is-active --quiet "$svc" \ + && ok "$svc démarré" \ + || warn "$svc — problème (journalctl -u $svc -n 20)" + done + + echo "" + echo "[ storage-01 — vérifications ]" + sleep 2 + check_postgresql; check_qdrant; check_litellm; check_hermes; check_nfs_export + + echo "" + echo "[ gpu-01 — démarrage ]" + if ping -c1 -W2 "$GPU01_IP" &>/dev/null; then + check_ok "gpu-01 déjà allumé" + else + wol "$GPU01_MAC" + info "Paquet WOL envoyé → gpu-01" + if ! wait_ping "$GPU01_IP" 120; then + check_fail "gpu-01 n'a pas répondu en 120s" + print_score; return 1 + fi + check_ok "gpu-01 joignable" + local ssh_tries=0 + while ! $GPU01_SSH "true" 2>/dev/null; do + sleep 3; ssh_tries=$((ssh_tries + 1)) + [ $ssh_tries -ge 10 ] && break + echo -ne " → Attente SSH... $((ssh_tries * 3))s\r" + done; echo "" + fi + + info "Vérification NFS sur gpu-01..." + local nfs_tries=0 + while ! $GPU01_SSH "mountpoint -q /mnt/models" 2>/dev/null; do + sleep 3; nfs_tries=$((nfs_tries + 1)) + [ $nfs_tries -ge 20 ] && break + echo -ne " → Attente NFS... $((nfs_tries * 3))s\r" + done; echo "" + $GPU01_SSH "mountpoint -q /mnt/models" 2>/dev/null \ + && check_ok "NFS /mnt/models monté" \ + || check_fail "NFS /mnt/models non monté" + + echo "" + echo "[ gpu-01 — llama-server GPU (port 1234) ]" + info "Attente chargement modèle GPU (jusqu'à 4 min)..." + if wait_llama 1234 240; then + check_llama_server 1234 "GPU (Qwen3-8B)"; check_embeddings + else + check_fail "llama-server GPU n'a pas répondu en 4 min" + fi + + echo "" + echo "[ gpu-01 — llama-server CPU (ports 1236/1237) ]" + info "Attente chargement modèles CPU (jusqu'à 2 min)..." + wait_llama 1236 120; check_llama_server 1236 "CPU1 (Qwen3-1.7B system)" + wait_llama 1237 60; check_llama_server 1237 "CPU2 (Qwen3-1.7B monitor)" + + if [ -n "$with_k8s" ]; then + echo "" + echo "[ cluster k8s — démarrage compute-01/02/03 ]" + for i in 0 1 2; do + local ip="${K8S_IPS[$i]}" name="${K8S_NAMES[$i]}" mac="${K8S_MACS[$i]}" + if ping -c1 -W2 "$ip" &>/dev/null; then + check_ok "$name déjà allumé" + else + wol "$mac" + ok "$name — WOL envoyé" + fi + done + info "Attente ping des nœuds..." + for ip in "${K8S_IPS[@]}"; do + wait_ping "$ip" 120 && ok "$ip joignable" || warn "$ip n'a pas répondu" + done + info "Attente nœuds Ready (jusqu'à 3 min)..." + local k8s_elapsed=0 + until [ $($KUBECTL get nodes --no-headers 2>/dev/null | grep -c " Ready" || echo 0) -ge 3 ] \ + || [ $k8s_elapsed -ge 180 ]; do + sleep 10; k8s_elapsed=$((k8s_elapsed + 10)) + echo -ne " → Attente nœuds Ready... ${k8s_elapsed}s\r" + done; echo "" + local ready=$($KUBECTL get nodes --no-headers 2>/dev/null | grep -c " Ready" || echo 0) + [ "$ready" -ge 1 ] \ + && check_ok "kubectl — $ready nœud(s) Ready" \ + || warn "Nœuds pas encore Ready (vérifier avec kubectl get nodes)" + fi + + print_score + [ $CHECKS_FAIL -eq 0 ] \ + && echo -e " ${GREEN}Homelab Funk opérationnel — bonne session !${NC}" \ + || echo -e " ${YELLOW}Homelab démarré avec des avertissements${NC}" + echo "" +} + +# ============================================================ +# STOP +# ============================================================ +cmd_stop() { + local with_k8s=$1 + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " FUNK — Arrêt${with_k8s:+ (+ cluster k8s)}" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + echo "" + echo "[ storage-01 — arrêt des services applicatifs ]" + for svc in hermes-agent litellm; do stop_service "$svc"; done + + if [ -n "$with_k8s" ]; then + echo "" + echo "[ cluster k8s — arrêt compute-01/02/03 ]" + local alive=() + for ip in "${K8S_IPS[@]}"; do + ping -c1 -W2 "$ip" &>/dev/null && alive+=("$ip") + done + if [ ${#alive[@]} -gt 0 ]; then + local nodes=$(IFS=,; echo "${alive[*]}") + info "talosctl shutdown → $nodes" + if $TALOSCTL shutdown --nodes "$nodes" 2>/dev/null; then + ok "Shutdown envoyé aux nœuds" + else + warn "Erreur talosctl shutdown — extinction manuelle requise" + return + fi + local tries=0 + while [ $tries -lt 20 ]; do + local still_up=0 + for ip in "${alive[@]}"; do ping -c1 -W1 "$ip" &>/dev/null && still_up=$((still_up + 1)); done + [ $still_up -eq 0 ] && break + sleep 3; tries=$((tries + 1)) + echo -ne " → Extinction k8s... $((tries * 3))s\r" + done; echo "" + ok "Cluster k8s éteint" + else + warn "Nœuds k8s déjà éteints" + fi + fi + + echo "" + echo "[ gpu-01 — arrêt ]" + if ping -c1 -W2 "$GPU01_IP" &>/dev/null; then + info "Arrêt de gpu-01..." + $GPU01_SSH "sudo shutdown -h now" 2>/dev/null + local tries=0 + while ping -c1 -W1 "$GPU01_IP" &>/dev/null; do + sleep 2; tries=$((tries + 1)) + [ $tries -ge 30 ] && break + echo -ne " → Extinction gpu-01... $((tries * 2))s\r" + done; echo "" + ok "gpu-01 éteint" + else + warn "gpu-01 déjà éteint" + fi + + echo "" + echo "[ storage-01 — arrêt des services données ]" + for svc in qdrant postgresql-16; do stop_service "$svc"; done + + echo "" + ok "Homelab Funk arrêté proprement" + echo "" + echo "[ storage-01 — extinction dans 3s ]" + info "La connexion SSH va être coupée..." + sleep 3 + sudo shutdown -h now +} + +# ============================================================ +CMD="${1:-status}" +WITH_K8S="" +for arg in "$@"; do [ "$arg" = "--k8s" ] && WITH_K8S=1; done + +case "$CMD" in + start) cmd_start "$WITH_K8S" ;; + stop) cmd_stop "$WITH_K8S" ;; + status) cmd_status ;; + *) + echo "Usage: funk-cluster [start|stop|status] [--k8s]" + echo "" + echo " start Démarre gpu-01 (WOL) + vérifie tous les services" + echo " start --k8s Idem + démarre le cluster k8s (compute-01/02/03)" + echo " stop Arrêt propre gpu-01 + services storage-01" + echo " stop --k8s Idem + éteint le cluster k8s (talosctl shutdown)" + echo " status État complet (storage-01 + gpu-01 + k8s)" + exit 1 + ;; +esac