diff --git a/ansible/roles/litellm/files/funk-cluster b/ansible/roles/litellm/files/funk-cluster index 1c2e89f..b77a4af 100644 --- a/ansible/roles/litellm/files/funk-cluster +++ b/ansible/roles/litellm/files/funk-cluster @@ -152,20 +152,20 @@ check_k8s() { check_k8s_pods() { local ns=$1 label=$2 - local total running - total=$($KUBECTL get pods -n "$ns" --no-headers 2>/dev/null | wc -l) - running=$($KUBECTL get pods -n "$ns" --no-headers 2>/dev/null | grep -c "^[^ ]* *[0-9]*/[0-9]* *Running" || echo 0) - - if [ "$total" -eq 0 ]; then + local pods_output total running + pods_output=$($KUBECTL get pods -n "$ns" --no-headers 2>/dev/null) + if [ -z "$pods_output" ]; then warn "$label — namespace vide ou inaccessible" return fi + total=$(echo "$pods_output" | wc -l) + running=$(echo "$pods_output" | grep -c " Running ") || running=0 if [ "$running" -eq "$total" ]; then check_ok "$label — $running/$total Running" else check_fail "$label — $running/$total Running" - $KUBECTL get pods -n "$ns" --no-headers 2>/dev/null \ + echo "$pods_output" \ | grep -v " Running " \ | awk '{printf " %-45s %s\n", $1, $3}' \ | head -6 @@ -419,11 +419,32 @@ cmd_start() { if [ $k8s_up -gt 0 ]; then local ready - ready=$($KUBECTL get nodes --no-headers 2>/dev/null | grep -c " Ready" || echo 0) + ready=$($KUBECTL get nodes --no-headers 2>/dev/null | grep -c " Ready") || ready=0 echo "" echo "[ cluster k8s — santé pods ]" if [ "$ready" -gt 0 ]; then check_ok "Nœuds — $ready/3 Ready" + + # Uncordon automatique si nœuds SchedulingDisabled (séquelle d'un funk-stop --k8s) + local uncordoned=0 + for name in "${K8S_NAMES[@]}"; do + if $KUBECTL get node "$name" --no-headers 2>/dev/null | grep -q "SchedulingDisabled"; then + $KUBECTL uncordon "$name" 2>/dev/null && ok "$name — uncordonné" || true + uncordoned=$((uncordoned + 1)) + fi + done + + # Après uncordon : attendre que les pods infra se re-schedulisent + if [ $uncordoned -gt 0 ]; then + info "Attente déploiement pods après uncordon (jusqu'à 2 min)..." + local sched_elapsed=0 + until [ "$($KUBECTL get pods -n infra --no-headers 2>/dev/null | grep -c " Running ")" -ge 1 ] \ + || [ $sched_elapsed -ge 120 ]; do + sleep 10; sched_elapsed=$((sched_elapsed + 10)) + echo -ne " → Attente pods... ${sched_elapsed}s\r" + done; echo "" + fi + check_k8s_workloads else warn "Nœuds joignables mais pas encore Ready"