mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-08 11:54:42 +02:00
feat(stt): enrichit les sondes (host_health, cluster_status, metrics) (#52)
- host_health : ajoute CPU%, disque /%, température CPU, uptime ; pour gpu-01 un bloc GPU complet (température, utilisation, VRAM%, puissance via rocm_scraper) en plus de llama-server. Requêtes concurrentes (asyncio.gather). - cluster_status : ajoute le nb de nœuds k8s Ready et une ligne CrashLoopBackOff (pods en boucle de crash) — complète le filtrage des pods terminés. - metrics_block (contexte grafana) : ajoute le résumé GPU (util/temp/VRAM). - Toutes les PromQL validées contre le Prometheus in-cluster. Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
GitHub
parent
421ec77ef9
commit
0d09dec3dc
4 changed files with 88 additions and 34 deletions
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "stt-server"
|
||||
version = "0.9.0"
|
||||
version = "0.10.0"
|
||||
description = "STT-server — orchestrateur AI du homelab Funk (API pour les clients STT)"
|
||||
requires-python = ">=3.11"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
"""STT-server — orchestrateur AI in-cluster pour les clients STT."""
|
||||
|
||||
__version__ = "0.9.0"
|
||||
__version__ = "0.10.0"
|
||||
|
|
|
|||
|
|
@ -137,6 +137,10 @@ async def alerts_block(client: httpx.AsyncClient) -> str:
|
|||
async def cluster_block(client: httpx.AsyncClient) -> str:
|
||||
try:
|
||||
nodes = await _prom_query(client, 'up{job=~"storage-01|gpu-01-node"}')
|
||||
knodes = await _prom_query(
|
||||
client, 'count(kube_node_status_condition{condition="Ready",status="true"} == 1)'
|
||||
)
|
||||
ready = await _prom_query(client, 'count(kube_pod_status_ready{condition="true"} == 1)')
|
||||
# « Non prêt » = ready false ET pod ENCORE actif (Running/Pending). Sans le filtre de
|
||||
# phase, les pods de CronJob terminés (Succeeded/Failed — ex. sacrifice-assign-renfort)
|
||||
# comptent comme « non prêts » → fausse alarme. Le join exclut les pods terminés.
|
||||
|
|
@ -145,20 +149,25 @@ async def cluster_block(client: httpx.AsyncClient) -> str:
|
|||
'kube_pod_status_ready{condition="false"} == 1'
|
||||
' and on(namespace,pod) kube_pod_status_phase{phase=~"Running|Pending"} == 1',
|
||||
)
|
||||
ready = await _prom_query(client, 'count(kube_pod_status_ready{condition="true"} == 1)')
|
||||
crash = await _prom_query(
|
||||
client, 'kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"} == 1'
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
return "Données indisponibles : Prometheus injoignable."
|
||||
hosts_up = sum(1 for _, v in nodes if v == 1)
|
||||
lines = [f"Hôtes hors-cluster joignables : {hosts_up}/{len(nodes) or 2} (storage-01, gpu-01)."]
|
||||
if knodes:
|
||||
lines.append(f"Nœuds k8s Ready : {int(knodes[0][1])}.")
|
||||
if ready:
|
||||
lines.append(f"Pods prêts (cluster) : {int(ready[0][1])}.")
|
||||
lines.append(f"Pods prêts : {int(ready[0][1])}.")
|
||||
if not_ready:
|
||||
names = ", ".join(
|
||||
m.get("pod", "?") for m, _ in not_ready[:6] if m.get("pod")
|
||||
)
|
||||
names = ", ".join(m.get("pod", "?") for m, _ in not_ready[:6] if m.get("pod"))
|
||||
lines.append(f"Pods NON prêts : {names or len(not_ready)}.")
|
||||
else:
|
||||
lines.append("Tous les pods scrutés sont prêts.")
|
||||
lines.append("Tous les pods actifs sont prêts.")
|
||||
if crash:
|
||||
cnames = ", ".join(sorted({m.get("pod", "?") for m, _ in crash[:6]}))
|
||||
lines.append(f"⚠️ CrashLoopBackOff : {cnames}.")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
|
|
@ -167,11 +176,23 @@ async def metrics_block(client: httpx.AsyncClient) -> str:
|
|||
llama = await _prom_query(client, 'up{job="llama-server-gpu"}')
|
||||
targets_up = await _prom_query(client, 'count(up == 1)')
|
||||
targets_all = await _prom_query(client, 'count(up)')
|
||||
gpu_util = await _prom_query(client, 'rocm_gpu_utilization_percent')
|
||||
gpu_temp = await _prom_query(client, 'rocm_gpu_temperature_celsius')
|
||||
gpu_vram = await _prom_query(client, '100*rocm_vram_used_bytes/rocm_vram_total_bytes')
|
||||
except httpx.HTTPError:
|
||||
return "Données indisponibles : Prometheus injoignable."
|
||||
lines = []
|
||||
if llama:
|
||||
lines.append("llama-server (GPU) : " + ("en ligne" if llama[0][1] == 1 else "hors ligne") + ".")
|
||||
if gpu_util or gpu_temp or gpu_vram:
|
||||
g = []
|
||||
if gpu_util:
|
||||
g.append(f"utilisation {round(gpu_util[0][1])}%")
|
||||
if gpu_temp:
|
||||
g.append(f"{round(gpu_temp[0][1])}°C")
|
||||
if gpu_vram:
|
||||
g.append(f"VRAM {round(gpu_vram[0][1])}%")
|
||||
lines.append("GPU RX 6700XT : " + ", ".join(g) + ".")
|
||||
if targets_up and targets_all:
|
||||
lines.append(f"Cibles Prometheus UP : {int(targets_up[0][1])}/{int(targets_all[0][1])}.")
|
||||
return "\n".join(lines) if lines else "Aucune métrique disponible."
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ Aucune action d'écriture ici : le volet « agir sur le homelab » passe par `ag
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
|
|
@ -23,6 +24,15 @@ from stt_server.config import settings
|
|||
_HOST_JOBS = {"gpu-01": "gpu-01-node", "storage-01": "storage-01"}
|
||||
|
||||
|
||||
async def _q1(client: httpx.AsyncClient, expr: str) -> float | None:
|
||||
"""Valeur scalaire d'une requête PromQL (1re série) ou None. Ne lève jamais."""
|
||||
try:
|
||||
rows = await sources._prom_query(client, expr)
|
||||
except httpx.HTTPError:
|
||||
return None
|
||||
return rows[0][1] if rows else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolDeps:
|
||||
"""Dépendances injectées aux exécuteurs (client HTTP partagé, RAG, vecteur de requête)."""
|
||||
|
|
@ -66,31 +76,53 @@ async def _host_health(args: dict, deps: ToolDeps) -> str:
|
|||
job = _HOST_JOBS.get(host)
|
||||
if not job:
|
||||
return f"Hôte inconnu : {host!r} (attendu : {', '.join(_HOST_JOBS)})."
|
||||
try:
|
||||
up = await sources._prom_query(deps.client, f'up{{job="{job}"}}')
|
||||
load = await sources._prom_query(deps.client, f'node_load1{{job="{job}"}}')
|
||||
mem = await sources._prom_query(
|
||||
deps.client,
|
||||
f'100*node_memory_MemAvailable_bytes{{job="{job}"}}'
|
||||
f'/node_memory_MemTotal_bytes{{job="{job}"}}',
|
||||
c = deps.client
|
||||
# Sondes hôte (concurrentes) : up, CPU%, RAM libre%, disque /%, temp CPU max, uptime h.
|
||||
up, cpu, mem, disk, temp, uptime = await asyncio.gather(
|
||||
_q1(c, f'up{{job="{job}"}}'),
|
||||
_q1(c, f'100-100*avg(rate(node_cpu_seconds_total{{job="{job}",mode="idle"}}[5m]))'),
|
||||
_q1(c, f'100*node_memory_MemAvailable_bytes{{job="{job}"}}'
|
||||
f'/node_memory_MemTotal_bytes{{job="{job}"}}'),
|
||||
_q1(c, f'100-100*node_filesystem_avail_bytes{{job="{job}",mountpoint="/"}}'
|
||||
f'/node_filesystem_size_bytes{{job="{job}",mountpoint="/"}}'),
|
||||
_q1(c, f'max(node_hwmon_temp_celsius{{job="{job}"}})'),
|
||||
_q1(c, f'(node_time_seconds{{job="{job}"}}-node_boot_time_seconds{{job="{job}"}})/3600'),
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
return "Données indisponibles : Prometheus injoignable."
|
||||
if not up:
|
||||
if up is None:
|
||||
return f"{host} : aucune métrique (cible Prometheus absente ou hôte éteint)."
|
||||
online = up[0][1] == 1
|
||||
parts = [f"{host} : {'en ligne' if online else 'HORS LIGNE'}"]
|
||||
if load:
|
||||
parts.append(f"charge 1 min {round(load[0][1], 2)}")
|
||||
if mem:
|
||||
parts.append(f"RAM libre {round(mem[0][1])}%")
|
||||
parts = [f"{host} : {'en ligne' if up == 1 else 'HORS LIGNE'}"]
|
||||
if cpu is not None:
|
||||
parts.append(f"CPU {round(cpu)}%")
|
||||
if mem is not None:
|
||||
parts.append(f"RAM libre {round(mem)}%")
|
||||
if disk is not None:
|
||||
parts.append(f"disque / {round(disk)}% utilisé")
|
||||
if temp is not None:
|
||||
parts.append(f"temp CPU {round(temp)}°C")
|
||||
if uptime is not None:
|
||||
parts.append(f"uptime {round(uptime)} h")
|
||||
if host == "gpu-01":
|
||||
try:
|
||||
llama = await sources._prom_query(deps.client, 'up{job="llama-server-gpu"}')
|
||||
if llama:
|
||||
parts.append("llama-server " + ("up" if llama[0][1] == 1 else "DOWN"))
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
# Métriques GPU (rocm_scraper → textfile collector node_exporter).
|
||||
gtemp, gutil, gvram, gpow, llama = await asyncio.gather(
|
||||
_q1(c, "rocm_gpu_temperature_celsius"),
|
||||
_q1(c, "rocm_gpu_utilization_percent"),
|
||||
_q1(c, "100*rocm_vram_used_bytes/rocm_vram_total_bytes"),
|
||||
_q1(c, "rocm_gpu_power_watts"),
|
||||
_q1(c, 'up{job="llama-server-gpu"}'),
|
||||
)
|
||||
gpu = []
|
||||
if gutil is not None:
|
||||
gpu.append(f"util {round(gutil)}%")
|
||||
if gtemp is not None:
|
||||
gpu.append(f"{round(gtemp)}°C")
|
||||
if gvram is not None:
|
||||
gpu.append(f"VRAM {round(gvram)}%")
|
||||
if gpow is not None:
|
||||
gpu.append(f"{round(gpow)} W")
|
||||
if gpu:
|
||||
parts.append("GPU (" + ", ".join(gpu) + ")")
|
||||
if llama is not None:
|
||||
parts.append("llama-server " + ("up" if llama == 1 else "DOWN"))
|
||||
return ", ".join(parts) + "."
|
||||
|
||||
|
||||
|
|
@ -179,9 +211,10 @@ _TOOLS: dict[str, tuple[dict, object]] = {
|
|||
"type": "function",
|
||||
"function": {
|
||||
"name": "host_health",
|
||||
"description": "État de santé LIVE d'une machine hors-cluster "
|
||||
"(en ligne ?, charge CPU, RAM libre ; pour gpu-01 aussi l'état de llama-server). "
|
||||
"À utiliser pour « est-ce que gpu-01 / storage-01 tourne bien ? ».",
|
||||
"description": "État de santé LIVE d'une machine hors-cluster (en ligne ?, CPU, "
|
||||
"RAM libre, disque, température, uptime ; pour gpu-01 aussi GPU température/"
|
||||
"utilisation/VRAM/puissance et l'état de llama-server). À utiliser pour "
|
||||
"« est-ce que gpu-01 / storage-01 tourne bien ? » ou une métrique machine précise.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue