From 0d09dec3dc0712d5311a75b5ebaf36d2d8d1c5a3 Mon Sep 17 00:00:00 2001 From: ALI YESILKAYA Date: Mon, 22 Jun 2026 23:15:49 +0200 Subject: [PATCH] feat(stt): enrichit les sondes (host_health, cluster_status, metrics) (#52) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - host_health : ajoute CPU%, disque /%, température CPU, uptime ; pour gpu-01 un bloc GPU complet (température, utilisation, VRAM%, puissance via rocm_scraper) en plus de llama-server. Requêtes concurrentes (asyncio.gather). - cluster_status : ajoute le nb de nœuds k8s Ready et une ligne CrashLoopBackOff (pods en boucle de crash) — complète le filtrage des pods terminés. - metrics_block (contexte grafana) : ajoute le résumé GPU (util/temp/VRAM). - Toutes les PromQL validées contre le Prometheus in-cluster. Co-authored-by: Claude Opus 4.8 --- stt/server/pyproject.toml | 2 +- stt/server/stt_server/__init__.py | 2 +- stt/server/stt_server/sources.py | 33 +++++++++--- stt/server/stt_server/tools.py | 85 +++++++++++++++++++++---------- 4 files changed, 88 insertions(+), 34 deletions(-) diff --git a/stt/server/pyproject.toml b/stt/server/pyproject.toml index 12890d0..5a4a1a6 100644 --- a/stt/server/pyproject.toml +++ b/stt/server/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "stt-server" -version = "0.9.0" +version = "0.10.0" description = "STT-server — orchestrateur AI du homelab Funk (API pour les clients STT)" requires-python = ">=3.11" diff --git a/stt/server/stt_server/__init__.py b/stt/server/stt_server/__init__.py index 900e40b..65c1d6e 100644 --- a/stt/server/stt_server/__init__.py +++ b/stt/server/stt_server/__init__.py @@ -1,3 +1,3 @@ """STT-server — orchestrateur AI in-cluster pour les clients STT.""" -__version__ = "0.9.0" +__version__ = "0.10.0" diff --git a/stt/server/stt_server/sources.py b/stt/server/stt_server/sources.py index fee616e..1dd3544 100644 --- a/stt/server/stt_server/sources.py +++ b/stt/server/stt_server/sources.py @@ -137,6 +137,10 @@ async def alerts_block(client: httpx.AsyncClient) -> str: async def cluster_block(client: httpx.AsyncClient) -> str: try: nodes = await _prom_query(client, 'up{job=~"storage-01|gpu-01-node"}') + knodes = await _prom_query( + client, 'count(kube_node_status_condition{condition="Ready",status="true"} == 1)' + ) + ready = await _prom_query(client, 'count(kube_pod_status_ready{condition="true"} == 1)') # « Non prêt » = ready false ET pod ENCORE actif (Running/Pending). Sans le filtre de # phase, les pods de CronJob terminés (Succeeded/Failed — ex. sacrifice-assign-renfort) # comptent comme « non prêts » → fausse alarme. Le join exclut les pods terminés. @@ -145,20 +149,25 @@ async def cluster_block(client: httpx.AsyncClient) -> str: 'kube_pod_status_ready{condition="false"} == 1' ' and on(namespace,pod) kube_pod_status_phase{phase=~"Running|Pending"} == 1', ) - ready = await _prom_query(client, 'count(kube_pod_status_ready{condition="true"} == 1)') + crash = await _prom_query( + client, 'kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"} == 1' + ) except httpx.HTTPError: return "Données indisponibles : Prometheus injoignable." hosts_up = sum(1 for _, v in nodes if v == 1) lines = [f"Hôtes hors-cluster joignables : {hosts_up}/{len(nodes) or 2} (storage-01, gpu-01)."] + if knodes: + lines.append(f"Nœuds k8s Ready : {int(knodes[0][1])}.") if ready: - lines.append(f"Pods prêts (cluster) : {int(ready[0][1])}.") + lines.append(f"Pods prêts : {int(ready[0][1])}.") if not_ready: - names = ", ".join( - m.get("pod", "?") for m, _ in not_ready[:6] if m.get("pod") - ) + names = ", ".join(m.get("pod", "?") for m, _ in not_ready[:6] if m.get("pod")) lines.append(f"Pods NON prêts : {names or len(not_ready)}.") else: - lines.append("Tous les pods scrutés sont prêts.") + lines.append("Tous les pods actifs sont prêts.") + if crash: + cnames = ", ".join(sorted({m.get("pod", "?") for m, _ in crash[:6]})) + lines.append(f"⚠️ CrashLoopBackOff : {cnames}.") return "\n".join(lines) @@ -167,11 +176,23 @@ async def metrics_block(client: httpx.AsyncClient) -> str: llama = await _prom_query(client, 'up{job="llama-server-gpu"}') targets_up = await _prom_query(client, 'count(up == 1)') targets_all = await _prom_query(client, 'count(up)') + gpu_util = await _prom_query(client, 'rocm_gpu_utilization_percent') + gpu_temp = await _prom_query(client, 'rocm_gpu_temperature_celsius') + gpu_vram = await _prom_query(client, '100*rocm_vram_used_bytes/rocm_vram_total_bytes') except httpx.HTTPError: return "Données indisponibles : Prometheus injoignable." lines = [] if llama: lines.append("llama-server (GPU) : " + ("en ligne" if llama[0][1] == 1 else "hors ligne") + ".") + if gpu_util or gpu_temp or gpu_vram: + g = [] + if gpu_util: + g.append(f"utilisation {round(gpu_util[0][1])}%") + if gpu_temp: + g.append(f"{round(gpu_temp[0][1])}°C") + if gpu_vram: + g.append(f"VRAM {round(gpu_vram[0][1])}%") + lines.append("GPU RX 6700XT : " + ", ".join(g) + ".") if targets_up and targets_all: lines.append(f"Cibles Prometheus UP : {int(targets_up[0][1])}/{int(targets_all[0][1])}.") return "\n".join(lines) if lines else "Aucune métrique disponible." diff --git a/stt/server/stt_server/tools.py b/stt/server/stt_server/tools.py index 0f8e67d..49c407e 100644 --- a/stt/server/stt_server/tools.py +++ b/stt/server/stt_server/tools.py @@ -12,6 +12,7 @@ Aucune action d'écriture ici : le volet « agir sur le homelab » passe par `ag from __future__ import annotations +import asyncio from dataclasses import dataclass import httpx @@ -23,6 +24,15 @@ from stt_server.config import settings _HOST_JOBS = {"gpu-01": "gpu-01-node", "storage-01": "storage-01"} +async def _q1(client: httpx.AsyncClient, expr: str) -> float | None: + """Valeur scalaire d'une requête PromQL (1re série) ou None. Ne lève jamais.""" + try: + rows = await sources._prom_query(client, expr) + except httpx.HTTPError: + return None + return rows[0][1] if rows else None + + @dataclass class ToolDeps: """Dépendances injectées aux exécuteurs (client HTTP partagé, RAG, vecteur de requête).""" @@ -66,31 +76,53 @@ async def _host_health(args: dict, deps: ToolDeps) -> str: job = _HOST_JOBS.get(host) if not job: return f"Hôte inconnu : {host!r} (attendu : {', '.join(_HOST_JOBS)})." - try: - up = await sources._prom_query(deps.client, f'up{{job="{job}"}}') - load = await sources._prom_query(deps.client, f'node_load1{{job="{job}"}}') - mem = await sources._prom_query( - deps.client, - f'100*node_memory_MemAvailable_bytes{{job="{job}"}}' - f'/node_memory_MemTotal_bytes{{job="{job}"}}', - ) - except httpx.HTTPError: - return "Données indisponibles : Prometheus injoignable." - if not up: + c = deps.client + # Sondes hôte (concurrentes) : up, CPU%, RAM libre%, disque /%, temp CPU max, uptime h. + up, cpu, mem, disk, temp, uptime = await asyncio.gather( + _q1(c, f'up{{job="{job}"}}'), + _q1(c, f'100-100*avg(rate(node_cpu_seconds_total{{job="{job}",mode="idle"}}[5m]))'), + _q1(c, f'100*node_memory_MemAvailable_bytes{{job="{job}"}}' + f'/node_memory_MemTotal_bytes{{job="{job}"}}'), + _q1(c, f'100-100*node_filesystem_avail_bytes{{job="{job}",mountpoint="/"}}' + f'/node_filesystem_size_bytes{{job="{job}",mountpoint="/"}}'), + _q1(c, f'max(node_hwmon_temp_celsius{{job="{job}"}})'), + _q1(c, f'(node_time_seconds{{job="{job}"}}-node_boot_time_seconds{{job="{job}"}})/3600'), + ) + if up is None: return f"{host} : aucune métrique (cible Prometheus absente ou hôte éteint)." - online = up[0][1] == 1 - parts = [f"{host} : {'en ligne' if online else 'HORS LIGNE'}"] - if load: - parts.append(f"charge 1 min {round(load[0][1], 2)}") - if mem: - parts.append(f"RAM libre {round(mem[0][1])}%") + parts = [f"{host} : {'en ligne' if up == 1 else 'HORS LIGNE'}"] + if cpu is not None: + parts.append(f"CPU {round(cpu)}%") + if mem is not None: + parts.append(f"RAM libre {round(mem)}%") + if disk is not None: + parts.append(f"disque / {round(disk)}% utilisé") + if temp is not None: + parts.append(f"temp CPU {round(temp)}°C") + if uptime is not None: + parts.append(f"uptime {round(uptime)} h") if host == "gpu-01": - try: - llama = await sources._prom_query(deps.client, 'up{job="llama-server-gpu"}') - if llama: - parts.append("llama-server " + ("up" if llama[0][1] == 1 else "DOWN")) - except httpx.HTTPError: - pass + # Métriques GPU (rocm_scraper → textfile collector node_exporter). + gtemp, gutil, gvram, gpow, llama = await asyncio.gather( + _q1(c, "rocm_gpu_temperature_celsius"), + _q1(c, "rocm_gpu_utilization_percent"), + _q1(c, "100*rocm_vram_used_bytes/rocm_vram_total_bytes"), + _q1(c, "rocm_gpu_power_watts"), + _q1(c, 'up{job="llama-server-gpu"}'), + ) + gpu = [] + if gutil is not None: + gpu.append(f"util {round(gutil)}%") + if gtemp is not None: + gpu.append(f"{round(gtemp)}°C") + if gvram is not None: + gpu.append(f"VRAM {round(gvram)}%") + if gpow is not None: + gpu.append(f"{round(gpow)} W") + if gpu: + parts.append("GPU (" + ", ".join(gpu) + ")") + if llama is not None: + parts.append("llama-server " + ("up" if llama == 1 else "DOWN")) return ", ".join(parts) + "." @@ -179,9 +211,10 @@ _TOOLS: dict[str, tuple[dict, object]] = { "type": "function", "function": { "name": "host_health", - "description": "État de santé LIVE d'une machine hors-cluster " - "(en ligne ?, charge CPU, RAM libre ; pour gpu-01 aussi l'état de llama-server). " - "À utiliser pour « est-ce que gpu-01 / storage-01 tourne bien ? ».", + "description": "État de santé LIVE d'une machine hors-cluster (en ligne ?, CPU, " + "RAM libre, disque, température, uptime ; pour gpu-01 aussi GPU température/" + "utilisation/VRAM/puissance et l'état de llama-server). À utiliser pour " + "« est-ce que gpu-01 / storage-01 tourne bien ? » ou une métrique machine précise.", "parameters": { "type": "object", "properties": {