apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: funk-ai-alerts
  namespace: monitoring
  labels:
    release: kube-prometheus-stack
spec:
  groups:
    - name: funk.ai
      interval: 1m
      rules:

        # --- llama-server ---
        - alert: LlamaServerGPUDown
          expr: up{job="llama-server-gpu"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "llama-server GPU (Qwen3-8B) inaccessible"
            description: "Le serveur LLM GPU sur gpu-01:1234 ne répond plus depuis 2 min. Inférence GPU unavailable."

        # --- GPU temperature (ROCm via textfile_collector) ---
        - alert: GPUTemperatureHigh
          expr: rocm_gpu_temperature_celsius > 80
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "GPU RX 6700XT température élevée : {{ $value | printf \"%.0f\" }}°C"
            description: "Température GPU à {{ $value | printf \"%.1f\" }}°C depuis 5 min. Vérifier la ventilation."

        - alert: GPUTemperatureCritical
          expr: rocm_gpu_temperature_celsius > 90
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "GPU RX 6700XT surchauffe : {{ $value | printf \"%.0f\" }}°C"
            description: "Température GPU critique à {{ $value | printf \"%.1f\" }}°C. Risque de throttling ou shutdown thermique."

        # --- VRAM ---
        - alert: GPUVRAMAlmostFull
          expr: |
            (rocm_vram_used_bytes / rocm_vram_total_bytes) * 100 > 95
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "VRAM GPU presque saturée"
            description: "VRAM utilisée à {{ $value | printf \"%.1f\" }}% ({{ $labels.gpu }}). Risque d'OOM sur les prochaines requêtes."

        # --- Requêtes en attente (backpressure) ---
        - alert: LlamaServerHighQueueGPU
          expr: llamacpp:requests_deferred{job="llama-server-gpu"} > 5
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: "File d'attente llama-server GPU élevée"
            description: "{{ $value | printf \"%.0f\" }} requêtes en attente sur le serveur GPU depuis 2 min."