mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-10 23:14:42 +02:00
feat(monitoring): surveillance réseau active via blackbox_exporter
Sondes actives ICMP/HTTP/DNS qui complètent le monitoring passif
(node_exporter + up{}) : joignabilité hôtes/passerelle, accès internet,
santé DNS *.lab.local + forwarding, latence LAN, services de bout en
bout via Traefik.
- blackbox_exporter (chart 11.13.0, ArgoCD, ns monitoring) — ICMP via
NET_RAW, DNS du pod = dnsmasq (chemin réel lab → Traefik)
- 3 jobs blackbox-icmp/-http/-dns (additionalScrapeConfigs)
- alerts-network.yaml : internet, DNS, passerelle, hôtes, services,
latence, exporter (→ pipeline AlertManager existant)
- Asa : outil network_status + contexte présélectionnable reseau
- doc admin/infra/reseau.md + CLAUDE.md + progress
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Rc3dLKfqEPHexvFCCx5ZEF
This commit is contained in:
Claude
parent
ebb2eec184
commit
649c6fde8e
11 changed files with 527 additions and 5 deletions
102
k8s/infra/monitoring/alerts/alerts-network.yaml
Normal file
102
k8s/infra/monitoring/alerts/alerts-network.yaml
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: funk-network-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: funk.network
|
||||
interval: 1m
|
||||
rules:
|
||||
|
||||
# --- Joignabilité internet (NAT storage-01 / lien WAN) ---
|
||||
- alert: InternetDown
|
||||
expr: sum(probe_success{job="blackbox-icmp",scope="internet"}) == 0
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Plus d'accès internet depuis le cluster"
|
||||
description: "Aucune ancre internet (Cloudflare/Quad9) ne répond au ping depuis 3 min. Lien WAN ou NAT storage-01 probablement en cause."
|
||||
|
||||
# --- DNS : résolution *.lab.local (cœur de dnsmasq) ---
|
||||
- alert: DnsLabResolutionFailed
|
||||
expr: probe_success{job="blackbox-dns",module="dns_lab"} == 0
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "dnsmasq ne résout plus *.lab.local"
|
||||
description: "La sonde DNS sur 192.168.10.1:53 (grafana.lab.local) échoue depuis 3 min. Tout l'accès aux services par nom est cassé."
|
||||
|
||||
# --- DNS : forwarding vers l'upstream ---
|
||||
- alert: DnsForwardingFailed
|
||||
expr: probe_success{job="blackbox-dns",module="dns_external"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Forwarding DNS externe en échec"
|
||||
description: "dnsmasq ne résout plus les noms externes (one.one.one.one) depuis 5 min. Résolution interne possiblement OK — vérifier l'upstream."
|
||||
|
||||
# --- Passerelle (storage-01) injoignable en ICMP ---
|
||||
- alert: GatewayUnreachable
|
||||
expr: probe_success{job="blackbox-icmp",instance="192.168.10.1"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Passerelle 192.168.10.1 (storage-01) injoignable"
|
||||
description: "storage-01 ne répond plus au ping depuis le cluster (2 min). NFS, DNS, NAT, PostgreSQL probablement impactés."
|
||||
|
||||
# --- Hôte du LAN cluster injoignable (hors passerelle) ---
|
||||
- alert: ClusterHostUnreachable
|
||||
expr: probe_success{job="blackbox-icmp",scope="lan",instance!="192.168.10.1"} == 0
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Hôte {{ $labels.instance }} injoignable (ICMP)"
|
||||
description: "{{ $labels.instance }} ne répond plus au ping depuis 3 min. Machine éteinte, lien réseau ou switch ?"
|
||||
|
||||
# --- Service exposé (Traefik / *.lab.local) en échec ---
|
||||
- alert: ServiceEndpointDown
|
||||
expr: probe_success{job="blackbox-http"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Service {{ $labels.instance }} ne répond pas (HTTP)"
|
||||
description: "La sonde HTTP de bout en bout (DNS lab → Traefik) sur {{ $labels.instance }} échoue depuis 5 min (pas de 2xx). Voir probe_http_status_code pour le code renvoyé."
|
||||
|
||||
# --- Latence LAN anormale ---
|
||||
- alert: HighLanLatency
|
||||
expr: probe_icmp_duration_seconds{job="blackbox-icmp",scope="lan",phase="rtt"} > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Latence LAN élevée vers {{ $labels.instance }}"
|
||||
description: "RTT ICMP = {{ $value | printf \"%.3f\" }}s vers {{ $labels.instance }} depuis 10 min (attendu < 5 ms sur le LAN cluster). Switch ou câble ?"
|
||||
|
||||
# --- Service lent (réponse HTTP) ---
|
||||
- alert: SlowServiceResponse
|
||||
expr: probe_duration_seconds{job="blackbox-http"} > 3
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Réponse lente de {{ $labels.instance }}"
|
||||
description: "La sonde HTTP met {{ $value | printf \"%.1f\" }}s sur {{ $labels.instance }} depuis 10 min."
|
||||
|
||||
# --- blackbox_exporter lui-même injoignable ---
|
||||
- alert: BlackboxExporterDown
|
||||
expr: up{job="blackbox-http"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "blackbox_exporter injoignable"
|
||||
description: "Prometheus ne peut plus scraper blackbox_exporter depuis 5 min : toute la surveillance réseau active est aveugle."
|
||||
|
|
@ -81,6 +81,82 @@ prometheus:
|
|||
model: qwen3-8b
|
||||
type: gpu
|
||||
|
||||
# --- Sondes réseau actives (blackbox_exporter) ---
|
||||
# Le relabeling envoie la cible réelle en ?target=… et fait pointer le
|
||||
# scrape vers le service blackbox. Cf. admin/infra/reseau.md.
|
||||
- job_name: blackbox-icmp
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [icmp]
|
||||
static_configs:
|
||||
- targets:
|
||||
- 192.168.10.1 # storage-01 — passerelle / NAT / DNS
|
||||
- 192.168.10.11 # compute-01 (control-plane)
|
||||
- 192.168.10.12 # compute-02
|
||||
- 192.168.10.13 # compute-03
|
||||
- 192.168.10.20 # gpu-01
|
||||
labels:
|
||||
scope: lan
|
||||
- targets:
|
||||
- 1.1.1.1 # Cloudflare — joignabilité internet
|
||||
- 9.9.9.9 # Quad9 — joignabilité internet
|
||||
labels:
|
||||
scope: internet
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
|
||||
- job_name: blackbox-http
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
- http://grafana.lab.local
|
||||
- http://prometheus.lab.local
|
||||
- http://alertmanager.lab.local
|
||||
- http://argocd.lab.local
|
||||
- http://openwebui.lab.local
|
||||
- http://n8n.lab.local
|
||||
- http://ghostfolio.lab.local
|
||||
- http://searxng.lab.local
|
||||
- http://stt.lab.local
|
||||
labels:
|
||||
scope: ingress
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
|
||||
# DNS : module choisi par cible (label module → __param_module), serveur
|
||||
# interrogé = dnsmasq (192.168.10.1:53). dns_lab teste le wildcard
|
||||
# *.lab.local, dns_external teste le forwarding vers l'upstream.
|
||||
- job_name: blackbox-dns
|
||||
metrics_path: /probe
|
||||
static_configs:
|
||||
- targets: ['192.168.10.1:53']
|
||||
labels:
|
||||
module: dns_lab
|
||||
instance: 'dnsmasq:grafana.lab.local'
|
||||
- targets: ['192.168.10.1:53']
|
||||
labels:
|
||||
module: dns_external
|
||||
instance: 'dnsmasq:one.one.one.one'
|
||||
relabel_configs:
|
||||
- source_labels: [module]
|
||||
target_label: __param_module
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
|
||||
alertmanager:
|
||||
ingress:
|
||||
enabled: true
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue