mirror of
https://github.com/Alkatrazz24/Funk-lab.git
synced 2026-07-08 08:44:43 +02:00
feat(monitoring): surveillance réseau active via blackbox_exporter
Sondes actives ICMP/HTTP/DNS qui complètent le monitoring passif
(node_exporter + up{}) : joignabilité hôtes/passerelle, accès internet,
santé DNS *.lab.local + forwarding, latence LAN, services de bout en
bout via Traefik.
- blackbox_exporter (chart 11.13.0, ArgoCD, ns monitoring) — ICMP via
NET_RAW, DNS du pod = dnsmasq (chemin réel lab → Traefik)
- 3 jobs blackbox-icmp/-http/-dns (additionalScrapeConfigs)
- alerts-network.yaml : internet, DNS, passerelle, hôtes, services,
latence, exporter (→ pipeline AlertManager existant)
- Asa : outil network_status + contexte présélectionnable reseau
- doc admin/infra/reseau.md + CLAUDE.md + progress
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Rc3dLKfqEPHexvFCCx5ZEF
This commit is contained in:
Claude
parent
ebb2eec184
commit
649c6fde8e
11 changed files with 527 additions and 5 deletions
29
k8s/apps-of-apps/apps/blackbox-exporter.yaml
Normal file
29
k8s/apps-of-apps/apps/blackbox-exporter.yaml
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: blackbox-exporter
|
||||
namespace: argocd
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
spec:
|
||||
project: default
|
||||
sources:
|
||||
- repoURL: https://prometheus-community.github.io/helm-charts
|
||||
chart: prometheus-blackbox-exporter
|
||||
targetRevision: 11.13.0
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/k8s/infra/blackbox-exporter/values.yaml
|
||||
- repoURL: git@github.com:Alkatrazz24/Funk-lab.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: monitoring
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ServerSideApply=true
|
||||
108
k8s/infra/blackbox-exporter/values.yaml
Normal file
108
k8s/infra/blackbox-exporter/values.yaml
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
# prometheus-blackbox-exporter — cluster Funk
|
||||
# Sondes réseau actives (ICMP / HTTP / DNS) scrappées par Prometheus.
|
||||
# Les cibles ne sont PAS définies ici : elles vivent dans
|
||||
# k8s/infra/monitoring/values.yaml (prometheusSpec.additionalScrapeConfigs,
|
||||
# jobs blackbox-*), comme les autres scrapes hors-cluster (storage-01, gpu-01).
|
||||
|
||||
# Nom de service prévisible → blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
fullnameOverride: blackbox-exporter
|
||||
|
||||
replicas: 1
|
||||
|
||||
# Petite empreinte (compute-02/03 = 8 GB, OOM-killer actif).
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 64Mi
|
||||
|
||||
# ICMP nécessite la capability NET_RAW (raw sockets). La ns monitoring est en
|
||||
# PodSecurity « privileged » → autorisé. On garde le reste verrouillé.
|
||||
securityContext:
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
runAsNonRoot: true
|
||||
readOnlyRootFilesystem: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["NET_RAW"]
|
||||
|
||||
# Résolution DNS du pod : on tape d'abord dnsmasq (192.168.10.1) pour que les
|
||||
# sondes HTTP des services *.lab.local empruntent le VRAI chemin (DNS lab →
|
||||
# Traefik). Repli sur 1.1.1.1 pour qu'une panne dnsmasq n'aveugle pas les
|
||||
# sondes internet. dnsPolicy=None car blackbox n'a pas besoin du DNS cluster.
|
||||
dnsPolicy: "None"
|
||||
dnsConfig:
|
||||
nameservers:
|
||||
- "192.168.10.1" # dnsmasq (storage-01) — résout *.lab.local
|
||||
- "1.1.1.1" # repli internet
|
||||
searches:
|
||||
- "lab.local"
|
||||
|
||||
# On scrape via additionalScrapeConfigs (cf. monitoring/values.yaml), donc pas
|
||||
# besoin du ServiceMonitor ni des règles packagées par le chart.
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
prometheusRule:
|
||||
enabled: false
|
||||
|
||||
# Modules de sonde. Chaque job Prometheus référence un module par son nom.
|
||||
config:
|
||||
modules:
|
||||
# --- HTTP : service vivant et code 2xx (suit les redirections) ---
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 5s
|
||||
http:
|
||||
valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
|
||||
follow_redirects: true
|
||||
preferred_ip_protocol: "ip4"
|
||||
ip_protocol_fallback: false
|
||||
|
||||
# --- HTTP en HTTPS auto-signé (lab) : ne vérifie pas le certificat ---
|
||||
http_2xx_insecure:
|
||||
prober: http
|
||||
timeout: 5s
|
||||
http:
|
||||
valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
|
||||
follow_redirects: true
|
||||
preferred_ip_protocol: "ip4"
|
||||
fail_if_not_ssl: false
|
||||
tls_config:
|
||||
insecure_skip_verify: true
|
||||
|
||||
# --- ICMP : joignabilité + latence L3 (hôtes, passerelle, internet) ---
|
||||
icmp:
|
||||
prober: icmp
|
||||
timeout: 5s
|
||||
icmp:
|
||||
preferred_ip_protocol: "ip4"
|
||||
ip_protocol_fallback: false
|
||||
|
||||
# --- DNS : dnsmasq résout-il un nom du lab ? (wildcard *.lab.local) ---
|
||||
dns_lab:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: "grafana.lab.local"
|
||||
query_type: "A"
|
||||
preferred_ip_protocol: "ip4"
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp:
|
||||
- ".*"
|
||||
|
||||
# --- DNS : forwarding externe (dnsmasq → upstream) fonctionne-t-il ? ---
|
||||
dns_external:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: "one.one.one.one"
|
||||
query_type: "A"
|
||||
preferred_ip_protocol: "ip4"
|
||||
|
||||
# --- TCP : poignée de main sur un port (réserve, ex. débogage) ---
|
||||
tcp_connect:
|
||||
prober: tcp
|
||||
timeout: 5s
|
||||
102
k8s/infra/monitoring/alerts/alerts-network.yaml
Normal file
102
k8s/infra/monitoring/alerts/alerts-network.yaml
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: funk-network-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: funk.network
|
||||
interval: 1m
|
||||
rules:
|
||||
|
||||
# --- Joignabilité internet (NAT storage-01 / lien WAN) ---
|
||||
- alert: InternetDown
|
||||
expr: sum(probe_success{job="blackbox-icmp",scope="internet"}) == 0
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Plus d'accès internet depuis le cluster"
|
||||
description: "Aucune ancre internet (Cloudflare/Quad9) ne répond au ping depuis 3 min. Lien WAN ou NAT storage-01 probablement en cause."
|
||||
|
||||
# --- DNS : résolution *.lab.local (cœur de dnsmasq) ---
|
||||
- alert: DnsLabResolutionFailed
|
||||
expr: probe_success{job="blackbox-dns",module="dns_lab"} == 0
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "dnsmasq ne résout plus *.lab.local"
|
||||
description: "La sonde DNS sur 192.168.10.1:53 (grafana.lab.local) échoue depuis 3 min. Tout l'accès aux services par nom est cassé."
|
||||
|
||||
# --- DNS : forwarding vers l'upstream ---
|
||||
- alert: DnsForwardingFailed
|
||||
expr: probe_success{job="blackbox-dns",module="dns_external"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Forwarding DNS externe en échec"
|
||||
description: "dnsmasq ne résout plus les noms externes (one.one.one.one) depuis 5 min. Résolution interne possiblement OK — vérifier l'upstream."
|
||||
|
||||
# --- Passerelle (storage-01) injoignable en ICMP ---
|
||||
- alert: GatewayUnreachable
|
||||
expr: probe_success{job="blackbox-icmp",instance="192.168.10.1"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Passerelle 192.168.10.1 (storage-01) injoignable"
|
||||
description: "storage-01 ne répond plus au ping depuis le cluster (2 min). NFS, DNS, NAT, PostgreSQL probablement impactés."
|
||||
|
||||
# --- Hôte du LAN cluster injoignable (hors passerelle) ---
|
||||
- alert: ClusterHostUnreachable
|
||||
expr: probe_success{job="blackbox-icmp",scope="lan",instance!="192.168.10.1"} == 0
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Hôte {{ $labels.instance }} injoignable (ICMP)"
|
||||
description: "{{ $labels.instance }} ne répond plus au ping depuis 3 min. Machine éteinte, lien réseau ou switch ?"
|
||||
|
||||
# --- Service exposé (Traefik / *.lab.local) en échec ---
|
||||
- alert: ServiceEndpointDown
|
||||
expr: probe_success{job="blackbox-http"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Service {{ $labels.instance }} ne répond pas (HTTP)"
|
||||
description: "La sonde HTTP de bout en bout (DNS lab → Traefik) sur {{ $labels.instance }} échoue depuis 5 min (pas de 2xx). Voir probe_http_status_code pour le code renvoyé."
|
||||
|
||||
# --- Latence LAN anormale ---
|
||||
- alert: HighLanLatency
|
||||
expr: probe_icmp_duration_seconds{job="blackbox-icmp",scope="lan",phase="rtt"} > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Latence LAN élevée vers {{ $labels.instance }}"
|
||||
description: "RTT ICMP = {{ $value | printf \"%.3f\" }}s vers {{ $labels.instance }} depuis 10 min (attendu < 5 ms sur le LAN cluster). Switch ou câble ?"
|
||||
|
||||
# --- Service lent (réponse HTTP) ---
|
||||
- alert: SlowServiceResponse
|
||||
expr: probe_duration_seconds{job="blackbox-http"} > 3
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Réponse lente de {{ $labels.instance }}"
|
||||
description: "La sonde HTTP met {{ $value | printf \"%.1f\" }}s sur {{ $labels.instance }} depuis 10 min."
|
||||
|
||||
# --- blackbox_exporter lui-même injoignable ---
|
||||
- alert: BlackboxExporterDown
|
||||
expr: up{job="blackbox-http"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "blackbox_exporter injoignable"
|
||||
description: "Prometheus ne peut plus scraper blackbox_exporter depuis 5 min : toute la surveillance réseau active est aveugle."
|
||||
|
|
@ -81,6 +81,82 @@ prometheus:
|
|||
model: qwen3-8b
|
||||
type: gpu
|
||||
|
||||
# --- Sondes réseau actives (blackbox_exporter) ---
|
||||
# Le relabeling envoie la cible réelle en ?target=… et fait pointer le
|
||||
# scrape vers le service blackbox. Cf. admin/infra/reseau.md.
|
||||
- job_name: blackbox-icmp
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [icmp]
|
||||
static_configs:
|
||||
- targets:
|
||||
- 192.168.10.1 # storage-01 — passerelle / NAT / DNS
|
||||
- 192.168.10.11 # compute-01 (control-plane)
|
||||
- 192.168.10.12 # compute-02
|
||||
- 192.168.10.13 # compute-03
|
||||
- 192.168.10.20 # gpu-01
|
||||
labels:
|
||||
scope: lan
|
||||
- targets:
|
||||
- 1.1.1.1 # Cloudflare — joignabilité internet
|
||||
- 9.9.9.9 # Quad9 — joignabilité internet
|
||||
labels:
|
||||
scope: internet
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
|
||||
- job_name: blackbox-http
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
- http://grafana.lab.local
|
||||
- http://prometheus.lab.local
|
||||
- http://alertmanager.lab.local
|
||||
- http://argocd.lab.local
|
||||
- http://openwebui.lab.local
|
||||
- http://n8n.lab.local
|
||||
- http://ghostfolio.lab.local
|
||||
- http://searxng.lab.local
|
||||
- http://stt.lab.local
|
||||
labels:
|
||||
scope: ingress
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
|
||||
# DNS : module choisi par cible (label module → __param_module), serveur
|
||||
# interrogé = dnsmasq (192.168.10.1:53). dns_lab teste le wildcard
|
||||
# *.lab.local, dns_external teste le forwarding vers l'upstream.
|
||||
- job_name: blackbox-dns
|
||||
metrics_path: /probe
|
||||
static_configs:
|
||||
- targets: ['192.168.10.1:53']
|
||||
labels:
|
||||
module: dns_lab
|
||||
instance: 'dnsmasq:grafana.lab.local'
|
||||
- targets: ['192.168.10.1:53']
|
||||
labels:
|
||||
module: dns_external
|
||||
instance: 'dnsmasq:one.one.one.one'
|
||||
relabel_configs:
|
||||
- source_labels: [module]
|
||||
target_label: __param_module
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
|
||||
alertmanager:
|
||||
ingress:
|
||||
enabled: true
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue