From 8beccae2b570d7c4a283905e5b0d11de017e5275 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=91=D0=BE=D0=B1?=
 =?UTF-8?q?=D1=8B=D1=80=D0=B5=D0=B2?= <andrejbobyrev@MacBook-Pro.local>
Date: Sun, 24 May 2026 01:39:12 +0300
Subject: [PATCH] fix(keenetic): sustain 5min before router offline alert

Phantom OFFLINE/online Telegram spam from brief poll failures.
Wait 5 min sustained offline, re-verify auth before alerting,
30 min cooldown between repeat alerts, single recovery message.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 README.md                  |  2 +-
 server/services/alerter.py | 85 ++++++++++++++++++++++++++++++++++----
 2 files changed, 79 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index def5719..0d92bda 100644
--- a/README.md
+++ b/README.md
@@ -111,7 +111,7 @@ cd /opt/vps-monitoring
 Настройка: `TELEGRAM_BOT_TOKEN` + `TELEGRAM_CHAT_ID` в systemd или settings.
 
 Keenetic алерты:
-- Router offline / back online
+- Router offline 5+ мин (sustained + re-verify) / back online
 - No internet
 - CPU/RAM > 90%
 - **VPN down 5+ минут** (sustained, deduplicated)
diff --git a/server/services/alerter.py b/server/services/alerter.py
index eb45302..145309d 100644
--- a/server/services/alerter.py
+++ b/server/services/alerter.py
@@ -23,6 +23,12 @@ logger = logging.getLogger(__name__)
 active_issues: Dict[Tuple[str, str, str], datetime] = {}
 # Sustained problems waiting for threshold: {issue_key: datetime_first_seen}
 _pending_sustain: Dict[Tuple[str, str, str], datetime] = {}
+# Keenetic router offline: last confirmed online + last alert sent (cooldown)
+_keenetic_last_online: Dict[str, datetime] = {}
+_keenetic_alert_sent: Dict[str, datetime] = {}
+
+KEENETIC_OFFLINE_SUSTAIN = 300  # 5 min sustained failure before alert
+KEENETIC_OFFLINE_COOLDOWN = 1800  # 30 min between repeat offline alerts
 
 
 def _issue_key(category: str, source: str, key: str) -> tuple:
@@ -331,6 +337,77 @@ async def _check_ha(settings: dict):
 
 # ==================== KEENETIC ====================
 
+async def _verify_keenetic_reachable(dev: dict) -> bool:
+    """Second auth check before firing offline alert (filter poll glitches)."""
+    from server.api.keenetic import _client_for_device
+
+    client = _client_for_device(dev)
+    try:
+        return await client.authenticate()
+    except Exception as e:
+        logger.warning(f"Keenetic re-verify {dev.get('name')}: {e}")
+        return False
+    finally:
+        await client.close()
+
+
+async def _check_keenetic_offline(name: str, dev: dict, online: bool):
+    """Router offline: 5 min sustain, re-verify, cooldown, single recovery."""
+    category = "keenetic"
+    key = "offline"
+    ik = _issue_key(category, name, key)
+    now = datetime.now()
+    was_active = ik in active_issues
+
+    if not online:
+        if ik not in _pending_sustain:
+            _pending_sustain[ik] = now
+        elapsed = (now - _pending_sustain[ik]).total_seconds()
+
+        if elapsed >= KEENETIC_OFFLINE_SUSTAIN and not was_active:
+            last_alert = _keenetic_alert_sent.get(name)
+            last_online = _keenetic_last_online.get(name)
+            if last_alert and (now - last_alert).total_seconds() < KEENETIC_OFFLINE_COOLDOWN:
+                if not last_online or last_online <= last_alert:
+                    return
+
+            if await _verify_keenetic_reachable(dev):
+                _pending_sustain.pop(ik, None)
+                _keenetic_last_online[name] = now
+                logger.info(f"Keenetic offline suppressed (re-verify OK): {name}")
+                return
+
+            active_issues[ik] = _pending_sustain[ik]
+            _keenetic_alert_sent[name] = now
+            host = dev.get("host", "")
+            ts = now.strftime("%H:%M %d.%m.%Y")
+            if not _is_device_muted(category, name):
+                await _fire_alert(
+                    f"🔴 *Router {name}* - OFFLINE 5+ мин!\n"
+                    f"Хост: `{host}`\n"
+                    f"Время: {ts}",
+                    f"Router {name}",
+                    category,
+                )
+                logger.info(f"Keenetic sustained offline alert: {name}")
+            else:
+                logger.info(f"Keenetic offline alert suppressed (muted): {name}")
+    else:
+        _keenetic_last_online[name] = now
+        _pending_sustain.pop(ik, None)
+        if was_active:
+            del active_issues[ik]
+            if not _is_device_muted(category, name):
+                await _fire_alert(
+                    f"🟢 *Router {name}* - back online",
+                    f"Router {name} resolved",
+                    category,
+                )
+                logger.info(f"Keenetic offline resolved: {name}")
+            else:
+                logger.info(f"Keenetic offline resolved (muted): {name}")
+
+
 async def _check_keenetic(settings: dict):
     """Check Keenetic routers."""
     from server.api.keenetic import keenetic_metrics, _load_keenetic
@@ -346,13 +423,7 @@ async def _check_keenetic(settings: dict):
 
         online = m.get("online", False)
 
-        await _check_issue(
-            "keenetic", name, "offline",
-            is_problem=not online,
-            alert_msg=f"🔴 *Router {name}* - OFFLINE!",
-            resolve_msg=f"🟢 *Router {name}* - back online",
-            subject=f"Router {name}",
-        )
+        await _check_keenetic_offline(name, dev, online)
 
         if not online:
             continue