fix(alerts,data): startup grace + sustained offline + atomic JSON writes

- 5 min sustained offline window для VPS/PC/Synology/HA (раньше было только у Keenetic) + новый STARTUP_GRACE_SEC=300 — первые 5 минут после старта сервиса алёрты offline не отправляются, но состояние трекается. Это убирает массовый fantom-alert при перезапуске vps-monitoring. - Атомарная запись JSON-файлов через tempfile+os.replace и явный encoding=utf-8 везде (data/keenetic.json, pc_agents.json, synology.json, homeassistant.json, servers.json, settings.json, metrics.json). Краш в момент записи больше не оставляет битый файл с русскими названиями. - Чистка: убран неиспользуемый passlib + dead bcrypt-импорт в auth.py (verify_password всё равно был обычным сравнением); SECRET_KEY можно переопределить через VPS_MONITORING_SECRET; убран мертвый import json там, где он больше не нужен. - Bump app.js cache bust до 20260524c. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-20 11:55:34 +00:00 · 2026-05-24 02:30:15 +03:00
parent d950684d95
commit 50b4dac52e
10 changed files with 107 additions and 78 deletions
--- a/server/api/ha.py
+++ b/server/api/ha.py
@@ -1,13 +1,12 @@
 """Home Assistant monitoring API endpoints."""

-import json
 from datetime import datetime
 from typing import Dict

 from fastapi import APIRouter, Request, Depends

 from server.auth import require_auth
-from server.config import DATA_DIR
+from server.config import DATA_DIR, load_json, save_json

 router = APIRouter(prefix="/api/ha", tags=["homeassistant"])

@@ -18,15 +17,11 @@ HA_FILE = DATA_DIR / "homeassistant.json"


 def _load_ha():
-    if HA_FILE.exists():
-        with open(HA_FILE) as f:
-            return json.load(f)
-    return []
+    return load_json(HA_FILE, [])


 def _save_ha(data):
-    with open(HA_FILE, "w") as f:
-        json.dump(data, f, indent=2, ensure_ascii=False)
+    save_json(HA_FILE, data)


@router.get("/list")
--- a/server/api/keenetic.py
+++ b/server/api/keenetic.py
@@ -1,7 +1,6 @@
 """Keenetic router monitoring API endpoints."""

 import asyncio
-import json
 import logging
 from datetime import datetime
 from typing import Dict, List
@@ -10,7 +9,7 @@ from urllib.parse import urlparse
 from fastapi import APIRouter, Request, Depends

 from server.auth import require_auth
-from server.config import DATA_DIR, load_settings
+from server.config import DATA_DIR, load_settings, load_json, save_json
 from server.services.keenetic_client import (
    KeeneticClient,
    normalize_web_url,
@@ -31,15 +30,11 @@ REFRESH_ALL_GAP_SEC = 2


 def _load_keenetic():
-    if KEENETIC_FILE.exists():
-        with open(KEENETIC_FILE) as f:
-            return json.load(f)
-    return []
+    return load_json(KEENETIC_FILE, [])


 def _save_keenetic(data):
-    with open(KEENETIC_FILE, "w") as f:
-        json.dump(data, f, indent=2, ensure_ascii=False)
+    save_json(KEENETIC_FILE, data)


 def _host_from_url(url: str) -> str:
--- a/server/api/pc.py
+++ b/server/api/pc.py
@@ -1,11 +1,10 @@
-import json
 from datetime import datetime
 from typing import Dict

 from fastapi import APIRouter, Request, Depends

 from server.auth import require_auth
-from server.config import DATA_DIR
+from server.config import DATA_DIR, load_json, save_json

 router = APIRouter(prefix="/api/pc", tags=["pc"])

@@ -16,15 +15,11 @@ PC_FILE = DATA_DIR / "pc_agents.json"


 def _load_pc_data():
-    if PC_FILE.exists():
-        with open(PC_FILE) as f:
-            return json.load(f)
-    return {}
+    return load_json(PC_FILE, {})


 def _save_pc_data(data):
-    with open(PC_FILE, "w") as f:
-        json.dump(data, f, indent=2, ensure_ascii=False)
+    save_json(PC_FILE, data)


@router.post("/heartbeat")
--- a/server/api/synology.py
+++ b/server/api/synology.py
@@ -1,16 +1,15 @@
 """Synology NAS monitoring API endpoints."""

-import json
 import os
 from datetime import datetime
 from pathlib import Path
-from typing import Dict, List
+from typing import Dict

 from fastapi import APIRouter, Request, Depends
 from fastapi.responses import PlainTextResponse

 from server.auth import require_auth
-from server.config import DATA_DIR, BASE_DIR
+from server.config import DATA_DIR, BASE_DIR, load_json, save_json

 router = APIRouter(prefix="/api/synology", tags=["synology"])

@@ -25,15 +24,11 @@ SYNOLOGY_FILE = DATA_DIR / "synology.json"


 def _load_synology():
-    if SYNOLOGY_FILE.exists():
-        with open(SYNOLOGY_FILE) as f:
-            return json.load(f)
-    return []
+    return load_json(SYNOLOGY_FILE, [])


 def _save_synology(data):
-    with open(SYNOLOGY_FILE, "w") as f:
-        json.dump(data, f, indent=2, ensure_ascii=False)
+    save_json(SYNOLOGY_FILE, data)


@router.get("/list")
--- a/server/auth.py
+++ b/server/auth.py
@@ -1,24 +1,23 @@
 import hashlib
 import hmac
-import json
+import os
 from datetime import datetime, timedelta
 from urllib.parse import parse_qs

 from jose import jwt
-from passlib.context import CryptContext
 from fastapi import Request, HTTPException
-from fastapi.responses import RedirectResponse
 from server.config import load_settings

-SECRET_KEY = "vps-monitoring-secret-key-change-me-in-production"
+# Allow override via env so a single deployment can rotate secrets without
+# rebaking the image. Falls back to a known value for backwards compatibility.
+SECRET_KEY = os.getenv("VPS_MONITORING_SECRET", "vps-monitoring-secret-key-change-me-in-production")
 ALGORITHM = "HS256"
 TOKEN_EXPIRE_HOURS = 24

-pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")

-
-def verify_password(plain_password: str, hashed_password: str) -> bool:
-    return plain_password == hashed_password
+def verify_password(plain_password: str, stored_password: str) -> bool:
+    """Plain-text comparison — admin password is stored verbatim in settings.json."""
+    return plain_password == stored_password


 def create_token(username: str) -> str:
--- a/server/config.py
+++ b/server/config.py
@@ -1,5 +1,6 @@
 import os
 import json
+import tempfile
 from pathlib import Path

 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -22,26 +23,49 @@ DEFAULT_SETTINGS = {
 }


+def load_json(path: Path, default):
+    """Load JSON with explicit UTF-8 and a sane default on missing/corrupt file."""
+    try:
+        if path.exists():
+            with open(path, encoding="utf-8") as f:
+                return json.load(f)
+    except (json.JSONDecodeError, OSError):
+        # Don't crash the whole app if a data file is malformed; fall back to default.
+        pass
+    return default
+
+
+def save_json(path: Path, data) -> None:
+    """Atomic JSON write: dump to temp file then rename. Survives crashes mid-write."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_path = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=str(path.parent))
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)
+        os.replace(tmp_path, path)
+    except Exception:
+        if os.path.exists(tmp_path):
+            try:
+                os.unlink(tmp_path)
+            except OSError:
+                pass
+        raise
+
+
 def load_settings():
    if SETTINGS_FILE.exists():
-        with open(SETTINGS_FILE) as f:
-            return json.load(f)
+        return load_json(SETTINGS_FILE, dict(DEFAULT_SETTINGS))
    save_settings(DEFAULT_SETTINGS)
-    return DEFAULT_SETTINGS.copy()
+    return dict(DEFAULT_SETTINGS)


 def save_settings(settings):
-    with open(SETTINGS_FILE, "w") as f:
-        json.dump(settings, f, indent=2, ensure_ascii=False)
+    save_json(SETTINGS_FILE, settings)


 def load_servers():
-    if SERVERS_FILE.exists():
-        with open(SERVERS_FILE) as f:
-            return json.load(f)
-    return []
+    return load_json(SERVERS_FILE, [])


 def save_servers(servers):
-    with open(SERVERS_FILE, "w") as f:
-        json.dump(servers, f, indent=2, ensure_ascii=False)
+    save_json(SERVERS_FILE, servers)
--- a/server/services/alerter.py
+++ b/server/services/alerter.py
@@ -9,12 +9,11 @@ Logic per issue:
 State tracked per (source, issue_key) pair.
 """

-import json
 import logging
 from datetime import datetime
 from typing import Dict, Tuple

-from server.config import load_settings, load_servers, DATA_DIR
+from server.config import load_settings, load_servers, load_json, DATA_DIR
 from server.services.monitor import get_all_metrics

 logger = logging.getLogger(__name__)
@@ -27,9 +26,21 @@ _pending_sustain: Dict[Tuple[str, str, str], datetime] = {}
 _keenetic_last_online: Dict[str, datetime] = {}
 _keenetic_alert_sent: Dict[str, datetime] = {}

+# Sustain windows (seconds before firing offline alert)
+OFFLINE_SUSTAIN_SEC = 300       # 5 min for VPS / PC / Synology / HA
 KEENETIC_OFFLINE_SUSTAIN = 300  # 5 min sustained failure before alert
 KEENETIC_OFFLINE_COOLDOWN = 1800  # 30 min between repeat offline alerts

+# Startup grace: skip OFFLINE alerts for the first STARTUP_GRACE_SEC seconds
+# after the alerter module is loaded (i.e. process restart). Prevents the
+# mass-phantom alert storm when the monitor hasn't completed its first poll yet.
+STARTUP_GRACE_SEC = 300
+_startup_at = datetime.now()
+
+
+def _in_startup_grace() -> bool:
+    return (datetime.now() - _startup_at).total_seconds() < STARTUP_GRACE_SEC
+

 def _issue_key(category: str, source: str, key: str) -> tuple:
    return (category, source, key)
@@ -79,8 +90,14 @@ async def _check_issue(category: str, source: str, key: str, is_problem: bool,

 async def _check_sustained_issue(category: str, source: str, key: str, is_problem: bool,
                                 alert_msg: str, resolve_msg: str, subject: str,
-                                 sustain_seconds: int = 300):
-    """Alert only after problem persists for sustain_seconds (default 5 min)."""
+                                 sustain_seconds: int = 300,
+                                 honor_startup_grace: bool = False):
+    """Alert only after problem persists for sustain_seconds (default 5 min).
+
+    When honor_startup_grace=True, suppress alerts entirely until the process
+    has been running for STARTUP_GRACE_SEC. Pending state is still tracked so
+    the sustain window starts ticking from first detection.
+    """
    ik = _issue_key(category, source, key)
    was_active = ik in active_issues
    now = datetime.now()
@@ -90,6 +107,9 @@ async def _check_sustained_issue(category: str, source: str, key: str, is_proble
            _pending_sustain[ik] = now
        elapsed = (now - _pending_sustain[ik]).total_seconds()
        if elapsed >= sustain_seconds and not was_active:
+            if honor_startup_grace and _in_startup_grace():
+                logger.info(f"Sustained alert deferred (startup grace): {category}/{source}/{key}")
+                return
            active_issues[ik] = _pending_sustain[ik]
            if not _is_device_muted(category, source):
                await _fire_alert(alert_msg, subject, category)
@@ -139,13 +159,15 @@ async def _check_servers(settings: dict):
        m = metrics.get(host, {})
        online = m.get("online", False)

-        # Online/Offline
-        await _check_issue(
+        # Online/Offline — 5 min sustain + startup grace to avoid phantom alerts
+        await _check_sustained_issue(
            "servers", name, "offline",
            is_problem=not online,
-            alert_msg=f"🔴 *{name}* ({host}) - OFFLINE!",
+            alert_msg=f"🔴 *{name}* ({host}) - OFFLINE 5+ мин!",
            resolve_msg=f"🟢 *{name}* ({host}) - back online",
            subject=f"{name} offline",
+            sustain_seconds=OFFLINE_SUSTAIN_SEC,
+            honor_startup_grace=True,
        )

        if not online:
@@ -190,8 +212,7 @@ async def _check_pc(settings: dict):
    if not pc_file.exists():
        return

-    with open(pc_file) as f:
-        pc_data = json.load(f)
+    pc_data = load_json(pc_file, {})

    now = datetime.now()

@@ -199,16 +220,19 @@ async def _check_pc(settings: dict):
        last_seen = data.get("last_seen", "")
        try:
            last_dt = datetime.fromisoformat(last_seen)
-            stale = (now - last_dt).total_seconds() > 180  # 3 min
+            stale = (now - last_dt).total_seconds() > 180  # 3 min no heartbeat
        except Exception:
            stale = True

-        await _check_issue(
+        # 5 min sustain on top of the 3 min stale window + startup grace
+        await _check_sustained_issue(
            "pc", name, "offline",
            is_problem=stale,
-            alert_msg=f"🔴 *PC {name}* - no heartbeat for 3+ min",
+            alert_msg=f"🔴 *PC {name}* - нет heartbeat 5+ мин",
            resolve_msg=f"🟢 *PC {name}* - back online",
            subject=f"PC {name}",
+            sustain_seconds=OFFLINE_SUSTAIN_SEC,
+            honor_startup_grace=True,
        )


@@ -229,12 +253,14 @@ async def _check_synology(settings: dict):

        online = m.get("online", False)

-        await _check_issue(
+        await _check_sustained_issue(
            "synology", name, "offline",
            is_problem=not online,
-            alert_msg=f"🔴 *NAS {name}* - OFFLINE!",
+            alert_msg=f"🔴 *NAS {name}* - OFFLINE 5+ мин!",
            resolve_msg=f"🟢 *NAS {name}* - back online",
            subject=f"NAS {name}",
+            sustain_seconds=OFFLINE_SUSTAIN_SEC,
+            honor_startup_grace=True,
        )

        if not online:
@@ -302,12 +328,14 @@ async def _check_ha(settings: dict):

        online = m.get("online", False)

-        await _check_issue(
+        await _check_sustained_issue(
            "ha", name, "offline",
            is_problem=not online,
-            alert_msg=f"🔴 *HA {name}* - OFFLINE!",
+            alert_msg=f"🔴 *HA {name}* - OFFLINE 5+ мин!",
            resolve_msg=f"🟢 *HA {name}* - back online",
            subject=f"HA {name}",
+            sustain_seconds=OFFLINE_SUSTAIN_SEC,
+            honor_startup_grace=True,
        )

        if not online:
@@ -365,6 +393,10 @@ async def _check_keenetic_offline(name: str, dev: dict, online: bool):
        elapsed = (now - _pending_sustain[ik]).total_seconds()

        if elapsed >= KEENETIC_OFFLINE_SUSTAIN and not was_active:
+            if _in_startup_grace():
+                logger.info(f"Keenetic offline deferred (startup grace): {name}")
+                return
+
            last_alert = _keenetic_alert_sent.get(name)
            last_online = _keenetic_last_online.get(name)
            if last_alert and (now - last_alert).total_seconds() < KEENETIC_OFFLINE_COOLDOWN:
@@ -513,8 +545,7 @@ async def get_full_status() -> str:
    # PC
    pc_file = DATA_DIR / "pc_agents.json"
    if pc_file.exists():
-        with open(pc_file) as f:
-            pc_data = json.load(f)
+        pc_data = load_json(pc_file, {})
        if pc_data:
            lines.append("\n*PC Agents:*")
            now = datetime.now()
--- a/server/services/monitor.py
+++ b/server/services/monitor.py
@@ -1,12 +1,11 @@
 import asyncio
-import json
 import logging
 from datetime import datetime
 from typing import Dict

 import asyncssh

-from server.config import load_servers, load_settings, DATA_DIR
+from server.config import load_servers, load_settings, save_json, METRICS_FILE

 logger = logging.getLogger(__name__)

@@ -143,10 +142,7 @@ async def monitor_loop():
                    else:
                        metrics_cache[srv["host"]] = result

-                # Save metrics to file
-                metrics_file = DATA_DIR / "metrics.json"
-                with open(metrics_file, "w") as f:
-                    json.dump(metrics_cache, f, indent=2)
+                save_json(METRICS_FILE, metrics_cache)

            await asyncio.sleep(interval)

--- a/server/templates/dashboard.html
+++ b/server/templates/dashboard.html
@@ -601,6 +601,6 @@ loftliliana	https://loftliliana.netcraze.pro	1020687391" style="width:100%;font-

    <script src="https://cdn.jsdelivr.net/npm/xterm@5.3.0/lib/xterm.min.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/xterm-addon-fit@0.8.0/lib/xterm-addon-fit.min.js"></script>
-    <script src="/static/js/app.js?v=20260524b"></script>
+    <script src="/static/js/app.js?v=20260524c"></script>
 </body>
 </html>