fix(alerts,data): startup grace + sustained offline + atomic JSON writes

- 5 min sustained offline window для VPS/PC/Synology/HA (раньше было только у
  Keenetic) + новый STARTUP_GRACE_SEC=300 — первые 5 минут после старта сервиса
  алёрты offline не отправляются, но состояние трекается. Это убирает массовый
  fantom-alert при перезапуске vps-monitoring.
- Атомарная запись JSON-файлов через tempfile+os.replace и явный encoding=utf-8
  везде (data/keenetic.json, pc_agents.json, synology.json, homeassistant.json,
  servers.json, settings.json, metrics.json). Краш в момент записи больше не
  оставляет битый файл с русскими названиями.
- Чистка: убран неиспользуемый passlib + dead bcrypt-импорт в auth.py
  (verify_password всё равно был обычным сравнением); SECRET_KEY можно
  переопределить через VPS_MONITORING_SECRET; убран мертвый import json там,
  где он больше не нужен.
- Bump app.js cache bust до 20260524c.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Андрей Бобырев
2026-05-24 02:30:15 +03:00
parent d950684d95
commit 50b4dac52e
10 changed files with 107 additions and 78 deletions

View File

@@ -10,6 +10,5 @@ psutil==6.0.0
aiohttp==3.10.0 aiohttp==3.10.0
httpx==0.27.2 httpx==0.27.2
pydantic==2.9.0 pydantic==2.9.0
passlib[bcrypt]==1.7.4
python-jose[cryptography]==3.3.0 python-jose[cryptography]==3.3.0
apscheduler==3.10.4 apscheduler==3.10.4

View File

@@ -1,13 +1,12 @@
"""Home Assistant monitoring API endpoints.""" """Home Assistant monitoring API endpoints."""
import json
from datetime import datetime from datetime import datetime
from typing import Dict from typing import Dict
from fastapi import APIRouter, Request, Depends from fastapi import APIRouter, Request, Depends
from server.auth import require_auth from server.auth import require_auth
from server.config import DATA_DIR from server.config import DATA_DIR, load_json, save_json
router = APIRouter(prefix="/api/ha", tags=["homeassistant"]) router = APIRouter(prefix="/api/ha", tags=["homeassistant"])
@@ -18,15 +17,11 @@ HA_FILE = DATA_DIR / "homeassistant.json"
def _load_ha(): def _load_ha():
if HA_FILE.exists(): return load_json(HA_FILE, [])
with open(HA_FILE) as f:
return json.load(f)
return []
def _save_ha(data): def _save_ha(data):
with open(HA_FILE, "w") as f: save_json(HA_FILE, data)
json.dump(data, f, indent=2, ensure_ascii=False)
@router.get("/list") @router.get("/list")

View File

@@ -1,7 +1,6 @@
"""Keenetic router monitoring API endpoints.""" """Keenetic router monitoring API endpoints."""
import asyncio import asyncio
import json
import logging import logging
from datetime import datetime from datetime import datetime
from typing import Dict, List from typing import Dict, List
@@ -10,7 +9,7 @@ from urllib.parse import urlparse
from fastapi import APIRouter, Request, Depends from fastapi import APIRouter, Request, Depends
from server.auth import require_auth from server.auth import require_auth
from server.config import DATA_DIR, load_settings from server.config import DATA_DIR, load_settings, load_json, save_json
from server.services.keenetic_client import ( from server.services.keenetic_client import (
KeeneticClient, KeeneticClient,
normalize_web_url, normalize_web_url,
@@ -31,15 +30,11 @@ REFRESH_ALL_GAP_SEC = 2
def _load_keenetic(): def _load_keenetic():
if KEENETIC_FILE.exists(): return load_json(KEENETIC_FILE, [])
with open(KEENETIC_FILE) as f:
return json.load(f)
return []
def _save_keenetic(data): def _save_keenetic(data):
with open(KEENETIC_FILE, "w") as f: save_json(KEENETIC_FILE, data)
json.dump(data, f, indent=2, ensure_ascii=False)
def _host_from_url(url: str) -> str: def _host_from_url(url: str) -> str:

View File

@@ -1,11 +1,10 @@
import json
from datetime import datetime from datetime import datetime
from typing import Dict from typing import Dict
from fastapi import APIRouter, Request, Depends from fastapi import APIRouter, Request, Depends
from server.auth import require_auth from server.auth import require_auth
from server.config import DATA_DIR from server.config import DATA_DIR, load_json, save_json
router = APIRouter(prefix="/api/pc", tags=["pc"]) router = APIRouter(prefix="/api/pc", tags=["pc"])
@@ -16,15 +15,11 @@ PC_FILE = DATA_DIR / "pc_agents.json"
def _load_pc_data(): def _load_pc_data():
if PC_FILE.exists(): return load_json(PC_FILE, {})
with open(PC_FILE) as f:
return json.load(f)
return {}
def _save_pc_data(data): def _save_pc_data(data):
with open(PC_FILE, "w") as f: save_json(PC_FILE, data)
json.dump(data, f, indent=2, ensure_ascii=False)
@router.post("/heartbeat") @router.post("/heartbeat")

View File

@@ -1,16 +1,15 @@
"""Synology NAS monitoring API endpoints.""" """Synology NAS monitoring API endpoints."""
import json
import os import os
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Dict, List from typing import Dict
from fastapi import APIRouter, Request, Depends from fastapi import APIRouter, Request, Depends
from fastapi.responses import PlainTextResponse from fastapi.responses import PlainTextResponse
from server.auth import require_auth from server.auth import require_auth
from server.config import DATA_DIR, BASE_DIR from server.config import DATA_DIR, BASE_DIR, load_json, save_json
router = APIRouter(prefix="/api/synology", tags=["synology"]) router = APIRouter(prefix="/api/synology", tags=["synology"])
@@ -25,15 +24,11 @@ SYNOLOGY_FILE = DATA_DIR / "synology.json"
def _load_synology(): def _load_synology():
if SYNOLOGY_FILE.exists(): return load_json(SYNOLOGY_FILE, [])
with open(SYNOLOGY_FILE) as f:
return json.load(f)
return []
def _save_synology(data): def _save_synology(data):
with open(SYNOLOGY_FILE, "w") as f: save_json(SYNOLOGY_FILE, data)
json.dump(data, f, indent=2, ensure_ascii=False)
@router.get("/list") @router.get("/list")

View File

@@ -1,24 +1,23 @@
import hashlib import hashlib
import hmac import hmac
import json import os
from datetime import datetime, timedelta from datetime import datetime, timedelta
from urllib.parse import parse_qs from urllib.parse import parse_qs
from jose import jwt from jose import jwt
from passlib.context import CryptContext
from fastapi import Request, HTTPException from fastapi import Request, HTTPException
from fastapi.responses import RedirectResponse
from server.config import load_settings from server.config import load_settings
SECRET_KEY = "vps-monitoring-secret-key-change-me-in-production" # Allow override via env so a single deployment can rotate secrets without
# rebaking the image. Falls back to a known value for backwards compatibility.
SECRET_KEY = os.getenv("VPS_MONITORING_SECRET", "vps-monitoring-secret-key-change-me-in-production")
ALGORITHM = "HS256" ALGORITHM = "HS256"
TOKEN_EXPIRE_HOURS = 24 TOKEN_EXPIRE_HOURS = 24
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
def verify_password(plain_password: str, stored_password: str) -> bool:
def verify_password(plain_password: str, hashed_password: str) -> bool: """Plain-text comparison — admin password is stored verbatim in settings.json."""
return plain_password == hashed_password return plain_password == stored_password
def create_token(username: str) -> str: def create_token(username: str) -> str:

View File

@@ -1,5 +1,6 @@
import os import os
import json import json
import tempfile
from pathlib import Path from pathlib import Path
BASE_DIR = Path(__file__).resolve().parent.parent BASE_DIR = Path(__file__).resolve().parent.parent
@@ -22,26 +23,49 @@ DEFAULT_SETTINGS = {
} }
def load_json(path: Path, default):
"""Load JSON with explicit UTF-8 and a sane default on missing/corrupt file."""
try:
if path.exists():
with open(path, encoding="utf-8") as f:
return json.load(f)
except (json.JSONDecodeError, OSError):
# Don't crash the whole app if a data file is malformed; fall back to default.
pass
return default
def save_json(path: Path, data) -> None:
"""Atomic JSON write: dump to temp file then rename. Survives crashes mid-write."""
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=str(path.parent))
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
os.replace(tmp_path, path)
except Exception:
if os.path.exists(tmp_path):
try:
os.unlink(tmp_path)
except OSError:
pass
raise
def load_settings(): def load_settings():
if SETTINGS_FILE.exists(): if SETTINGS_FILE.exists():
with open(SETTINGS_FILE) as f: return load_json(SETTINGS_FILE, dict(DEFAULT_SETTINGS))
return json.load(f)
save_settings(DEFAULT_SETTINGS) save_settings(DEFAULT_SETTINGS)
return DEFAULT_SETTINGS.copy() return dict(DEFAULT_SETTINGS)
def save_settings(settings): def save_settings(settings):
with open(SETTINGS_FILE, "w") as f: save_json(SETTINGS_FILE, settings)
json.dump(settings, f, indent=2, ensure_ascii=False)
def load_servers(): def load_servers():
if SERVERS_FILE.exists(): return load_json(SERVERS_FILE, [])
with open(SERVERS_FILE) as f:
return json.load(f)
return []
def save_servers(servers): def save_servers(servers):
with open(SERVERS_FILE, "w") as f: save_json(SERVERS_FILE, servers)
json.dump(servers, f, indent=2, ensure_ascii=False)

View File

@@ -9,12 +9,11 @@ Logic per issue:
State tracked per (source, issue_key) pair. State tracked per (source, issue_key) pair.
""" """
import json
import logging import logging
from datetime import datetime from datetime import datetime
from typing import Dict, Tuple from typing import Dict, Tuple
from server.config import load_settings, load_servers, DATA_DIR from server.config import load_settings, load_servers, load_json, DATA_DIR
from server.services.monitor import get_all_metrics from server.services.monitor import get_all_metrics
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -27,9 +26,21 @@ _pending_sustain: Dict[Tuple[str, str, str], datetime] = {}
_keenetic_last_online: Dict[str, datetime] = {} _keenetic_last_online: Dict[str, datetime] = {}
_keenetic_alert_sent: Dict[str, datetime] = {} _keenetic_alert_sent: Dict[str, datetime] = {}
# Sustain windows (seconds before firing offline alert)
OFFLINE_SUSTAIN_SEC = 300 # 5 min for VPS / PC / Synology / HA
KEENETIC_OFFLINE_SUSTAIN = 300 # 5 min sustained failure before alert KEENETIC_OFFLINE_SUSTAIN = 300 # 5 min sustained failure before alert
KEENETIC_OFFLINE_COOLDOWN = 1800 # 30 min between repeat offline alerts KEENETIC_OFFLINE_COOLDOWN = 1800 # 30 min between repeat offline alerts
# Startup grace: skip OFFLINE alerts for the first STARTUP_GRACE_SEC seconds
# after the alerter module is loaded (i.e. process restart). Prevents the
# mass-phantom alert storm when the monitor hasn't completed its first poll yet.
STARTUP_GRACE_SEC = 300
_startup_at = datetime.now()
def _in_startup_grace() -> bool:
return (datetime.now() - _startup_at).total_seconds() < STARTUP_GRACE_SEC
def _issue_key(category: str, source: str, key: str) -> tuple: def _issue_key(category: str, source: str, key: str) -> tuple:
return (category, source, key) return (category, source, key)
@@ -79,8 +90,14 @@ async def _check_issue(category: str, source: str, key: str, is_problem: bool,
async def _check_sustained_issue(category: str, source: str, key: str, is_problem: bool, async def _check_sustained_issue(category: str, source: str, key: str, is_problem: bool,
alert_msg: str, resolve_msg: str, subject: str, alert_msg: str, resolve_msg: str, subject: str,
sustain_seconds: int = 300): sustain_seconds: int = 300,
"""Alert only after problem persists for sustain_seconds (default 5 min).""" honor_startup_grace: bool = False):
"""Alert only after problem persists for sustain_seconds (default 5 min).
When honor_startup_grace=True, suppress alerts entirely until the process
has been running for STARTUP_GRACE_SEC. Pending state is still tracked so
the sustain window starts ticking from first detection.
"""
ik = _issue_key(category, source, key) ik = _issue_key(category, source, key)
was_active = ik in active_issues was_active = ik in active_issues
now = datetime.now() now = datetime.now()
@@ -90,6 +107,9 @@ async def _check_sustained_issue(category: str, source: str, key: str, is_proble
_pending_sustain[ik] = now _pending_sustain[ik] = now
elapsed = (now - _pending_sustain[ik]).total_seconds() elapsed = (now - _pending_sustain[ik]).total_seconds()
if elapsed >= sustain_seconds and not was_active: if elapsed >= sustain_seconds and not was_active:
if honor_startup_grace and _in_startup_grace():
logger.info(f"Sustained alert deferred (startup grace): {category}/{source}/{key}")
return
active_issues[ik] = _pending_sustain[ik] active_issues[ik] = _pending_sustain[ik]
if not _is_device_muted(category, source): if not _is_device_muted(category, source):
await _fire_alert(alert_msg, subject, category) await _fire_alert(alert_msg, subject, category)
@@ -139,13 +159,15 @@ async def _check_servers(settings: dict):
m = metrics.get(host, {}) m = metrics.get(host, {})
online = m.get("online", False) online = m.get("online", False)
# Online/Offline # Online/Offline — 5 min sustain + startup grace to avoid phantom alerts
await _check_issue( await _check_sustained_issue(
"servers", name, "offline", "servers", name, "offline",
is_problem=not online, is_problem=not online,
alert_msg=f"🔴 *{name}* ({host}) - OFFLINE!", alert_msg=f"🔴 *{name}* ({host}) - OFFLINE 5+ мин!",
resolve_msg=f"🟢 *{name}* ({host}) - back online", resolve_msg=f"🟢 *{name}* ({host}) - back online",
subject=f"{name} offline", subject=f"{name} offline",
sustain_seconds=OFFLINE_SUSTAIN_SEC,
honor_startup_grace=True,
) )
if not online: if not online:
@@ -190,8 +212,7 @@ async def _check_pc(settings: dict):
if not pc_file.exists(): if not pc_file.exists():
return return
with open(pc_file) as f: pc_data = load_json(pc_file, {})
pc_data = json.load(f)
now = datetime.now() now = datetime.now()
@@ -199,16 +220,19 @@ async def _check_pc(settings: dict):
last_seen = data.get("last_seen", "") last_seen = data.get("last_seen", "")
try: try:
last_dt = datetime.fromisoformat(last_seen) last_dt = datetime.fromisoformat(last_seen)
stale = (now - last_dt).total_seconds() > 180 # 3 min stale = (now - last_dt).total_seconds() > 180 # 3 min no heartbeat
except Exception: except Exception:
stale = True stale = True
await _check_issue( # 5 min sustain on top of the 3 min stale window + startup grace
await _check_sustained_issue(
"pc", name, "offline", "pc", name, "offline",
is_problem=stale, is_problem=stale,
alert_msg=f"🔴 *PC {name}* - no heartbeat for 3+ min", alert_msg=f"🔴 *PC {name}* - нет heartbeat 5+ мин",
resolve_msg=f"🟢 *PC {name}* - back online", resolve_msg=f"🟢 *PC {name}* - back online",
subject=f"PC {name}", subject=f"PC {name}",
sustain_seconds=OFFLINE_SUSTAIN_SEC,
honor_startup_grace=True,
) )
@@ -229,12 +253,14 @@ async def _check_synology(settings: dict):
online = m.get("online", False) online = m.get("online", False)
await _check_issue( await _check_sustained_issue(
"synology", name, "offline", "synology", name, "offline",
is_problem=not online, is_problem=not online,
alert_msg=f"🔴 *NAS {name}* - OFFLINE!", alert_msg=f"🔴 *NAS {name}* - OFFLINE 5+ мин!",
resolve_msg=f"🟢 *NAS {name}* - back online", resolve_msg=f"🟢 *NAS {name}* - back online",
subject=f"NAS {name}", subject=f"NAS {name}",
sustain_seconds=OFFLINE_SUSTAIN_SEC,
honor_startup_grace=True,
) )
if not online: if not online:
@@ -302,12 +328,14 @@ async def _check_ha(settings: dict):
online = m.get("online", False) online = m.get("online", False)
await _check_issue( await _check_sustained_issue(
"ha", name, "offline", "ha", name, "offline",
is_problem=not online, is_problem=not online,
alert_msg=f"🔴 *HA {name}* - OFFLINE!", alert_msg=f"🔴 *HA {name}* - OFFLINE 5+ мин!",
resolve_msg=f"🟢 *HA {name}* - back online", resolve_msg=f"🟢 *HA {name}* - back online",
subject=f"HA {name}", subject=f"HA {name}",
sustain_seconds=OFFLINE_SUSTAIN_SEC,
honor_startup_grace=True,
) )
if not online: if not online:
@@ -365,6 +393,10 @@ async def _check_keenetic_offline(name: str, dev: dict, online: bool):
elapsed = (now - _pending_sustain[ik]).total_seconds() elapsed = (now - _pending_sustain[ik]).total_seconds()
if elapsed >= KEENETIC_OFFLINE_SUSTAIN and not was_active: if elapsed >= KEENETIC_OFFLINE_SUSTAIN and not was_active:
if _in_startup_grace():
logger.info(f"Keenetic offline deferred (startup grace): {name}")
return
last_alert = _keenetic_alert_sent.get(name) last_alert = _keenetic_alert_sent.get(name)
last_online = _keenetic_last_online.get(name) last_online = _keenetic_last_online.get(name)
if last_alert and (now - last_alert).total_seconds() < KEENETIC_OFFLINE_COOLDOWN: if last_alert and (now - last_alert).total_seconds() < KEENETIC_OFFLINE_COOLDOWN:
@@ -513,8 +545,7 @@ async def get_full_status() -> str:
# PC # PC
pc_file = DATA_DIR / "pc_agents.json" pc_file = DATA_DIR / "pc_agents.json"
if pc_file.exists(): if pc_file.exists():
with open(pc_file) as f: pc_data = load_json(pc_file, {})
pc_data = json.load(f)
if pc_data: if pc_data:
lines.append("\n*PC Agents:*") lines.append("\n*PC Agents:*")
now = datetime.now() now = datetime.now()

View File

@@ -1,12 +1,11 @@
import asyncio import asyncio
import json
import logging import logging
from datetime import datetime from datetime import datetime
from typing import Dict from typing import Dict
import asyncssh import asyncssh
from server.config import load_servers, load_settings, DATA_DIR from server.config import load_servers, load_settings, save_json, METRICS_FILE
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -143,10 +142,7 @@ async def monitor_loop():
else: else:
metrics_cache[srv["host"]] = result metrics_cache[srv["host"]] = result
# Save metrics to file save_json(METRICS_FILE, metrics_cache)
metrics_file = DATA_DIR / "metrics.json"
with open(metrics_file, "w") as f:
json.dump(metrics_cache, f, indent=2)
await asyncio.sleep(interval) await asyncio.sleep(interval)

View File

@@ -601,6 +601,6 @@ loftliliana https://loftliliana.netcraze.pro 1020687391" style="width:100%;font-
<script src="https://cdn.jsdelivr.net/npm/xterm@5.3.0/lib/xterm.min.js"></script> <script src="https://cdn.jsdelivr.net/npm/xterm@5.3.0/lib/xterm.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/xterm-addon-fit@0.8.0/lib/xterm-addon-fit.min.js"></script> <script src="https://cdn.jsdelivr.net/npm/xterm-addon-fit@0.8.0/lib/xterm-addon-fit.min.js"></script>
<script src="/static/js/app.js?v=20260524b"></script> <script src="/static/js/app.js?v=20260524c"></script>
</body> </body>
</html> </html>