mirror of
https://github.com/andrey271192/PCA_Phobos.git
synced 2026-09-20 11:55:32 +00:00
v1.2.3: debounce server DOWN alerts (3 cycles + retry) fix flapping; update.sh aborts on missing version + --check/--versions + beta channel; panel Check button
This commit is contained in:
46
app.py
46
app.py
@@ -479,6 +479,7 @@ def kick_client_by_id(client_id):
|
|||||||
|
|
||||||
prev_session_keys = None
|
prev_session_keys = None
|
||||||
prev_server_status = {}
|
prev_server_status = {}
|
||||||
|
_down_streak = {}
|
||||||
_last_fanout = 0
|
_last_fanout = 0
|
||||||
server_stats_cache = {}
|
server_stats_cache = {}
|
||||||
server_handshakes_cache = {}
|
server_handshakes_cache = {}
|
||||||
@@ -503,12 +504,19 @@ def get_local_stats():
|
|||||||
|
|
||||||
|
|
||||||
def get_remote_stats(server):
|
def get_remote_stats(server):
|
||||||
try:
|
|
||||||
import urllib.request
|
import urllib.request
|
||||||
url = f"http://{server['ip']}:8444/api/health"
|
url = f"http://{server['ip']}:8444/api/health"
|
||||||
req = urllib.request.Request(url, headers={"X-API-Key": server.get("api_key", "")})
|
req = urllib.request.Request(url, headers={"X-API-Key": server.get("api_key", "")})
|
||||||
resp = urllib.request.urlopen(req, timeout=3)
|
data = None
|
||||||
data = json.loads(resp.read())
|
for _attempt in range(2): # retry once: smooths transient timeouts (1-worker agent)
|
||||||
|
try:
|
||||||
|
data = json.loads(urllib.request.urlopen(req, timeout=6).read())
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
data = None
|
||||||
|
try:
|
||||||
|
if data is None:
|
||||||
|
raise Exception("no response")
|
||||||
# cache handshakes so page renders never block on remote HTTP
|
# cache handshakes so page renders never block on remote HTTP
|
||||||
server_handshakes_cache[server.get("ip", "")] = data.get("handshakes", {}) or {}
|
server_handshakes_cache[server.get("ip", "")] = data.get("handshakes", {}) or {}
|
||||||
# Filter peers: only count known client public keys
|
# Filter peers: only count known client public keys
|
||||||
@@ -777,7 +785,7 @@ def push_config_to_router(client_id):
|
|||||||
|
|
||||||
|
|
||||||
def session_monitor():
|
def session_monitor():
|
||||||
global prev_session_keys, prev_server_status, server_stats_cache, _last_fanout
|
global prev_session_keys, prev_server_status, server_stats_cache, _last_fanout, _down_streak
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
s = load_settings()
|
s = load_settings()
|
||||||
@@ -814,19 +822,24 @@ def session_monitor():
|
|||||||
local_stats = get_local_stats()
|
local_stats = get_local_stats()
|
||||||
server_stats_cache[SERVER_IP] = local_stats
|
server_stats_cache[SERVER_IP] = local_stats
|
||||||
|
|
||||||
|
DOWN_CONFIRM = 3 # consecutive failed cycles before declaring a server DOWN
|
||||||
for srv in servers:
|
for srv in servers:
|
||||||
ip = srv["ip"]
|
ip = srv["ip"]
|
||||||
stats = get_remote_stats(srv)
|
stats = get_remote_stats(srv)
|
||||||
|
raw = stats.get("status", "down")
|
||||||
|
streak = _down_streak.get(ip, 0)
|
||||||
|
streak = streak + 1 if raw == "down" else 0
|
||||||
|
_down_streak[ip] = streak
|
||||||
|
# update cache on OK, or only once a DOWN is confirmed (keep last-good on transient)
|
||||||
|
if raw == "ok" or streak >= DOWN_CONFIRM:
|
||||||
server_stats_cache[ip] = stats
|
server_stats_cache[ip] = stats
|
||||||
|
eff = "down" if streak >= DOWN_CONFIRM else "ok"
|
||||||
was_up = prev_server_status.get(ip, "ok")
|
was_up = prev_server_status.get(ip, "ok")
|
||||||
now_status = stats["status"]
|
if was_up == "ok" and eff == "down":
|
||||||
|
|
||||||
if was_up == "ok" and now_status == "down":
|
|
||||||
tg_send(f"🔴 <b>Server {ip}</b> DOWN!")
|
tg_send(f"🔴 <b>Server {ip}</b> DOWN!")
|
||||||
elif was_up == "down" and now_status == "ok":
|
elif was_up == "down" and eff == "ok":
|
||||||
tg_send(f"🟢 <b>Server {ip}</b> back ONLINE")
|
tg_send(f"🟢 <b>Server {ip}</b> back ONLINE")
|
||||||
|
prev_server_status[ip] = eff
|
||||||
prev_server_status[ip] = now_status
|
|
||||||
|
|
||||||
# periodic config fan-out to secondaries (covers CLI-created clients
|
# periodic config fan-out to secondaries (covers CLI-created clients
|
||||||
# and config drift; gated ~5 min, skips down servers)
|
# and config drift; gated ~5 min, skips down servers)
|
||||||
@@ -1371,9 +1384,12 @@ def labels_page():
|
|||||||
def settings_page():
|
def settings_page():
|
||||||
s = load_settings()
|
s = load_settings()
|
||||||
msg = ""
|
msg = ""
|
||||||
if request.method == "POST" and request.form.get("action") in ("update", "rollback"):
|
if request.method == "POST" and request.form.get("action") in ("update", "rollback", "check"):
|
||||||
if request.form.get("action") == "update":
|
_act = request.form.get("action")
|
||||||
|
if _act == "update":
|
||||||
_out = run_update(request.form.get("target", "").strip() or "main")
|
_out = run_update(request.form.get("target", "").strip() or "main")
|
||||||
|
elif _act == "check":
|
||||||
|
_out = run_update("--check")
|
||||||
else:
|
else:
|
||||||
_out = run_update("--rollback")
|
_out = run_update("--rollback")
|
||||||
s = load_settings()
|
s = load_settings()
|
||||||
@@ -1414,9 +1430,13 @@ def settings_page():
|
|||||||
<p>{tr("Установленная версия","Installed version")}: <b style="color:#a5b4fc">{panel_version()}</b></p>
|
<p>{tr("Установленная версия","Installed version")}: <b style="color:#a5b4fc">{panel_version()}</b></p>
|
||||||
<form method="post" class="form-row">
|
<form method="post" class="form-row">
|
||||||
<input type="hidden" name="action" value="update">
|
<input type="hidden" name="action" value="update">
|
||||||
<input type="text" name="target" placeholder="main / v1.1.0" style="width:150px">
|
<input type="text" name="target" placeholder="main / beta / v1.2.3" style="width:150px">
|
||||||
<button class="btn btn-primary" type="submit" onclick="return confirm('{tr("Обновить панель и скрипты?","Update panel and scripts?")}')">{tr("Обновить","Update")}</button>
|
<button class="btn btn-primary" type="submit" onclick="return confirm('{tr("Обновить панель и скрипты?","Update panel and scripts?")}')">{tr("Обновить","Update")}</button>
|
||||||
</form>
|
</form>
|
||||||
|
<form method="post" style="display:inline">
|
||||||
|
<input type="hidden" name="action" value="check">
|
||||||
|
<button class="btn btn-sm" style="background:#0ea5e9" type="submit">{tr("Проверить обновления","Check for updates")}</button>
|
||||||
|
</form>
|
||||||
<form method="post" style="margin-top:8px">
|
<form method="post" style="margin-top:8px">
|
||||||
<input type="hidden" name="action" value="rollback">
|
<input type="hidden" name="action" value="rollback">
|
||||||
<button class="btn btn-danger btn-sm" type="submit" onclick="return confirm('{tr("Откатить на предыдущую версию?","Roll back to the previous version?")}')">{tr("Откатить на предыдущую","Rollback")}</button>
|
<button class="btn btn-danger btn-sm" type="submit" onclick="return confirm('{tr("Откатить на предыдущую версию?","Roll back to the previous version?")}')">{tr("Откатить на предыдущую","Rollback")}</button>
|
||||||
|
|||||||
18
update.sh
18
update.sh
@@ -60,10 +60,19 @@ apply_ref() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ref_exists() { curl -fsSL -m12 "$RAW/$1/app.py" -o /dev/null 2>/dev/null; }
|
||||||
|
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--version|-v)
|
--version|-v|--check)
|
||||||
echo "Установлено: $CUR"
|
echo "Установлено: $CUR"
|
||||||
echo "Доступно: $(latest_version)"
|
echo "Стабильная: $(latest_version)"
|
||||||
|
beta=$(curl -fsSL -m8 "$RAW/beta/VERSION" 2>/dev/null | tr -d ' \r\n')
|
||||||
|
[ -n "$beta" ] && echo "Бета (beta): $beta"
|
||||||
|
;;
|
||||||
|
--versions|--tags)
|
||||||
|
echo "Доступные версии (git tags):"
|
||||||
|
curl -fsSL -m10 "https://api.github.com/repos/$REPO/tags?per_page=100" 2>/dev/null \
|
||||||
|
| grep -oE '"name": *"[^"]+"' | sed 's/.*"name": *"\(.*\)"/ \1/' || echo " (нет данных)"
|
||||||
;;
|
;;
|
||||||
--list)
|
--list)
|
||||||
echo "Бэкапы (для отката):"
|
echo "Бэкапы (для отката):"
|
||||||
@@ -83,6 +92,11 @@ case "${1:-}" in
|
|||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
ref="${1:-main}"; [ "$ref" = "latest" ] && ref="main"
|
ref="${1:-main}"; [ "$ref" = "latest" ] && ref="main"
|
||||||
|
if ! ref_exists "$ref"; then
|
||||||
|
echo "Версия/ветка '$ref' не найдена на GitHub. Обновление отменено (текущая $CUR не тронута)."
|
||||||
|
echo "Доступные: phobos-update --versions · стабильная: $(latest_version)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
target=$(curl -fsSL -m10 "$RAW/$ref/VERSION" 2>/dev/null | tr -d ' \r\n'); [ -z "$target" ] && target="$ref"
|
target=$(curl -fsSL -m10 "$RAW/$ref/VERSION" 2>/dev/null | tr -d ' \r\n'); [ -z "$target" ] && target="$ref"
|
||||||
echo "Текущая: $CUR -> целевая: $target ($ref)"
|
echo "Текущая: $CUR -> целевая: $target ($ref)"
|
||||||
b=$(do_backup); echo "Бэкап: $b"
|
b=$(do_backup); echo "Бэкап: $b"
|
||||||
|
|||||||
Reference in New Issue
Block a user