PCA/PCA_Phobos
1
0
Fork 0
mirror of https://github.com/andrey271192/PCA_Phobos.git synced 2026-09-20 11:55:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

turnkey installer: full primary stack one-command + session improvements

Browse Source 
- install.sh: rewritten as self-contained turnkey primary installer (deps,
  wg-obfuscator from Ground-Zerro, wg0, obfuscator services, Phobos repo +
  PCA overlay patches, web panel, nginx, router watchdog).
- app.py: current panel (RU/EN, tunnel-pull config endpoint, fan-out, load-aware
  rebalance, online-anywhere status, '?' help).
- overlay/: patched onboarding scripts (phobos-client.sh 403 fix,
  install-router.sh.template tunnel-pull+cron+client_id, router-configure-wireguard
  public WG, phobos-pull.sh tunnel-first).
- server/: phobos-health.sh (self-heal+apply-server), phobos-pull.sh,
  phobos-router-watchdog.py, api.py (agent + /api/router-config).
This commit is contained in:
phobos
2026-05-30 13:39:49 +03:00
parent 5badb15d75
commit fff45ca342
11 changed files with 3686 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
server/api.py Normal file
View File

@@ -0,0 +1,147 @@
#!/usr/bin/env python3
"""Phobos Secondary Server API — peer management + health."""
import json, os, subprocess
from flask import Flask, request, jsonify
app = Flask(__name__)
def load_env():
env = {}
with open("/opt/Phobos/server/server.env") as f:
for line in f:
if "=" in line and not line.startswith("#"):
k, v = line.strip().split("=", 1)
env[k] = v
return env
def check_api_key():
env = load_env()
key = request.headers.get("X-API-Key", "")
return key == env.get("MAIN_API_KEY", "")
@app.route("/api/health")
def health():
try:
out = subprocess.check_output(["wg", "show", "wg0", "dump"], text=True, timeout=5)
peer_keys = []
handshakes = {}
for line in out.strip().split("\n")[1:]:
parts = line.split("\t")
if len(parts) >= 4:
pub = parts[0]
peer_keys.append(pub)
try:
handshakes[pub] = int(parts[4])
except Exception:
handshakes[pub] = 0
peers = len(peer_keys)
except Exception:
peers = 0
peer_keys = []
handshakes = {}
try:
out = subprocess.check_output("top -bn1 | grep Cpu", shell=True, text=True, timeout=5)
idle = float([x for x in out.split(",") if "id" in x][0].split()[0])
cpu = f"{round(100 - idle, 1)}%"
except Exception:
cpu = "?"
try:
mem = subprocess.check_output("free -m", shell=True, text=True).split("\n")[1].split()
mem_str = f"{mem[2]}/{mem[1]}MB"
except Exception:
mem_str = "?"
return jsonify({"status": "ok", "peers": peers, "peer_keys": peer_keys, "handshakes": handshakes, "cpu": cpu, "mem": mem_str})
@app.route("/api/peers", methods=["GET"])
def list_peers():
if not check_api_key():
return jsonify({"error": "unauthorized"}), 401
try:
out = subprocess.check_output(["wg", "show", "wg0", "allowed-ips"], text=True, timeout=5)
peers = {}
for line in out.strip().split("\n"):
if "\t" in line:
pub, ips = line.split("\t", 1)
peers[pub.strip()] = ips.strip()
return jsonify({"peers": peers})
except Exception as e:
return jsonify({"error": str(e)}), 500
@app.route("/api/peers/add", methods=["POST"])
def add_peer():
if not check_api_key():
return jsonify({"error": "unauthorized"}), 401
data = request.json
pub_key = data.get("public_key", "")
allowed_ips = data.get("allowed_ips", "")
if not pub_key or not allowed_ips:
return jsonify({"error": "missing public_key or allowed_ips"}), 400
try:
subprocess.run(["wg", "set", "wg0", "peer", pub_key, "allowed-ips", allowed_ips], check=True, timeout=5)
subprocess.run(["wg-quick", "save", "wg0"], timeout=5)
return jsonify({"status": "ok"})
except Exception as e:
return jsonify({"error": str(e)}), 500
@app.route("/api/peers/remove", methods=["POST"])
def remove_peer():
if not check_api_key():
return jsonify({"error": "unauthorized"}), 401
pub_key = request.json.get("public_key", "")
if not pub_key:
return jsonify({"error": "missing public_key"}), 400
try:
subprocess.run(["wg", "set", "wg0", "peer", pub_key, "remove"], check=True, timeout=5)
subprocess.run(["wg-quick", "save", "wg0"], timeout=5)
return jsonify({"status": "ok"})
except Exception as e:
return jsonify({"error": str(e)}), 500
@app.route("/api/info")
def info():
if not check_api_key():
return jsonify({"error": "unauthorized"}), 401
env = load_env()
return jsonify({
"ip": env.get("SERVER_PUBLIC_IP_V4"),
"wg_public_key": env.get("SERVER_WG_PUBLIC_KEY"),
"obfuscator_key": env.get("OBFUSCATOR_KEY"),
"ports": env.get("OBFUSCATOR_PORTS", "2083").split(","),
"role": "secondary"
})
ROUTER_CONFIGS_DIR = "/opt/Phobos/server/router-configs"
@app.route("/api/router-config/<client_id>")
def router_config(client_id):
"""Serve a client's failover.conf so routers can PULL via the tunnel
(http://10.25.0.1:8444/...). Token via ?token= or X-API-Key."""
token = request.args.get("token", "") or request.headers.get("X-API-Key", "")
if token != load_env().get("MAIN_API_KEY", ""):
return ("forbidden", 403)
path = os.path.join(ROUTER_CONFIGS_DIR, client_id + ".conf")
if not os.path.exists(path):
return ("not found", 404)
with open(path) as fh:
return (fh.read(), 200, {"Content-Type": "text/plain; charset=utf-8"})
@app.route("/api/router-config-set", methods=["POST"])
def router_config_set():
"""Panel fan-out: store a client's failover.conf on this server."""
if not check_api_key():
return jsonify({"status": "error", "msg": "unauthorized"}), 403
data = request.get_json(force=True, silent=True) or {}
cid = data.get("client_id", "")
conf = data.get("conf", "")
if not cid or "SERVER_1=" not in conf:
return jsonify({"status": "error", "msg": "bad payload"}), 400
os.makedirs(ROUTER_CONFIGS_DIR, exist_ok=True)
with open(os.path.join(ROUTER_CONFIGS_DIR, cid + ".conf"), "w") as fh:
fh.write(conf)
return jsonify({"status": "ok"})
if __name__ == "__main__":
app.run(host="0.0.0.0", port=8444)

446
server/phobos-health.sh Executable file
View File

@@ -0,0 +1,446 @@
#!/bin/sh
# ============================================================
# Phobos Health Monitor v2 — Keenetic Edition
# Uses ndmc/RCI instead of wg-tools. Connectivity-based failover.
# Runs via cron every 60 seconds
# ============================================================
PHOBOS_DIR="/opt/etc/Phobos"
CONF="$PHOBOS_DIR/failover.conf"
STATE="$PHOBOS_DIR/state"
LOG="$PHOBOS_DIR/health.log"
OBF_CONF="$PHOBOS_DIR/wg-obfuscator.conf"
LOCKFILE="/tmp/phobos-health.lock"
WG_IF="Wireguard3"
MAX_LOG_LINES=200
# Thresholds (seconds)
HANDSHAKE_WARN=150
HANDSHAKE_PORT_HOP=300
HANDSHAKE_SERVER_SWITCH=600
PRIMARY_CHECK_INTERVAL=300
# Primary-alive probe: servers block ICMP and busybox `nc -z` is unreliable,
# so reachability is tested with curl against the panel's obfuscator-health
# endpoint on the primary VPS. It returns http 200 ONLY when all
# wg-obfuscator-* services are active — the bare panel port stays up even
# when the tunnel path is dead, so we must NOT switch back on panel liveness
# alone (that caused premature switchback to a dead primary).
PRIMARY_PROBE_PORT=10514
PRIMARY_PROBE_PATH="/api/obf-health"
# Connectivity check targets (logged for context only)
CHECK_HOST_1="8.8.8.8"
CHECK_HOST_2="1.1.1.1"
log() {
echo "$(date '+%H:%M:%S') $1" >> "$LOG"
if [ "$(wc -l < "$LOG" 2>/dev/null)" -gt "$MAX_LOG_LINES" ]; then
tail -n 100 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"
fi
}
# Prevent concurrent runs
if [ -f "$LOCKFILE" ]; then
pid=$(cat "$LOCKFILE" 2>/dev/null)
if kill -0 "$pid" 2>/dev/null; then
exit 0
fi
fi
echo $$ > "$LOCKFILE"
trap 'rm -f "$LOCKFILE"' EXIT
mkdir -p "$STATE"
# ── Detect WG interface ──
detect_wg_interface() {
for i in 0 1 2 3 4 5 6 7 8 9; do
desc=$(ndmc -c "show interface Wireguard${i}" 2>/dev/null | grep description | head -1)
if echo "$desc" | grep -qi phobos; then
WG_IF="Wireguard${i}"
return 0
fi
done
return 1
}
# ── Read failover config ──
if [ ! -f "$CONF" ]; then
log "ERROR: no failover.conf"
exit 1
fi
SERVER_COUNT=0
idx=1
while true; do
val=$(grep "^SERVER_${idx}=" "$CONF" 2>/dev/null | cut -d= -f2-)
[ -z "$val" ] && break
eval "SERVER_${idx}_HOST=$(echo "$val" | cut -d: -f1)"
eval "SERVER_${idx}_PORTS=$(echo "$val" | cut -d: -f2-)"
eval "SERVER_${idx}_KEY=$(grep "^KEY_${idx}=" "$CONF" 2>/dev/null | cut -d= -f2-)"
eval "SERVER_${idx}_WGKEY=$(grep "^WGKEY_${idx}=" "$CONF" 2>/dev/null | cut -d= -f2-)"
SERVER_COUNT=$idx
idx=$((idx + 1))
done
if [ "$SERVER_COUNT" -eq 0 ]; then
log "ERROR: no servers in failover.conf"
exit 1
fi
# Read state
CURRENT_SERVER=$(cat "$STATE/current_server" 2>/dev/null || echo "1")
CURRENT_PORT_IDX=$(cat "$STATE/current_port_idx" 2>/dev/null || echo "0")
FAIL_COUNT=$(cat "$STATE/fail_count" 2>/dev/null || echo "0")
PRIMARY_CHECK_TS=$(cat "$STATE/primary_check_ts" 2>/dev/null || echo "0")
# ── Detect Phobos WG interface (non-fatal, fallback to Wireguard3) ──
detect_wg_interface || {
log "WARN: detect failed, using default $WG_IF"
}
# ── Get handshake age via ndmc ──
# A peer switch can leave the removed peer's stale handshake listed in ndmc
# output (often a huge sentinel like 2147483647). Take the FRESHEST (minimum
# positive, sane) handshake across all peers — that is the live tunnel.
get_handshake_age() {
hs=$(ndmc -c "show interface $WG_IF" 2>/dev/null \
| grep "last-handshake" \
| awk '{v=$2} v>0 && v<86400 {print v}' \
| sort -n | head -1)
if [ -z "$hs" ]; then
echo "9999"
else
echo "$hs"
fi
}
# ── Check real connectivity ──
check_connectivity() {
ping -c 1 -W 3 "$CHECK_HOST_1" >/dev/null 2>&1 && return 0
ping -c 1 -W 3 "$CHECK_HOST_2" >/dev/null 2>&1 && return 0
return 1
}
# ── Get port by index ──
get_port() {
server_idx=$1; port_idx=$2
eval "ports=\$SERVER_${server_idx}_PORTS"
echo "$ports" | tr ',' '\n' | sed -n "$((port_idx + 1))p"
}
get_port_count() {
server_idx=$1
eval "ports=\$SERVER_${server_idx}_PORTS"
echo "$ports" | tr ',' '\n' | wc -l
}
# ── Keys currently on the interface (includes the interface's OWN key) ──
# WG public keys are 43 base64 chars + '='. The interface's own public-key is
# also matched here — callers must only act on KNOWN server WGKEYs so the
# local key is never touched.
list_iface_keys() {
ndmc -c "show interface $WG_IF" 2>/dev/null \
| grep -oE '[A-Za-z0-9+/]{43}=' | sort -u
}
# Count how many KNOWN server WGKEYs are currently attached as peers.
count_server_peers() {
present=$(list_iface_keys)
n=0
i=1
while [ "$i" -le "$SERVER_COUNT" ]; do
eval "wk=\$SERVER_${i}_WGKEY"
if [ -n "$wk" ] && echo "$present" | grep -q "^${wk}$"; then
n=$((n + 1))
fi
i=$((i + 1))
done
echo "$n"
}
# ── Switch WG peer so EXACTLY ONE server peer (new_key) remains ──
# Failover adds a new peer; the RCI "remove" is unreliable on Keenetic and
# leaves dead peers behind. Multiple peers each with allow-ips 0.0.0.0/0 make
# egress routing ambiguous. We hard-purge every OTHER known server key via the
# ndmc CLI (reliable) — never the interface's own key — then ensure new_key.
switch_wg_peer() {
new_key=$1
if [ -z "$new_key" ]; then
log "WARN: no WGKEY for target server, skip peer switch"
return 1
fi
present=$(list_iface_keys)
# Purge any OTHER known server key that is attached
i=1
while [ "$i" -le "$SERVER_COUNT" ]; do
eval "wk=\$SERVER_${i}_WGKEY"
if [ -n "$wk" ] && [ "$wk" != "$new_key" ] && echo "$present" | grep -q "^${wk}$"; then
log "WG PEER purge: $wk"
ndmc -c "interface $WG_IF no wireguard peer $wk" >/dev/null 2>&1
fi
i=$((i + 1))
done
# Add the target peer if it is not already present
if ! echo "$present" | grep -q "^${new_key}$"; then
log "WG PEER add: $new_key"
curl -s -X POST "http://localhost:79/rci/" \
-H "Content-Type: application/json" \
-d "{\"interface\":{\"${WG_IF}\":{\"wireguard\":{\"peer\":{\"key\":\"${new_key}\",\"comment\":\"Phobos VPS Server\",\"endpoint\":{\"address\":\"127.0.0.1:13255\"},\"keepalive-interval\":{\"interval\":25},\"allow-ips\":[{\"address\":\"0.0.0.0\",\"mask\":\"0.0.0.0\"},{\"address\":\"::\",\"mask\":\"0\"}]}}}}}" >/dev/null 2>&1
fi
# Persist config
ndmc -c "system configuration save" >/dev/null 2>&1
return 0
}
# ── Switch to specific server:port ──
switch_endpoint() {
server_idx=$1
port_idx=$2
eval "host=\$SERVER_${server_idx}_HOST"
eval "obf_key=\$SERVER_${server_idx}_KEY"
eval "wg_key=\$SERVER_${server_idx}_WGKEY"
port=$(get_port "$server_idx" "$port_idx")
if [ -z "$host" ] || [ -z "$port" ]; then
log "ERROR: invalid server $server_idx port_idx $port_idx"
return 1
fi
log "SWITCH → server $server_idx ($host:$port)"
# 1. Switch WG peer key if different server
switch_wg_peer "$wg_key"
# 2. Update obfuscator config
if [ -f "$OBF_CONF" ]; then
sed -i "s|^target = .*|target = ${host}:${port}|" "$OBF_CONF"
if [ -n "$obf_key" ]; then
sed -i "s|^key = .*|key = ${obf_key}|" "$OBF_CONF"
fi
fi
# 3. Restart obfuscator
if [ -f /opt/etc/init.d/S49wg-obfuscator ]; then
/opt/etc/init.d/S49wg-obfuscator restart >/dev/null 2>&1
else
killall wg-obfuscator 2>/dev/null
sleep 1
wg-obfuscator --config "$OBF_CONF" &
fi
# 4. Save state (fail_count is managed by the caller, NOT reset here —
# resetting on a port-hop would prevent escalation to server failover)
echo "$server_idx" > "$STATE/current_server"
echo "$port_idx" > "$STATE/current_port_idx"
}
# ── Try next port on current server ──
try_next_port() {
port_count=$(get_port_count "$CURRENT_SERVER")
next_idx=$(( (CURRENT_PORT_IDX + 1) % port_count ))
[ "$next_idx" -eq 0 ] && return 1
log "PORT HOP → port idx $next_idx on server $CURRENT_SERVER"
switch_endpoint "$CURRENT_SERVER" "$next_idx"
return 0
}
# ── Try next server ──
try_next_server() {
next=$((CURRENT_SERVER + 1))
[ "$next" -gt "$SERVER_COUNT" ] && next=1
[ "$next" -eq "$CURRENT_SERVER" ] && return 1
log "FAILOVER → server $next"
switch_endpoint "$next" "0"
return 0
}
# ── Check if primary is back ──
check_primary() {
[ "$CURRENT_SERVER" -eq 1 ] && return
now=$(date +%s)
elapsed=$((now - PRIMARY_CHECK_TS))
[ "$elapsed" -lt "$PRIMARY_CHECK_INTERVAL" ] && return
echo "$now" > "$STATE/primary_check_ts"
eval "host=\$SERVER_1_HOST"
# Probe the obfuscator-health endpoint: http 200 ONLY when the primary's
# obfuscator path is actually up. Anything else (000 no-response, 503
# obf-down, redirects) means the tunnel path is NOT viable → stay put.
code=$(curl -s -m 4 -o /dev/null -w '%{http_code}' "http://${host}:${PRIMARY_PROBE_PORT}${PRIMARY_PROBE_PATH}" 2>/dev/null)
if [ "$code" = "200" ]; then
log "PRIMARY ($host obf-health http=200) alive, switching back"
switch_endpoint 1 0
echo "0" > "$STATE/fail_count"
else
log "PRIMARY still down (obf-health http=${code:-none}), staying on server $CURRENT_SERVER"
fi
}
# ── LAN routing self-heal ──────────────────────
# A router REBOOT silently drops the per-device `ip hotspot host <mac> policy`
# binding and can reset the WG interface security-level, so LAN clients leak to
# WAN (no VPN). This:
# 1) ensures WG_IF security-level = public (needed for masquerade),
# 2) LEARNS current host->policy bindings into state/lan-hosts (append-only),
# 3) RE-APPLIES any learned binding that is currently missing.
# Append-only learning means a reboot (which clears live bindings) never erases
# the record, so the next run restores them. ndmc show running-config is heavy,
# so the caller gates this to run only every few minutes.
heal_lan_routing() {
LANHOSTS="$STATE/lan-hosts"
rc=$(ndmc -c "show running-config" 2>/dev/null)
[ -z "$rc" ] && return 0
# 1) WG interface must be public
sl=$(echo "$rc" | awk -v ifc="interface $WG_IF" '$0~ifc{f=1} f&&/security-level/{print $2; exit}')
if [ -n "$sl" ] && [ "$sl" != "public" ]; then
ndmc -c "interface $WG_IF security-level public" >/dev/null 2>&1
ndmc -c "system configuration save" >/dev/null 2>&1
log "HEAL: $WG_IF security-level -> public"
fi
# 2) learn live bindings (append-only) into lan-hosts
touch "$LANHOSTS"
echo "$rc" | grep -oE "host [0-9a-f:]+ policy [A-Za-z0-9_]+" | while read -r _h mac _p pol; do
grep -q "^$mac " "$LANHOSTS" 2>/dev/null || echo "$mac $pol" >> "$LANHOSTS"
done
# 3) re-apply any learned binding that is missing live
changed=0
while read -r mac pol; do
[ -z "$mac" ] && continue
case "$mac" in \#*) continue;; esac
if ! echo "$rc" | grep -q "host $mac policy $pol"; then
ndmc -c "ip hotspot host $mac policy $pol" >/dev/null 2>&1
log "HEAL: re-bound host $mac -> $pol"
changed=1
fi
done < "$LANHOSTS"
[ "$changed" = 1 ] && ndmc -c "system configuration save" >/dev/null 2>&1
}
# ══════════════════════════════════════════════
# Explicit apply hook (used by phobos-pull.sh after a config change).
# `phobos-health.sh apply-server N` re-points the tunnel to SERVER_N
# immediately, skipping the failure-escalation logic. Reuses
# switch_endpoint so the WG-peer/obfuscator/state changes stay identical
# to a normal failover. Resets fail_count so the new server starts clean.
# ══════════════════════════════════════════════
if [ "$1" = "apply-server" ]; then
idx="${2:-1}"
eval "ahost=\$SERVER_${idx}_HOST"
if [ -z "$ahost" ]; then
log "APPLY: server $idx not in conf, ignore"
exit 1
fi
log "APPLY: pull requested server $idx ($ahost)"
switch_endpoint "$idx" "0"
echo "0" > "$STATE/fail_count"
echo "$idx" > "$STATE/current_server"
exit 0
fi
# ══════════════════════════════════════════════
# Pre-check: fix desync (obfuscator targeting wrong server)
# ══════════════════════════════════════════════
if [ -f "$OBF_CONF" ]; then
cur_target=$(grep "^target = " "$OBF_CONF" 2>/dev/null | sed 's/target = //')
eval "expected_host=\$SERVER_${CURRENT_SERVER}_HOST"
if [ -n "$cur_target" ] && [ -n "$expected_host" ]; then
echo "$cur_target" | grep -q "$expected_host" || {
log "DESYNC: obf=$cur_target state=server${CURRENT_SERVER}($expected_host). Resync."
switch_endpoint "$CURRENT_SERVER" "$CURRENT_PORT_IDX"
sleep 5
}
fi
fi
# ── LAN routing self-heal (gated ~5 min; a reboot drops host->policy bindings) ──
HEAL_TS=$(cat "$STATE/heal_ts" 2>/dev/null || echo 0)
now_heal=$(date +%s)
if [ $((now_heal - HEAL_TS)) -ge 300 ]; then
echo "$now_heal" > "$STATE/heal_ts"
heal_lan_routing
fi
# ── Peer hygiene: keep exactly ONE WG peer = current server's key ──
# Failovers can leave dead/duplicate peers; multiple 0.0.0.0/0 peers cause
# ambiguous egress routing. Self-heal here every run (cheap when already clean).
eval "cur_wgkey=\$SERVER_${CURRENT_SERVER}_WGKEY"
if [ -n "$cur_wgkey" ]; then
peer_n=$(count_server_peers)
if [ "$peer_n" -gt 1 ]; then
log "HYGIENE: $peer_n server peers present, purging to server${CURRENT_SERVER}"
switch_wg_peer "$cur_wgkey"
fi
fi
# ══════════════════════════════════════════════
# Main logic
# Tunnel health = WireGuard handshake age (authoritative: a fresh
# handshake only happens through the full obfuscator→server→WG path).
# WAN ping is logged for context ONLY — it does NOT gate decisions,
# because the router's default route is not the tunnel, so WAN can be
# up while the tunnel is dead (and a WAN blip must not cause failover).
# ══════════════════════════════════════════════
AGE=$(get_handshake_age)
WAN=$(check_connectivity && echo "yes" || echo "no")
# Tunnel healthy = fresh handshake
if [ "$AGE" -lt "$HANDSHAKE_WARN" ]; then
if [ "$FAIL_COUNT" -gt 0 ]; then
log "OK: tunnel up (handshake=${AGE}s, server=$CURRENT_SERVER, wan=$WAN)"
echo "0" > "$STATE/fail_count"
fi
check_primary
exit 0
fi
# Stale handshake = tunnel down → escalate.
# fail_count is MONOTONIC: it climbs across stages and is only reset by the
# OK branch (real recovery) or after a successful failover to a NEW server
# (so the new server gets a fresh restart→port-hop→failover cycle).
FAIL_COUNT=$((FAIL_COUNT + 1))
echo "$FAIL_COUNT" > "$STATE/fail_count"
log "STALE: handshake=${AGE}s wan=${WAN} server=$CURRENT_SERVER port=$CURRENT_PORT_IDX fails=$FAIL_COUNT"
# Stage 1 (fail 1): Restart obfuscator
if [ "$FAIL_COUNT" -eq 1 ]; then
log "ACTION: restart obfuscator"
if [ -f /opt/etc/init.d/S49wg-obfuscator ]; then
/opt/etc/init.d/S49wg-obfuscator restart >/dev/null 2>&1
else
killall wg-obfuscator 2>/dev/null
sleep 1
wg-obfuscator --config "$OBF_CONF" &
fi
exit 0
fi
# Stage 2 (fail 2): Port hop to an alternate port on the SAME server
if [ "$FAIL_COUNT" -eq 2 ]; then
if try_next_port; then
exit 0
fi
# only one port → fall through to server failover
log "single port, escalate to failover"
fi
# Stage 3 (fail 3+): Server failover. Reset fail_count so the new server
# gets its own restart→port-hop→failover cycle next ticks.
log "ACTION: server failover (fail $FAIL_COUNT)"
if try_next_server; then
echo "0" > "$STATE/fail_count"
else
# Only one server — cycle back to port 0 and restart escalation
echo "0" > "$STATE/fail_count"
switch_endpoint "$CURRENT_SERVER" "0"
fi

93
server/phobos-pull.sh Executable file
View File

@@ -0,0 +1,93 @@
#!/bin/sh
# ============================================================
# Phobos Config Pull — NAT-friendly management channel.
# Runs ON the router via cron (every 2 min). OUTBOUND ONLY:
# fetches this client's failover.conf from the panel over
# HTTPS/HTTP and applies it. No inbound SSH needed, so it works
# behind any NAT and regardless of which Phobos server is active.
#
# Why pull (not panel->router SSH):
# - routers sit behind NAT with no public IP (KeenDNS is HTTP-
# only, no port 22),
# - WG3 must be security-level PUBLIC for LAN client routing,
# which blocks inbound SSH on the tunnel,
# - on failover the router's 10.25.0.2 moves to another server's
# wg0, so a fixed panel host cannot reach it.
# Pulling sidesteps all three.
#
# Files (written by installer / bootstrap):
# /opt/etc/Phobos/client_id -> this router's client id (e.g. home)
# /opt/etc/Phobos/pull_token -> shared secret for the panel endpoint
# PANEL env or default below -> panel base URL
# ============================================================
PHOBOS_DIR="/opt/etc/Phobos"
CONF="$PHOBOS_DIR/failover.conf"
HEALTH="$PHOBOS_DIR/phobos-health.sh"
LOG="$PHOBOS_DIR/health.log"
# Config sources, tried in order. TUNNEL FIRST: 10.25.0.1 is the wg0 of whatever
# server the tunnel currently terminates on, so management rides the same
# obfuscated channel as data — survives a public-IP ban and follows failover.
# - 10.25.0.1:8444 -> secondary server agent (phobos-api)
# - 10.25.0.1:10514 -> primary server panel
# - public panel IP -> bootstrap / if tunnel is down
PANEL="${PANEL:-http://212.118.52.193:10514}"
PANEL_URLS="${PANEL_URLS:-http://10.25.0.1:8444 http://10.25.0.1:10514 $PANEL}"
CLIENT_ID=$(cat "$PHOBOS_DIR/client_id" 2>/dev/null || echo "home")
TOKEN=$(cat "$PHOBOS_DIR/pull_token" 2>/dev/null)
TMP="/tmp/failover.conf.pull"
LOCK="/tmp/phobos-pull.lock"
log() { echo "$(date '+%H:%M:%S') $1" >> "$LOG"; }
# single instance
if [ -f "$LOCK" ]; then
pid=$(cat "$LOCK" 2>/dev/null)
kill -0 "$pid" 2>/dev/null && exit 0
fi
echo $$ > "$LOCK"
trap 'rm -f "$LOCK"' EXIT
[ -z "$TOKEN" ] && exit 0
# One fetch + compare + apply pass. Returns 0 always (best-effort).
do_pull() {
# Try each source (tunnel first); accept first that returns a valid conf.
got=0
for base in $PANEL_URLS; do
curl -s -m 8 -o "$TMP" "${base}/api/router-config/${CLIENT_ID}?token=${TOKEN}" 2>/dev/null || continue
if grep -q "^SERVER_1=" "$TMP" 2>/dev/null; then got=1; break; fi
done
[ "$got" = 1 ] || { rm -f "$TMP"; return 0; }
new=$(md5sum "$TMP" 2>/dev/null | cut -d' ' -f1)
old=$(md5sum "$CONF" 2>/dev/null | cut -d' ' -f1)
if [ "$new" = "$old" ]; then
rm -f "$TMP"
return 0
fi
old_s1=$(grep "^SERVER_1=" "$CONF" 2>/dev/null | cut -d= -f2- | cut -d: -f1)
new_s1=$(grep "^SERVER_1=" "$TMP" 2>/dev/null | cut -d= -f2- | cut -d: -f1)
cp "$CONF" "$CONF.prev" 2>/dev/null
mv "$TMP" "$CONF"
log "PULL: failover.conf updated (SERVER_1 ${old_s1:-?} -> ${new_s1:-?})"
# apply the new primary now (only if it actually changed)
if [ "$old_s1" != "$new_s1" ] && [ -f "$HEALTH" ]; then
sh "$HEALTH" apply-server 1
fi
}
# Inner loop: cron fires this every 60s, but we poll ~4x per minute so a
# panel "Set" applies within ~12-15s instead of up to a full minute. The
# loop stays UNDER 60s and exits so the next cron tick takes over cleanly
# (the lockfile blocks any overlap). POLL_INTERVAL/POLL_PASSES overridable.
POLL_INTERVAL="${POLL_INTERVAL:-12}"
POLL_PASSES="${POLL_PASSES:-4}"
i=1
while [ "$i" -le "$POLL_PASSES" ]; do
do_pull
[ "$i" -lt "$POLL_PASSES" ] && sleep "$POLL_INTERVAL"
i=$((i + 1))
done

203
server/phobos-router-watchdog.py Normal file
View File

@@ -0,0 +1,203 @@
#!/usr/bin/env python3
"""
Phobos Router Watchdog (server-side).
Problem it solves: on some Keenetic firmware (seen on 5.1 Beta), after a reboot
the Entware /opt disk mounts but the init hook (rc.unslung) does NOT run, so the
obfuscator + cron + dropbear never start and the router's Phobos tunnel stays
down. Because cron itself didn't start, the on-router self-heal can't help.
This watchdog runs on the primary server (cron, every few minutes). For each
router that has KeenDNS web access configured, it checks whether the client has
a fresh WG handshake on ANY server. If a router has been offline past a grace
period, it logs into the router's web UI over KeenDNS (ndm challenge auth) and
re-triggers the opkg init (which runs rc.unslung -> starts everything). Sends a
Telegram note on down / recovery / action.
Per-router config lives in /opt/phobos-panel/settings.json under
router_access[<client_id>]:
keendns_host : e.g. "homesmart.netcraze.pro"
web_login : Keenetic web user
web_pass : Keenetic web password
opkg_disk : opkg disk id, e.g. "EXT4-XXXX:/" (default below)
Routers without these fields are skipped (watchdog is opt-in per router).
"""
import json, os, time, ssl, hashlib, http.cookiejar, urllib.request, urllib.error, subprocess
SETTINGS = "/opt/phobos-panel/settings.json"
SERVERS_FILE = "/opt/phobos-panel/servers.json"
CLIENTS_DIR = "/opt/Phobos/clients"
STATE_FILE = "/opt/Phobos/server/watchdog-state.json"
LOG = "/opt/Phobos/server/watchdog.log"
OFFLINE_SECS = int(os.environ.get("WD_OFFLINE_SECS", "300")) # offline if newest handshake older than this
COOLDOWN = int(os.environ.get("WD_COOLDOWN", "600")) # min seconds between recovery attempts per router
DEFAULT_DISK = "EXT4-V88axM0d:/"
def log(msg):
try:
with open(LOG, "a") as f:
f.write(time.strftime("%Y-%m-%d %H:%M:%S ") + msg + "\n")
except Exception:
pass
def load(path, default):
try:
with open(path) as f:
return json.load(f)
except Exception:
return default
def tg(token, chat, text):
if not token or not chat:
return
try:
url = f"https://api.telegram.org/bot{token}/sendMessage"
data = json.dumps({"chat_id": chat, "text": text}).encode()
urllib.request.urlopen(urllib.request.Request(
url, data=data, headers={"Content-Type": "application/json"}), timeout=8)
except Exception:
pass
def client_pub(cid):
try:
return json.load(open(f"{CLIENTS_DIR}/{cid}/metadata.json")).get("public_key", "")
except Exception:
return ""
def newest_handshake_age(pub, servers):
"""Smallest handshake age (s) for pub across local wg0 + secondary agents."""
best = 99999
try:
out = subprocess.check_output(["wg", "show", "wg0", "dump"], text=True, timeout=5)
for ln in out.strip().split("\n")[1:]:
f = ln.split("\t")
if f and f[0] == pub and len(f) >= 5 and f[4].isdigit() and int(f[4]) > 0:
best = min(best, int(time.time()) - int(f[4]))
except Exception:
pass
for srv in servers:
try:
req = urllib.request.Request(f"http://{srv['ip']}:8444/api/health",
headers={"X-API-Key": srv.get("api_key", "")})
d = json.loads(urllib.request.urlopen(req, timeout=5).read())
ts = d.get("handshakes", {}).get(pub, 0)
if ts:
best = min(best, int(time.time()) - int(ts))
except Exception:
pass
return best
def rci_session(host, login, pw):
"""Keenetic ndm challenge auth over KeenDNS. Returns (opener, base) or (None, None)."""
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
cj = http.cookiejar.CookieJar()
op = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj),
urllib.request.HTTPSHandler(context=ctx))
base = f"https://{host}"
realm = chal = None
try:
op.open(base + "/auth", timeout=10)
except urllib.error.HTTPError as e:
realm = e.headers.get("X-NDM-Realm")
chal = e.headers.get("X-NDM-Challenge")
except Exception:
return None, None
if not realm or not chal:
return None, None
md5 = hashlib.md5(f"{login}:{realm}:{pw}".encode()).hexdigest()
sha = hashlib.sha256((chal + md5).encode()).hexdigest()
body = json.dumps({"login": login, "password": sha}).encode()
try:
op.open(urllib.request.Request(base + "/auth", data=body,
headers={"Content-Type": "application/json"}, method="POST"), timeout=10)
except Exception:
return None, None
return op, base
def retrigger_opkg(op, base, disk):
"""Force opkg 'disk changed' so Keenetic re-runs initrc (rc.unslung)."""
cur = ""
try:
cur = json.loads(op.open(base + "/rci/show/rc/opkg", timeout=8).read()).get("disk", {}).get("disk", "")
except Exception:
pass
newdisk = disk
if cur.strip() == disk.strip():
newdisk = disk.rstrip("/") if disk.endswith("/") else disk + "/"
body = json.dumps([{"opkg": {"disk": newdisk}},
{"system": {"configuration": {"save": {}}}}]).encode()
try:
op.open(urllib.request.Request(base + "/rci/", data=body,
headers={"Content-Type": "application/json"}, method="POST"), timeout=20)
return True
except Exception:
return False
def main():
s = load(SETTINGS, {})
st = load(STATE_FILE, {})
token = s.get("tg_bot_token")
chat = s.get("tg_chat_id")
servers = load(SERVERS_FILE, [])
ra = s.get("router_access", {})
now = int(time.time())
changed = False
for cid, acc in ra.items():
host = acc.get("keendns_host")
login = acc.get("web_login")
pw = acc.get("web_pass")
disk = acc.get("opkg_disk", DEFAULT_DISK)
if not (host and login and pw):
continue
pub = client_pub(cid)
if not pub:
continue
age = newest_handshake_age(pub, servers)
rec = st.get(cid, {})
if age <= OFFLINE_SECS:
if rec.get("offline"):
log(f"{cid}: recovered (handshake {age}s)")
tg(token, chat, f"✅ Router {cid} recovered (handshake {age}s).")
st[cid] = {"offline": False, "last_recover": rec.get("last_recover", 0)}
changed = True
continue
# offline
if now - rec.get("last_recover", 0) < COOLDOWN:
continue
op, base = rci_session(host, login, pw)
if not op:
if not rec.get("offline"):
log(f"{cid}: offline, web unreachable")
tg(token, chat, f"\U0001F534 Router {cid} OFFLINE, web unreachable (powered off / no internet?).")
st[cid] = {"offline": True, "last_recover": rec.get("last_recover", 0)}
changed = True
continue
ok = retrigger_opkg(op, base, disk)
log(f"{cid}: offline ({age}s), re-triggered opkg via RCI -> {'ok' if ok else 'FAIL'}")
tg(token, chat, f"\U0001F6E0 Router {cid} Entware down (reboot didn't autostart) — re-triggered via RCI ({'ok' if ok else 'FAILED'}).")
st[cid] = {"offline": True, "last_recover": now}
changed = True
if changed:
try:
json.dump(st, open(STATE_FILE, "w"))
except Exception:
pass
if __name__ == "__main__":
main()

0
server/secondary-setup.sh Normal file → Executable file
View File