mirror of
https://github.com/andrey271192/PCA_Phobos.git
synced 2026-09-20 11:55:32 +00:00
feat: multi-server failover, health monitor, port hopping
- client/phobos-health.sh: router-side health monitor + auto-heal - Cron every 60s, checks WG handshake age - Auto-restart obfuscator (3min), WireGuard (5min) - Port hopping (10min), server failover (15min) - Auto-return to primary when recovered - client/failover.conf.example: multi-server config template - server/secondary-setup.sh: one-command secondary server deploy - WG + multi-port obfuscator + mini-API on :8444 - Auto-registers with main server - Panel: new "Серверы" page - Add/remove secondary servers - Health check, peer sync, fetch info - API key management - Secondary server install command Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Андрей Бобырев
247
client/phobos-health.sh
Normal file
247
client/phobos-health.sh
Normal file
@@ -0,0 +1,247 @@
|
||||
#!/bin/sh
|
||||
# ============================================================
|
||||
# Phobos Health Monitor — Auto-heal + Failover for routers
|
||||
# Runs via cron every 60 seconds on Keenetic/Entware
|
||||
#
|
||||
# Install: crontab -e → */1 * * * * /opt/etc/Phobos/phobos-health.sh
|
||||
# ============================================================
|
||||
|
||||
PHOBOS_DIR="/opt/etc/Phobos"
|
||||
CONF="$PHOBOS_DIR/failover.conf"
|
||||
STATE="$PHOBOS_DIR/state"
|
||||
LOG="$PHOBOS_DIR/health.log"
|
||||
WG_CONF="$PHOBOS_DIR/wg0.conf"
|
||||
OBF_CONF="$PHOBOS_DIR/wg-obfuscator.conf"
|
||||
LOCKFILE="/tmp/phobos-health.lock"
|
||||
|
||||
MAX_LOG_LINES=200
|
||||
HANDSHAKE_WARN=180
|
||||
HANDSHAKE_RESTART_OBF=300
|
||||
HANDSHAKE_RESTART_WG=600
|
||||
HANDSHAKE_FAILOVER=900
|
||||
|
||||
log() {
|
||||
echo "$(date '+%H:%M:%S') $1" >> "$LOG"
|
||||
# Trim log
|
||||
if [ "$(wc -l < "$LOG" 2>/dev/null)" -gt "$MAX_LOG_LINES" ]; then
|
||||
tail -n 100 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"
|
||||
fi
|
||||
}
|
||||
|
||||
# Prevent concurrent runs
|
||||
if [ -f "$LOCKFILE" ]; then
|
||||
pid=$(cat "$LOCKFILE" 2>/dev/null)
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
echo $$ > "$LOCKFILE"
|
||||
trap 'rm -f "$LOCKFILE"' EXIT
|
||||
|
||||
# Create state dir
|
||||
mkdir -p "$STATE"
|
||||
|
||||
# ── Read failover config ──
|
||||
if [ ! -f "$CONF" ]; then
|
||||
log "ERROR: no failover.conf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Parse servers: SERVER_1=ip:port1,port2,port3
|
||||
# Parse current state
|
||||
CURRENT_SERVER=$(cat "$STATE/current_server" 2>/dev/null || echo "1")
|
||||
CURRENT_PORT_IDX=$(cat "$STATE/current_port_idx" 2>/dev/null || echo "0")
|
||||
RESTART_COUNT=$(cat "$STATE/restart_count" 2>/dev/null || echo "0")
|
||||
PRIMARY_CHECK_TS=$(cat "$STATE/primary_check_ts" 2>/dev/null || echo "0")
|
||||
|
||||
# Load server list
|
||||
SERVER_COUNT=0
|
||||
idx=1
|
||||
while true; do
|
||||
val=$(grep "^SERVER_${idx}=" "$CONF" 2>/dev/null | cut -d= -f2-)
|
||||
if [ -z "$val" ]; then
|
||||
break
|
||||
fi
|
||||
eval "SERVER_${idx}_HOST=$(echo "$val" | cut -d: -f1)"
|
||||
eval "SERVER_${idx}_PORTS=$(echo "$val" | cut -d: -f2-)"
|
||||
eval "SERVER_${idx}_KEY=$(grep "^KEY_${idx}=" "$CONF" 2>/dev/null | cut -d= -f2-)"
|
||||
SERVER_COUNT=$idx
|
||||
idx=$((idx + 1))
|
||||
done
|
||||
|
||||
if [ "$SERVER_COUNT" -eq 0 ]; then
|
||||
log "ERROR: no servers in failover.conf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Get current handshake age ──
|
||||
get_handshake_age() {
|
||||
hs=$(wg show wg0 latest-handshakes 2>/dev/null | awk '{print $2}')
|
||||
if [ -z "$hs" ] || [ "$hs" = "0" ]; then
|
||||
echo "9999"
|
||||
return
|
||||
fi
|
||||
now=$(date +%s)
|
||||
age=$((now - hs))
|
||||
echo "$age"
|
||||
}
|
||||
|
||||
# ── Get port by index from comma-separated list ──
|
||||
get_port() {
|
||||
server_idx=$1
|
||||
port_idx=$2
|
||||
eval "ports=\$SERVER_${server_idx}_PORTS"
|
||||
echo "$ports" | tr ',' '\n' | sed -n "$((port_idx + 1))p"
|
||||
}
|
||||
|
||||
get_port_count() {
|
||||
server_idx=$1
|
||||
eval "ports=\$SERVER_${server_idx}_PORTS"
|
||||
echo "$ports" | tr ',' '\n' | wc -l
|
||||
}
|
||||
|
||||
# ── Switch to specific server:port ──
|
||||
switch_endpoint() {
|
||||
server_idx=$1
|
||||
port_idx=$2
|
||||
|
||||
eval "host=\$SERVER_${server_idx}_HOST"
|
||||
eval "obf_key=\$SERVER_${server_idx}_KEY"
|
||||
port=$(get_port "$server_idx" "$port_idx")
|
||||
|
||||
if [ -z "$host" ] || [ -z "$port" ]; then
|
||||
log "ERROR: invalid server $server_idx port_idx $port_idx"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "SWITCH → server $server_idx ($host:$port)"
|
||||
|
||||
# Update obfuscator config target
|
||||
if [ -f "$OBF_CONF" ]; then
|
||||
sed -i "s|^target = .*|target = ${host}:${port}|" "$OBF_CONF"
|
||||
if [ -n "$obf_key" ]; then
|
||||
sed -i "s|^key = .*|key = ${obf_key}|" "$OBF_CONF"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Restart obfuscator
|
||||
killall wg-obfuscator 2>/dev/null
|
||||
sleep 1
|
||||
wg-obfuscator --config "$OBF_CONF" &
|
||||
|
||||
# Save state
|
||||
echo "$server_idx" > "$STATE/current_server"
|
||||
echo "$port_idx" > "$STATE/current_port_idx"
|
||||
echo "0" > "$STATE/restart_count"
|
||||
}
|
||||
|
||||
# ── Try next port on current server ──
|
||||
try_next_port() {
|
||||
port_count=$(get_port_count "$CURRENT_SERVER")
|
||||
next_idx=$(( (CURRENT_PORT_IDX + 1) % port_count ))
|
||||
|
||||
if [ "$next_idx" -eq 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "PORT HOP → port idx $next_idx on server $CURRENT_SERVER"
|
||||
switch_endpoint "$CURRENT_SERVER" "$next_idx"
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Try next server ──
|
||||
try_next_server() {
|
||||
next=$((CURRENT_SERVER + 1))
|
||||
if [ "$next" -gt "$SERVER_COUNT" ]; then
|
||||
next=1
|
||||
fi
|
||||
|
||||
if [ "$next" -eq "$CURRENT_SERVER" ]; then
|
||||
log "WARN: only one server, can't failover"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "FAILOVER → server $next"
|
||||
switch_endpoint "$next" "0"
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Check if primary is back (every 5 min when on secondary) ──
|
||||
check_primary() {
|
||||
if [ "$CURRENT_SERVER" -eq 1 ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
now=$(date +%s)
|
||||
elapsed=$((now - PRIMARY_CHECK_TS))
|
||||
if [ "$elapsed" -lt 300 ]; then
|
||||
return
|
||||
fi
|
||||
echo "$now" > "$STATE/primary_check_ts"
|
||||
|
||||
eval "host=\$SERVER_1_HOST"
|
||||
port=$(get_port 1 0)
|
||||
|
||||
# Quick UDP probe — send a byte, see if port responds
|
||||
if ping -c 1 -W 2 "$host" >/dev/null 2>&1; then
|
||||
log "PRIMARY alive, switching back"
|
||||
switch_endpoint 1 0
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Main logic ──
|
||||
AGE=$(get_handshake_age)
|
||||
|
||||
if [ "$AGE" -lt "$HANDSHAKE_WARN" ]; then
|
||||
# All good, reset counters
|
||||
if [ "$RESTART_COUNT" -gt 0 ]; then
|
||||
log "OK: connection restored (age=${AGE}s)"
|
||||
echo "0" > "$STATE/restart_count"
|
||||
fi
|
||||
check_primary
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "STALE handshake: ${AGE}s (server=$CURRENT_SERVER port_idx=$CURRENT_PORT_IDX restarts=$RESTART_COUNT)"
|
||||
|
||||
if [ "$AGE" -lt "$HANDSHAKE_RESTART_OBF" ]; then
|
||||
# Restart obfuscator only
|
||||
RESTART_COUNT=$((RESTART_COUNT + 1))
|
||||
echo "$RESTART_COUNT" > "$STATE/restart_count"
|
||||
log "ACTION: restart obfuscator (attempt $RESTART_COUNT)"
|
||||
killall wg-obfuscator 2>/dev/null
|
||||
sleep 1
|
||||
wg-obfuscator --config "$OBF_CONF" &
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$AGE" -lt "$HANDSHAKE_RESTART_WG" ]; then
|
||||
# Restart WireGuard + obfuscator
|
||||
RESTART_COUNT=$((RESTART_COUNT + 1))
|
||||
echo "$RESTART_COUNT" > "$STATE/restart_count"
|
||||
log "ACTION: restart WG + obfuscator (attempt $RESTART_COUNT)"
|
||||
wg-quick down wg0 2>/dev/null
|
||||
killall wg-obfuscator 2>/dev/null
|
||||
sleep 2
|
||||
wg-quick up wg0 2>/dev/null
|
||||
sleep 1
|
||||
wg-obfuscator --config "$OBF_CONF" &
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$AGE" -lt "$HANDSHAKE_FAILOVER" ]; then
|
||||
# Try next port
|
||||
log "ACTION: try port hop"
|
||||
if ! try_next_port; then
|
||||
log "ACTION: all ports exhausted, try next server"
|
||||
try_next_server
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Full failover
|
||||
log "ACTION: failover (handshake ${AGE}s)"
|
||||
if ! try_next_server; then
|
||||
# Only one server, cycle ports
|
||||
try_next_port
|
||||
fi
|
||||
Reference in New Issue
Block a user