#!/usr/bin/env python3
import argparse
import json
import os
import re
import shutil
import subprocess
import time
from datetime import datetime
from pathlib import Path
STATE_DIR = Path("/var/lib/wpmm-watchdog")
STATUS_FILE = STATE_DIR / "status.json"
STATE_FILE = STATE_DIR / "state.json"
LOG_FILE = Path("/var/log/wpmm-watchdog.log")
CONFIG_FILE = Path("/etc/wpmm-watchdog.json")
HAWK_IP = "172.96.184.87"
BAD_CONNECTION_THRESHOLD = 800
CONSECUTIVE_THRESHOLD = 3
GRACEFUL_COOLDOWN = 20 * 60
def run(command, timeout=60):
proc = subprocess.run(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True, timeout=timeout,
)
return proc.returncode, (proc.stdout + proc.stderr).strip()
def log(event, **detail):
record = {"time": datetime.now().astimezone().isoformat(timespec="seconds"), "event": event, **detail}
with LOG_FILE.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
def load_json(path, default):
try:
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return default
def save_json(path, data):
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
os.replace(temporary, path)
def service_status(name):
code, output = run(f"systemctl is-active {name}", timeout=10)
return output.splitlines()[0] if output else ("active" if code == 0 else "unknown")
def memory_metrics():
values = {}
for line in Path("/proc/meminfo").read_text().splitlines():
key, raw = line.split(":", 1)
values[key] = int(raw.strip().split()[0])
return {
"total_mb": values.get("MemTotal", 0) // 1024,
"available_mb": values.get("MemAvailable", 0) // 1024,
"swap_total_mb": values.get("SwapTotal", 0) // 1024,
"swap_free_mb": values.get("SwapFree", 0) // 1024,
"swap_used_mb": (values.get("SwapTotal", 0) - values.get("SwapFree", 0)) // 1024,
}
def socket_metrics():
code, output = run("ss -antH", timeout=20)
counts = {}
if code == 0:
for line in output.splitlines():
state = line.split(None, 1)[0] if line.split() else "UNKNOWN"
counts[state] = counts.get(state, 0) + 1
return counts
def disk_metrics():
code, output = run("df -P / | tail -1", timeout=10)
parts = output.split()
return {"root_used_percent": int(parts[4].rstrip("%")) if code == 0 and len(parts) >= 5 else None}
def quick_report():
config = load_json(CONFIG_FILE, {})
sockets = socket_metrics()
memory = memory_metrics()
load1, load5, load15 = os.getloadavg()
ssh_port = int(config.get("ssh_port", 22))
_, listen_output = run(f"ss -lntH 'sport = :{ssh_port}'", timeout=10)
report = {
"checked_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"hostname": os.uname().nodename,
"ssh_port": ssh_port,
"ssh_listening": bool(listen_output.strip()),
"services": {name: service_status(name) for name in ("sshd", "httpd", "mariadb")},
"load": {"1m": round(load1, 2), "5m": round(load5, 2), "15m": round(load15, 2)},
"memory": memory,
"sockets": sockets,
"disk": disk_metrics(),
"php_processes": int(run("pgrep -af 'php|lsphp' | wc -l", timeout=10)[1] or 0),
"zombies": int(run("ps -eo stat= | grep -c '^Z' || true", timeout=10)[1] or 0),
}
warnings = []
if report["load"]["1m"] > max(8, (os.cpu_count() or 1) * 2): warnings.append("load_high")
if memory["available_mb"] < 1024: warnings.append("memory_low")
if memory["swap_total_mb"] and memory["swap_used_mb"] / memory["swap_total_mb"] >= 0.85: warnings.append("swap_high")
if sockets.get("LAST-ACK", 0) + sockets.get("CLOSE-WAIT", 0) >= BAD_CONNECTION_THRESHOLD: warnings.append("http_connections_stuck")
if not report["ssh_listening"]: warnings.append("ssh_not_listening")
if any(value != "active" for value in report["services"].values()): warnings.append("service_inactive")
if report["disk"]["root_used_percent"] is not None and report["disk"]["root_used_percent"] >= 90: warnings.append("disk_high")
report["warnings"] = warnings
report["severity"] = "critical" if any(x in warnings for x in ("memory_low", "ssh_not_listening", "service_inactive")) else ("warning" if warnings else "healthy")
return report
def discover_sites():
result = {}
path = Path("/etc/userdatadomains")
if not path.exists():
return result
priority = {"main": 3, "addon": 2, "sub": 1}
for raw in path.read_text(errors="replace").splitlines():
parts = raw.split("==")
if len(parts) < 5 or ":" not in parts[0]: continue
domain = parts[0].split(":", 1)[0].strip()
kind, root = parts[2].strip(), parts[4].strip()
if root and (root not in result or priority.get(kind, 0) > priority.get(result[root][1], 0)):
result[root] = (domain, kind)
return result
def deep_scan():
known = {"wp-log1n.php", "buy.php", "click.php", "1337.txt", "defaults.php", "networks.php"}
summary = {"sites": 0, "known_files": 0, "random_dirs": 0, "uploads_php": 0, "config_missing": 0, "core_missing": 0}
samples = []
for root, (domain, _) in discover_sites().items():
summary["sites"] += 1
path = Path(root)
finding = {"domain": domain, "known_files": [], "random_dirs": [], "uploads_php": [], "missing": []}
if not (path / "wp-config.php").is_file(): finding["missing"].append("wp-config.php"); summary["config_missing"] += 1
for name in ("wp-admin", "wp-includes", "wp-content"):
if not (path / name).is_dir(): finding["missing"].append(name)
if any(name in finding["missing"] for name in ("wp-admin", "wp-includes", "wp-content")): summary["core_missing"] += 1
if path.is_dir():
try:
for child in path.iterdir():
if child.is_file() and child.name.lower() in known: finding["known_files"].append(str(child))
if child.is_dir() and re.fullmatch(r"[0-9a-fA-F]{5,6}", child.name): finding["random_dirs"].append(str(child))
except Exception: pass
uploads = path / "wp-content" / "uploads"
sucuri_installed = (path / "wp-content" / "plugins" / "sucuri-scanner").is_dir()
if uploads.is_dir():
for current, _, files in os.walk(uploads):
for name in files:
lower = name.lower()
if ".php" in lower and lower not in {"index.php", "locoy.php"}:
candidate = Path(current) / name
try:
relative = candidate.relative_to(uploads)
if sucuri_installed and relative.parts and relative.parts[0].lower() == "sucuri" and "exit(0);" in candidate.read_text(errors="replace")[:300]:
continue
except Exception:
pass
finding["uploads_php"].append(str(candidate))
if len(finding["uploads_php"]) >= 30: break
if len(finding["uploads_php"]) >= 30: break
for key in ("known_files", "random_dirs", "uploads_php"):
summary[key] += len(finding[key])
if any(finding.values()) and (finding["known_files"] or finding["random_dirs"] or finding["uploads_php"] or finding["missing"]):
if len(samples) < 50: samples.append(finding)
return {"summary": summary, "samples": samples}
def ensure_hawk_allow():
if not shutil.which("csf"):
return "csf_not_installed"
allow_file = Path("/etc/csf/csf.allow")
if allow_file.exists() and HAWK_IP in allow_file.read_text(errors="replace"):
return "already_allowed"
code, output = run(f"csf -a {HAWK_IP} 'Allow Hawk wp-MM-System watchdog'", timeout=30)
return "allowed" if code == 0 else "failed: " + output[-300:]
def repair(emergency=False, automatic=False):
before = quick_report()
actions = []
actions.append({"hawk_allow": ensure_hawk_allow()})
ssh_port = before["ssh_port"]
if before["services"]["sshd"] != "active" or not before["ssh_listening"]:
code, check = run("sshd -t", timeout=20)
if code == 0:
actions.append({"restart_sshd": run("systemctl restart sshd", timeout=30)[0] == 0})
else:
actions.append({"restart_sshd": False, "reason": check[-300:]})
for service in ("httpd", "mariadb"):
if before["services"][service] != "active":
actions.append({"start_" + service: run(f"systemctl start {service}", timeout=60)[0] == 0})
stuck = before["sockets"].get("LAST-ACK", 0) + before["sockets"].get("CLOSE-WAIT", 0)
if stuck >= BAD_CONNECTION_THRESHOLD or emergency:
command = "/scripts/restartsrv_httpd --graceful" if Path("/scripts/restartsrv_httpd").exists() else "apachectl graceful"
code, output = run(command, timeout=90)
actions.append({"apache_graceful": code == 0, "stuck_before": stuck, "detail": output[-300:]})
if emergency:
php_command = "/scripts/restartsrv_apache_php_fpm --graceful"
if Path("/scripts/restartsrv_apache_php_fpm").exists():
code, output = run(php_command, timeout=120)
actions.append({"php_fpm_graceful": code == 0, "detail": output[-300:]})
memory = before["memory"]
if memory["swap_used_mb"] > 0 and memory["available_mb"] >= memory["swap_used_mb"] + 2048:
code, output = run("swapoff -a && swapon -a", timeout=180)
actions.append({"swap_reset": code == 0, "detail": output[-300:]})
else:
actions.append({"swap_reset": False, "reason": "可用記憶體不足，為安全起見跳過"})
time.sleep(2)
after = quick_report()
result = {"mode": "automatic" if automatic else ("emergency" if emergency else "safe"), "before": before, "actions": actions, "after": after}
log("repair", mode=result["mode"], actions=actions, before_severity=before["severity"], after_severity=after["severity"])
save_json(STATUS_FILE, after)
return result
def automatic_check():
state = load_json(STATE_FILE, {"pressure_runs": 0, "last_graceful": 0})
report = quick_report()
ensure_hawk_allow()
for service in ("sshd", "httpd", "mariadb"):
if report["services"][service] != "active":
run(f"systemctl start {service}", timeout=60)
log("service_self_heal", service=service)
if not report["ssh_listening"] and run("sshd -t", timeout=20)[0] == 0:
run("systemctl restart sshd", timeout=30)
log("ssh_listener_self_heal", port=report["ssh_port"])
stuck = report["sockets"].get("LAST-ACK", 0) + report["sockets"].get("CLOSE-WAIT", 0)
state["pressure_runs"] = state.get("pressure_runs", 0) + 1 if stuck >= BAD_CONNECTION_THRESHOLD else 0
now = int(time.time())
if state["pressure_runs"] >= CONSECUTIVE_THRESHOLD and now - state.get("last_graceful", 0) >= GRACEFUL_COOLDOWN:
command = "/scripts/restartsrv_httpd --graceful" if Path("/scripts/restartsrv_httpd").exists() else "apachectl graceful"
code, output = run(command, timeout=90)
state["last_graceful"] = now
state["pressure_runs"] = 0
log("automatic_apache_graceful", success=code == 0, stuck_connections=stuck, detail=output[-300:])
save_json(STATE_FILE, state)
save_json(STATUS_FILE, report)
log("check", severity=report["severity"], warnings=report["warnings"], stuck_connections=stuck)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--report", action="store_true")
parser.add_argument("--deep", action="store_true")
parser.add_argument("--repair", action="store_true")
parser.add_argument("--emergency", action="store_true")
args = parser.parse_args()
STATE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True)
if args.repair or args.emergency:
print(json.dumps(repair(emergency=args.emergency), ensure_ascii=False, indent=2))
elif args.report:
result = quick_report()
if args.deep: result["wordpress_scan"] = deep_scan()
print(json.dumps(result, ensure_ascii=False, indent=2))
else:
automatic_check()
if __name__ == "__main__":
main()
