#!/usr/bin/env python3
import argparse
import json
import os
import re
import shutil
import subprocess
import time
from datetime import datetime
from pathlib import Path
STATE_DIR = Path("/var/lib/wpmm-watchdog")
STATUS_FILE = STATE_DIR / "status.json"
STATE_FILE = STATE_DIR / "state.json"
COUNTERS_FILE = STATE_DIR / "counters.json"
SECURITY_FILE = STATE_DIR / "security.json"
LOG_FILE = Path("/var/log/wpmm-watchdog.log")
CONFIG_FILE = Path("/etc/wpmm-watchdog.json")
HAWK_IP = "172.96.184.87"
BAD_CONNECTION_THRESHOLD = 800
CONSECUTIVE_THRESHOLD = 3
GRACEFUL_COOLDOWN = 20 * 60
def run(command, timeout=60):
proc = subprocess.run(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True, timeout=timeout,
)
return proc.returncode, (proc.stdout + proc.stderr).strip()
def log(event, **detail):
record = {"time": datetime.now().astimezone().isoformat(timespec="seconds"), "event": event, **detail}
with LOG_FILE.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
def load_json(path, default):
try:
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return default
def save_json(path, data):
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
os.replace(temporary, path)
def service_status(name):
code, output = run(f"systemctl is-active {name}", timeout=10)
return output.splitlines()[0] if output else ("active" if code == 0 else "unknown")
def memory_metrics():
values = {}
for line in Path("/proc/meminfo").read_text().splitlines():
key, raw = line.split(":", 1)
values[key] = int(raw.strip().split()[0])
return {
"total_mb": values.get("MemTotal", 0) // 1024,
"available_mb": values.get("MemAvailable", 0) // 1024,
"swap_total_mb": values.get("SwapTotal", 0) // 1024,
"swap_free_mb": values.get("SwapFree", 0) // 1024,
"swap_used_mb": (values.get("SwapTotal", 0) - values.get("SwapFree", 0)) // 1024,
}
def socket_metrics():
code, output = run("ss -antH", timeout=20)
counts = {}
if code == 0:
for line in output.splitlines():
state = line.split(None, 1)[0] if line.split() else "UNKNOWN"
counts[state] = counts.get(state, 0) + 1
return counts
def disk_metrics():
result = {}
for label, mount in (("root", "/"), ("home", "/home")):
try:
stat = os.statvfs(mount)
total = stat.f_blocks * stat.f_frsize
available = stat.f_bavail * stat.f_frsize
inode_total = stat.f_files
inode_free = stat.f_favail
result[label + "_total_gb"] = round(total / 1073741824.0, 2)
result[label + "_used_percent"] = round((total - available) * 100.0 / total, 1) if total else None
result[label + "_inode_used_percent"] = round((inode_total - inode_free) * 100.0 / inode_total, 1) if inode_total else None
except Exception:
result[label + "_total_gb"] = None
result[label + "_used_percent"] = None
result[label + "_inode_used_percent"] = None
return result
def raw_counters():
cpu = [int(value) for value in Path("/proc/stat").read_text().splitlines()[0].split()[1:]]
cpu_total = sum(cpu)
cpu_idle = (cpu[3] if len(cpu) > 3 else 0) + (cpu[4] if len(cpu) > 4 else 0)
cpu_iowait = cpu[4] if len(cpu) > 4 else 0
rx_bytes = tx_bytes = 0
for line in Path("/proc/net/dev").read_text().splitlines()[2:]:
interface, values = line.split(":", 1)
if interface.strip() == "lo":
continue
fields = values.split()
rx_bytes += int(fields[0])
tx_bytes += int(fields[8])
read_sectors = write_sectors = 0
for device in Path("/sys/block").iterdir():
if device.name.startswith(("loop", "ram", "fd")):
continue
try:
fields = (device / "stat").read_text().split()
read_sectors += int(fields[2])
write_sectors += int(fields[6])
except Exception:
pass
return {
"time": time.time(), "cpu_total": cpu_total, "cpu_idle": cpu_idle,
"cpu_iowait": cpu_iowait, "rx_bytes": rx_bytes, "tx_bytes": tx_bytes,
"read_sectors": read_sectors, "write_sectors": write_sectors,
}
def rate_metrics():
current = raw_counters()
previous = load_json(COUNTERS_FILE, {})
save_json(COUNTERS_FILE, current)
elapsed = max(0.001, current["time"] - float(previous.get("time", current["time"])))
cpu_delta = current["cpu_total"] - int(previous.get("cpu_total", current["cpu_total"]))
idle_delta = current["cpu_idle"] - int(previous.get("cpu_idle", current["cpu_idle"]))
iowait_delta = current["cpu_iowait"] - int(previous.get("cpu_iowait", current["cpu_iowait"]))
return {
"cpu_percent": round(max(0.0, min(100.0, (cpu_delta - idle_delta) * 100.0 / cpu_delta)), 1) if cpu_delta > 0 else None,
"io_wait_percent": round(max(0.0, iowait_delta * 100.0 / cpu_delta), 1) if cpu_delta > 0 else None,
"network_rx_kbps": round(max(0, current["rx_bytes"] - int(previous.get("rx_bytes", current["rx_bytes"]))) / elapsed / 1024.0, 1),
"network_tx_kbps": round(max(0, current["tx_bytes"] - int(previous.get("tx_bytes", current["tx_bytes"]))) / elapsed / 1024.0, 1),
"disk_read_kbps": round(max(0, current["read_sectors"] - int(previous.get("read_sectors", current["read_sectors"]))) * 512.0 / elapsed / 1024.0, 1),
"disk_write_kbps": round(max(0, current["write_sectors"] - int(previous.get("write_sectors", current["write_sectors"]))) * 512.0 / elapsed / 1024.0, 1),
}
def top_processes(sort_key):
command = "ps -eo pid=,user=,pcpu=,pmem=,etimes=,comm= --sort=-{} | head -10".format(sort_key)
code, output = run(command, timeout=15)
rows = []
if code == 0:
for line in output.splitlines():
parts = line.split(None, 5)
if len(parts) == 6:
try:
rows.append({"pid": int(parts[0]), "user": parts[1], "cpu": float(parts[2]), "memory": float(parts[3]), "seconds": int(parts[4]), "process": parts[5][:80]})
except Exception:
pass
return rows
def root_security():
_, password_output = run("chage -l root 2>/dev/null | head -1", timeout=10)
password_changed = password_output.split(":", 1)[1].strip() if ":" in password_output else password_output
keys = []
key_path = Path("/root/.ssh/authorized_keys")
if key_path.is_file():
_, key_output = run("ssh-keygen -lf /root/.ssh/authorized_keys 2>/dev/null", timeout=10)
keys = [line[:240] for line in key_output.splitlines() if line.strip()]
_, login_output = run("last -n 5 -Fai root 2>/dev/null", timeout=10)
logins = [line[:300] for line in login_output.splitlines() if line.startswith("root ")][:5]
return {
"password_changed": password_changed,
"authorized_key_count": len(keys),
"authorized_key_fingerprints": keys,
"authorized_keys_mtime": datetime.fromtimestamp(key_path.stat().st_mtime).astimezone().isoformat(timespec="seconds") if key_path.exists() else "",
"recent_root_logins": logins,
}
def php_fpm_status():
return "active" if run("pgrep -x php-fpm >/dev/null || pgrep -x lsphp >/dev/null", timeout=10)[0] == 0 else "inactive"
def quick_report():
config = load_json(CONFIG_FILE, {})
sockets = socket_metrics()
memory = memory_metrics()
load1, load5, load15 = os.getloadavg()
ssh_port = int(config.get("ssh_port", 22))
_, listen_output = run(f"ss -lntH 'sport = :{ssh_port}'", timeout=10)
rates = rate_metrics()
services = {name: service_status(name) for name in ("sshd", "httpd", "mariadb", "cpanel")}
services["php_fpm"] = php_fpm_status()
_, whm_code = run("curl -k -sS -o /dev/null -w '%{http_code}' --max-time 8 https://127.0.0.1:2087/", timeout=12)
report = {
"checked_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"hostname": os.uname().nodename,
"ssh_port": ssh_port,
"ssh_listening": bool(listen_output.strip()),
"services": services,
"whm_http_status": int(whm_code) if whm_code.isdigit() else 0,
"cpu": {"cores": os.cpu_count() or 1, "usage_percent": rates["cpu_percent"], "io_wait_percent": rates["io_wait_percent"]},
"load": {"1m": round(load1, 2), "5m": round(load5, 2), "15m": round(load15, 2)},
"memory": memory,
"sockets": sockets,
"disk": dict(disk_metrics(), read_kbps=rates["disk_read_kbps"], write_kbps=rates["disk_write_kbps"]),
"network": {"rx_kbps": rates["network_rx_kbps"], "tx_kbps": rates["network_tx_kbps"]},
"top_cpu": top_processes("pcpu"),
"top_memory": top_processes("pmem"),
"root_security": root_security(),
"wordpress_security": load_json(SECURITY_FILE, {"checked_at": "", "summary": {}}),
"php_processes": int(run("pgrep -af 'php|lsphp' | wc -l", timeout=10)[1] or 0),
"zombies": int(run("ps -eo stat= | grep -c '^Z' || true", timeout=10)[1] or 0),
}
warnings = []
if report["load"]["1m"] > max(8, (os.cpu_count() or 1) * 2): warnings.append("load_high")
if rates["cpu_percent"] is not None and rates["cpu_percent"] >= 90: warnings.append("cpu_high")
if rates["io_wait_percent"] is not None and rates["io_wait_percent"] >= 30: warnings.append("io_wait_high")
if memory["available_mb"] < 1024: warnings.append("memory_low")
if memory["swap_total_mb"] and memory["swap_used_mb"] / memory["swap_total_mb"] >= 0.85: warnings.append("swap_high")
if sockets.get("SYN-RECV", 0) >= 200: warnings.append("syn_flood_suspected")
if sockets.get("LAST-ACK", 0) + sockets.get("CLOSE-WAIT", 0) >= BAD_CONNECTION_THRESHOLD: warnings.append("http_connections_stuck")
if not report["ssh_listening"]: warnings.append("ssh_not_listening")
if any(value != "active" for value in report["services"].values()): warnings.append("service_inactive")
if report["whm_http_status"] not in (200, 301, 302): warnings.append("whm_unhealthy")
if report["disk"]["root_used_percent"] is not None and report["disk"]["root_used_percent"] >= 90: warnings.append("disk_high")
report["warnings"] = warnings
report["severity"] = "critical" if any(x in warnings for x in ("memory_low", "ssh_not_listening", "service_inactive")) else ("warning" if warnings else "healthy")
return report
def discover_sites():
result = {}
path = Path("/etc/userdatadomains")
if not path.exists():
return result
priority = {"main": 3, "addon": 2, "sub": 1}
for raw in path.read_text(errors="replace").splitlines():
parts = raw.split("==")
if len(parts) < 5 or ":" not in parts[0]: continue
domain = parts[0].split(":", 1)[0].strip()
kind, root = parts[2].strip(), parts[4].strip()
if root and (root not in result or priority.get(kind, 0) > priority.get(result[root][1], 0)):
result[root] = (domain, kind)
return result
def deep_scan():
known = {"wp-log1n.php", "buy.php", "click.php", "1337.txt", "defaults.php", "networks.php"}
summary = {"sites": 0, "known_files": 0, "random_dirs": 0, "uploads_php": 0, "config_missing": 0, "core_missing": 0}
samples = []
for root, (domain, _) in discover_sites().items():
summary["sites"] += 1
path = Path(root)
finding = {"domain": domain, "known_files": [], "random_dirs": [], "uploads_php": [], "missing": []}
if not (path / "wp-config.php").is_file(): finding["missing"].append("wp-config.php"); summary["config_missing"] += 1
for name in ("wp-admin", "wp-includes", "wp-content"):
if not (path / name).is_dir(): finding["missing"].append(name)
if any(name in finding["missing"] for name in ("wp-admin", "wp-includes", "wp-content")): summary["core_missing"] += 1
if path.is_dir():
try:
for child in path.iterdir():
if child.is_file() and child.name.lower() in known: finding["known_files"].append(str(child))
if child.is_dir() and re.fullmatch(r"[0-9a-fA-F]{5,6}", child.name): finding["random_dirs"].append(str(child))
except Exception: pass
uploads = path / "wp-content" / "uploads"
sucuri_installed = (path / "wp-content" / "plugins" / "sucuri-scanner").is_dir()
if uploads.is_dir():
for current, _, files in os.walk(uploads):
for name in files:
lower = name.lower()
if ".php" in lower and lower not in {"index.php", "locoy.php"}:
candidate = Path(current) / name
try:
relative = candidate.relative_to(uploads)
if sucuri_installed and relative.parts and relative.parts[0].lower() == "sucuri" and "exit(0);" in candidate.read_text(errors="replace")[:300]:
continue
except Exception:
pass
finding["uploads_php"].append(str(candidate))
if len(finding["uploads_php"]) >= 30: break
if len(finding["uploads_php"]) >= 30: break
for key in ("known_files", "random_dirs", "uploads_php"):
summary[key] += len(finding[key])
if any(finding.values()) and (finding["known_files"] or finding["random_dirs"] or finding["uploads_php"] or finding["missing"]):
if len(samples) < 50: samples.append(finding)
return {"summary": summary, "samples": samples}
def ensure_hawk_allow():
if not shutil.which("csf"):
return "csf_not_installed"
allow_file = Path("/etc/csf/csf.allow")
if allow_file.exists() and HAWK_IP in allow_file.read_text(errors="replace"):
return "already_allowed"
code, output = run(f"csf -a {HAWK_IP} 'Allow Hawk wp-MM-System watchdog'", timeout=30)
return "allowed" if code == 0 else "failed: " + output[-300:]
def repair(emergency=False, automatic=False):
before = quick_report()
actions = []
actions.append({"hawk_allow": ensure_hawk_allow()})
ssh_port = before["ssh_port"]
if before["services"]["sshd"] != "active" or not before["ssh_listening"]:
code, check = run("sshd -t", timeout=20)
if code == 0:
actions.append({"restart_sshd": run("systemctl restart sshd", timeout=30)[0] == 0})
else:
actions.append({"restart_sshd": False, "reason": check[-300:]})
for service in ("httpd", "mariadb"):
if before["services"][service] != "active":
actions.append({"start_" + service: run(f"systemctl start {service}", timeout=60)[0] == 0})
stuck = before["sockets"].get("LAST-ACK", 0) + before["sockets"].get("CLOSE-WAIT", 0)
if stuck >= BAD_CONNECTION_THRESHOLD or emergency:
command = "/scripts/restartsrv_httpd --graceful" if Path("/scripts/restartsrv_httpd").exists() else "apachectl graceful"
code, output = run(command, timeout=90)
actions.append({"apache_graceful": code == 0, "stuck_before": stuck, "detail": output[-300:]})
if emergency:
php_command = "/scripts/restartsrv_apache_php_fpm --graceful"
if Path("/scripts/restartsrv_apache_php_fpm").exists():
code, output = run(php_command, timeout=120)
actions.append({"php_fpm_graceful": code == 0, "detail": output[-300:]})
memory = before["memory"]
if memory["swap_used_mb"] > 0 and memory["available_mb"] >= memory["swap_used_mb"] + 2048:
code, output = run("swapoff -a && swapon -a", timeout=180)
actions.append({"swap_reset": code == 0, "detail": output[-300:]})
else:
actions.append({"swap_reset": False, "reason": "可用記憶體不足，為安全起見跳過"})
time.sleep(2)
after = quick_report()
result = {"mode": "automatic" if automatic else ("emergency" if emergency else "safe"), "before": before, "actions": actions, "after": after}
log("repair", mode=result["mode"], actions=actions, before_severity=before["severity"], after_severity=after["severity"])
save_json(STATUS_FILE, after)
return result
def automatic_check():
state = load_json(STATE_FILE, {"pressure_runs": 0, "last_graceful": 0})
report = quick_report()
ensure_hawk_allow()
for service in ("sshd", "httpd", "mariadb"):
if report["services"][service] != "active":
run(f"systemctl start {service}", timeout=60)
log("service_self_heal", service=service)
if not report["ssh_listening"] and run("sshd -t", timeout=20)[0] == 0:
run("systemctl restart sshd", timeout=30)
log("ssh_listener_self_heal", port=report["ssh_port"])
stuck = report["sockets"].get("LAST-ACK", 0) + report["sockets"].get("CLOSE-WAIT", 0)
state["pressure_runs"] = state.get("pressure_runs", 0) + 1 if stuck >= BAD_CONNECTION_THRESHOLD else 0
now = int(time.time())
if state["pressure_runs"] >= CONSECUTIVE_THRESHOLD and now - state.get("last_graceful", 0) >= GRACEFUL_COOLDOWN:
command = "/scripts/restartsrv_httpd --graceful" if Path("/scripts/restartsrv_httpd").exists() else "apachectl graceful"
code, output = run(command, timeout=90)
state["last_graceful"] = now
state["pressure_runs"] = 0
log("automatic_apache_graceful", success=code == 0, stuck_connections=stuck, detail=output[-300:])
security = load_json(SECURITY_FILE, {})
if time.time() - float(security.get("checked_epoch", 0) or 0) >= 86400:
scan = deep_scan()
security = {"checked_at": datetime.now().astimezone().isoformat(timespec="seconds"), "checked_epoch": time.time(), "summary": scan.get("summary", {})}
save_json(SECURITY_FILE, security)
report["wordpress_security"] = security
save_json(STATE_FILE, state)
save_json(STATUS_FILE, report)
log("check", severity=report["severity"], warnings=report["warnings"], stuck_connections=stuck)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--report", action="store_true")
parser.add_argument("--deep", action="store_true")
parser.add_argument("--repair", action="store_true")
parser.add_argument("--emergency", action="store_true")
args = parser.parse_args()
STATE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True)
if args.repair or args.emergency:
print(json.dumps(repair(emergency=args.emergency), ensure_ascii=False, indent=2))
elif args.report:
result = quick_report()
if args.deep:
result["wordpress_scan"] = deep_scan()
security = {"checked_at": datetime.now().astimezone().isoformat(timespec="seconds"), "checked_epoch": time.time(), "summary": result["wordpress_scan"].get("summary", {})}
save_json(SECURITY_FILE, security)
result["wordpress_security"] = security
print(json.dumps(result, ensure_ascii=False, indent=2))
else:
automatic_check()
if __name__ == "__main__":
main()
