Files
oa-monitor/bin/resin-preventive-restart.sh
T

99 lines
4.1 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin 凌晨预防性重启
# 每天 02:00 执行,重建 push 线程池,静默死亡前主动清场
# crontab: 0 2 * * * /opt/oa-monitor/bin/resin-preventive-restart.sh >> /opt/oa-monitor/logs/resin-restart.log 2>&1
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
OA_HOST="39.99.236.138"
OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
LOG_FILE="${MONITOR_LOGS}/resin-preventive-restart.log"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
START_TS=$(date '+%Y-%m-%d %H:%M:%S')
echo "=== ${START_TS} Resin 预防性重启开始 ===" | tee -a "${LOG_FILE}"
# Step 1: 写 _oa_ssh.py(paramiko + errors=replace,复用 watchdog 方式)
cat > "${OA_SSH_PY}" << 'PYEOF'
import paramiko, sys
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=15)
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=180)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
sys.stderr.write(stderr.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
sys.exit(1)
PYEOF
chmod +x "${OA_SSH_PY}"
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
# Step 2: 钉钉提前通知
dingtalk_alert "P1" "Resin 凌晨预防性重启(开始)" \
"**🛠️ Resin 预防性重启开始**\n\n时间:${START_TS}\n\n> 每天凌晨 02:00 执行,重建推送线程池,防止静默死亡。停机窗口约 30-60 秒。"
# Step 3: 记录旧 PID
OLD_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
echo "旧 PID=${OLD_PID}" | tee -a "${LOG_FILE}"
# Step 4: 停 Resin
echo "--- stopresin.sh ---" | tee -a "${LOG_FILE}"
${OA_SSH} "bash /usr/weaver/Resin/bin/stopresin.sh 2>&1" | tee -a "${LOG_FILE}"
# Step 5: 等端口关闭(最多 30 秒)
for i in $(seq 1 30); do
NEW_CHECK=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null)
if [[ "${NEW_CHECK}" != "200" && "${NEW_CHECK}" != "404" && "${NEW_CHECK}" != "401" ]]; then
echo "[${i}s] 80 端口已关闭 (HTTP=${NEW_CHECK})" | tee -a "${LOG_FILE}"
break
fi
sleep 1
done
sleep 3
# Step 6: 起 Resin
echo "--- startresin.sh ---" | tee -a "${LOG_FILE}"
${OA_SSH} "bash /usr/weaver/Resin/bin/startresin.sh 2>&1" | tee -a "${LOG_FILE}"
# Step 7: 等 Resin 恢复(最多 90 秒)
HTTP_CODE="000"
for i in $(seq 1 90); do
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null)
if [[ "${HTTP_CODE}" == "200" || "${HTTP_CODE}" == "404" || "${HTTP_CODE}" == "401" || "${HTTP_CODE}" == "500" ]]; then
echo "[${i}s] Resin 恢复 (HTTP=${HTTP_CODE})" | tee -a "${LOG_FILE}"
break
fi
sleep 1
done
# Step 8: 记录新 PID
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
END_TS=$(date '+%Y-%m-%d %H:%M:%S')
# Step 9: 钉钉结果通知
BODY="**🛠️ Resin 预防性重启完成**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| 旧 PID | \`${OLD_PID}\` |\n"
BODY="${BODY}| 新 PID | \`${NEW_PID}\` |\n"
BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n"
BODY="${BODY}| 耗时 | $(($(date +%s) - $(date -d "${START_TS}" +%s 2>/dev/null || date +%s)))秒 |\n"
BODY="${BODY}| 结束时间 | ${END_TS} |\n"
if [[ "${NEW_PID}" == "${OLD_PID}" ]]; then
BODY="${BODY}\n> ⚠️ 新旧 PID 相同,重启可能未生效,请检查!"
dingtalk_alert "P1" "Resin 预防性重启(⚠️ PID 未变)" "${BODY}"
else
dingtalk_alert "P1" "Resin 预防性重启(✅ 成功)" "${BODY}"
fi
echo "=== ${END_TS} Resin 预防性重启结束(${OLD_PID}→${NEW_PID}, HTTP=${HTTP_CODE}) ===" | tee -a "${LOG_FILE}"