99 lines
4.1 KiB
Bash
Executable File
99 lines
4.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# ============================================================================
|
||
# OA Monitor · Resin 凌晨预防性重启
|
||
# 每天 02:00 执行,重建 push 线程池,静默死亡前主动清场
|
||
# crontab: 0 2 * * * /opt/oa-monitor/bin/resin-preventive-restart.sh >> /opt/oa-monitor/logs/resin-restart.log 2>&1
|
||
# ============================================================================
|
||
set -u
|
||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||
|
||
OA_HOST="39.99.236.138"
|
||
OA_USER="root"
|
||
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
|
||
LOG_FILE="${MONITOR_LOGS}/resin-preventive-restart.log"
|
||
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
|
||
|
||
START_TS=$(date '+%Y-%m-%d %H:%M:%S')
|
||
echo "=== ${START_TS} Resin 预防性重启开始 ===" | tee -a "${LOG_FILE}"
|
||
|
||
# Step 1: 写 _oa_ssh.py(paramiko + errors=replace,复用 watchdog 方式)
|
||
cat > "${OA_SSH_PY}" << 'PYEOF'
|
||
import paramiko, sys
|
||
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
|
||
ssh = paramiko.SSHClient()
|
||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||
try:
|
||
ssh.connect(host, username=user, password=password, timeout=15)
|
||
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=180)
|
||
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
|
||
sys.stderr.write(stderr.read().decode("utf-8", errors="replace"))
|
||
ssh.close()
|
||
except Exception as e:
|
||
print(f"SSH_ERROR: {e}", file=sys.stderr)
|
||
sys.exit(1)
|
||
PYEOF
|
||
chmod +x "${OA_SSH_PY}"
|
||
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
|
||
|
||
# Step 2: 钉钉提前通知
|
||
dingtalk_alert "P1" "Resin 凌晨预防性重启(开始)" \
|
||
"**🛠️ Resin 预防性重启开始**\n\n时间:${START_TS}\n\n> 每天凌晨 02:00 执行,重建推送线程池,防止静默死亡。停机窗口约 30-60 秒。"
|
||
|
||
# Step 3: 记录旧 PID
|
||
OLD_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
|
||
echo "旧 PID=${OLD_PID}" | tee -a "${LOG_FILE}"
|
||
|
||
# Step 4: 停 Resin
|
||
echo "--- stopresin.sh ---" | tee -a "${LOG_FILE}"
|
||
${OA_SSH} "bash /usr/weaver/Resin/bin/stopresin.sh 2>&1" | tee -a "${LOG_FILE}"
|
||
|
||
# Step 5: 等端口关闭(最多 30 秒)
|
||
for i in $(seq 1 30); do
|
||
NEW_CHECK=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null)
|
||
if [[ "${NEW_CHECK}" != "200" && "${NEW_CHECK}" != "404" && "${NEW_CHECK}" != "401" ]]; then
|
||
echo "[${i}s] 80 端口已关闭 (HTTP=${NEW_CHECK})" | tee -a "${LOG_FILE}"
|
||
break
|
||
fi
|
||
sleep 1
|
||
done
|
||
|
||
sleep 3
|
||
|
||
# Step 6: 起 Resin
|
||
echo "--- startresin.sh ---" | tee -a "${LOG_FILE}"
|
||
${OA_SSH} "bash /usr/weaver/Resin/bin/startresin.sh 2>&1" | tee -a "${LOG_FILE}"
|
||
|
||
# Step 7: 等 Resin 恢复(最多 90 秒)
|
||
HTTP_CODE="000"
|
||
for i in $(seq 1 90); do
|
||
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null)
|
||
if [[ "${HTTP_CODE}" == "200" || "${HTTP_CODE}" == "404" || "${HTTP_CODE}" == "401" || "${HTTP_CODE}" == "500" ]]; then
|
||
echo "[${i}s] Resin 恢复 (HTTP=${HTTP_CODE})" | tee -a "${LOG_FILE}"
|
||
break
|
||
fi
|
||
sleep 1
|
||
done
|
||
|
||
# Step 8: 记录新 PID
|
||
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
|
||
END_TS=$(date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
# Step 9: 钉钉结果通知
|
||
BODY="**🛠️ Resin 预防性重启完成**\n\n"
|
||
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
||
BODY="${BODY}| 旧 PID | \`${OLD_PID}\` |\n"
|
||
BODY="${BODY}| 新 PID | \`${NEW_PID}\` |\n"
|
||
BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n"
|
||
BODY="${BODY}| 耗时 | $(($(date +%s) - $(date -d "${START_TS}" +%s 2>/dev/null || date +%s)))秒 |\n"
|
||
BODY="${BODY}| 结束时间 | ${END_TS} |\n"
|
||
|
||
if [[ "${NEW_PID}" == "${OLD_PID}" ]]; then
|
||
BODY="${BODY}\n> ⚠️ 新旧 PID 相同,重启可能未生效,请检查!"
|
||
dingtalk_alert "P1" "Resin 预防性重启(⚠️ PID 未变)" "${BODY}"
|
||
else
|
||
dingtalk_alert "P1" "Resin 预防性重启(✅ 成功)" "${BODY}"
|
||
fi
|
||
|
||
echo "=== ${END_TS} Resin 预防性重启结束(${OLD_PID}→${NEW_PID}, HTTP=${HTTP_CODE}) ===" | tee -a "${LOG_FILE}"
|