#!/usr/bin/env bash # ============================================================================ # OA Monitor · Resin 凌晨预防性重启 # 每天 02:00 执行,重建 push 线程池,静默死亡前主动清场 # crontab: 0 2 * * * /opt/oa-monitor/bin/resin-preventive-restart.sh >> /opt/oa-monitor/logs/resin-restart.log 2>&1 # ============================================================================ set -u SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "${SCRIPT_DIR}/../lib/common.sh" OA_HOST="39.99.236.138" OA_USER="root" OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM" LOG_FILE="${MONITOR_LOGS}/resin-preventive-restart.log" OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py" START_TS=$(date '+%Y-%m-%d %H:%M:%S') echo "=== ${START_TS} Resin 预防性重启开始 ===" | tee -a "${LOG_FILE}" # Step 1: 写 _oa_ssh.py(paramiko + errors=replace,复用 watchdog 方式) cat > "${OA_SSH_PY}" << 'PYEOF' import paramiko, sys host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: ssh.connect(host, username=user, password=password, timeout=15) stdin, stdout, stderr = ssh.exec_command(cmd, timeout=180) sys.stdout.write(stdout.read().decode("utf-8", errors="replace")) sys.stderr.write(stderr.read().decode("utf-8", errors="replace")) ssh.close() except Exception as e: print(f"SSH_ERROR: {e}", file=sys.stderr) sys.exit(1) PYEOF chmod +x "${OA_SSH_PY}" OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}" # Step 2: 钉钉提前通知 dingtalk_alert "P1" "Resin 凌晨预防性重启(开始)" \ "**🛠️ Resin 预防性重启开始**\n\n时间:${START_TS}\n\n> 每天凌晨 02:00 执行,重建推送线程池,防止静默死亡。停机窗口约 30-60 秒。" # Step 3: 记录旧 PID OLD_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null) echo "旧 PID=${OLD_PID}" | tee -a "${LOG_FILE}" # Step 4: 停 Resin echo "--- stopresin.sh ---" | tee -a "${LOG_FILE}" ${OA_SSH} "bash /usr/weaver/Resin/bin/stopresin.sh 2>&1" | tee -a "${LOG_FILE}" # Step 5: 等端口关闭(最多 30 秒) for i in $(seq 1 30); do NEW_CHECK=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null) if [[ "${NEW_CHECK}" != "200" && "${NEW_CHECK}" != "404" && "${NEW_CHECK}" != "401" ]]; then echo "[${i}s] 80 端口已关闭 (HTTP=${NEW_CHECK})" | tee -a "${LOG_FILE}" break fi sleep 1 done sleep 3 # Step 6: 起 Resin echo "--- startresin.sh ---" | tee -a "${LOG_FILE}" ${OA_SSH} "bash /usr/weaver/Resin/bin/startresin.sh 2>&1" | tee -a "${LOG_FILE}" # Step 7: 等 Resin 恢复(最多 90 秒) HTTP_CODE="000" for i in $(seq 1 90); do HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null) if [[ "${HTTP_CODE}" == "200" || "${HTTP_CODE}" == "404" || "${HTTP_CODE}" == "401" || "${HTTP_CODE}" == "500" ]]; then echo "[${i}s] Resin 恢复 (HTTP=${HTTP_CODE})" | tee -a "${LOG_FILE}" break fi sleep 1 done # Step 8: 记录新 PID NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null) END_TS=$(date '+%Y-%m-%d %H:%M:%S') # Step 9: 钉钉结果通知 BODY="**🛠️ Resin 预防性重启完成**\n\n" BODY="${BODY}| 指标 | 值 |\n|---|---|\n" BODY="${BODY}| 旧 PID | \`${OLD_PID}\` |\n" BODY="${BODY}| 新 PID | \`${NEW_PID}\` |\n" BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n" BODY="${BODY}| 耗时 | $(($(date +%s) - $(date -d "${START_TS}" +%s 2>/dev/null || date +%s)))秒 |\n" BODY="${BODY}| 结束时间 | ${END_TS} |\n" if [[ "${NEW_PID}" == "${OLD_PID}" ]]; then BODY="${BODY}\n> ⚠️ 新旧 PID 相同,重启可能未生效,请检查!" dingtalk_alert "P1" "Resin 预防性重启(⚠️ PID 未变)" "${BODY}" else dingtalk_alert "P1" "Resin 预防性重启(✅ 成功)" "${BODY}" fi echo "=== ${END_TS} Resin 预防性重启结束(${OLD_PID}→${NEW_PID}, HTTP=${HTTP_CODE}) ===" | tee -a "${LOG_FILE}"