Files
oa-monitor/bin/resin-preventive-restart.sh
T
gsadmin 90be249a72 fix: 修复 check-alert.sh 假阳性 + collect-business.sh 窗口对齐
- check-alert.sh 新增 Step0 数据新鲜度前置检查:business_metrics 超 30min 过期 →
  先告警采集脚本挂了,跳过业务层检测,防止读旧数据乱报推送断档
- collect-business.sh msg_log_hourly 窗口从 1h 改为 15min(和 crontab 频率对齐)
- CURR_HOUR 用 10# 算术强制十进制,防 08/09 被解析成 octal
2026-10-28 18:38:13 +08:00

99 lines
4.1 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin 凌晨预防性重启
# 每天 02:00 执行,重建 push 线程池,静默死亡前主动清场
# crontab: 0 2 * * * /opt/oa-monitor/bin/resin-preventive-restart.sh >> /opt/oa-monitor/logs/resin-restart.log 2>&1
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
OA_HOST="39.99.236.138"
OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
LOG_FILE="${MONITOR_LOGS}/resin-preventive-restart.log"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
START_TS=$(date '+%Y-%m-%d %H:%M:%S')
echo "=== ${START_TS} Resin 预防性重启开始 ===" | tee -a "${LOG_FILE}"
# Step 1: 写 _oa_ssh.py(paramiko + errors=replace,复用 watchdog 方式)
cat > "${OA_SSH_PY}" << 'PYEOF'
import paramiko, sys
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=15)
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=180)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
sys.stderr.write(stderr.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
sys.exit(1)
PYEOF
chmod +x "${OA_SSH_PY}"
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
# Step 2: 钉钉提前通知
dingtalk_alert "P1" "Resin 凌晨预防性重启(开始)" \
"**🛠️ Resin 预防性重启开始**\n\n时间:${START_TS}\n\n> 每天凌晨 02:00 执行,重建推送线程池,防止静默死亡。停机窗口约 30-60 秒。"
# Step 3: 记录旧 PID
OLD_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
echo "旧 PID=${OLD_PID}" | tee -a "${LOG_FILE}"
# Step 4: 停 Resin
echo "--- stopresin.sh ---" | tee -a "${LOG_FILE}"
${OA_SSH} "bash /usr/weaver/Resin/bin/stopresin.sh 2>&1" | tee -a "${LOG_FILE}"
# Step 5: 等端口关闭(最多 30 秒)
for i in $(seq 1 30); do
NEW_CHECK=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null)
if [[ "${NEW_CHECK}" != "200" && "${NEW_CHECK}" != "404" && "${NEW_CHECK}" != "401" ]]; then
echo "[${i}s] 80 端口已关闭 (HTTP=${NEW_CHECK})" | tee -a "${LOG_FILE}"
break
fi
sleep 1
done
sleep 3
# Step 6: 起 Resin
echo "--- startresin.sh ---" | tee -a "${LOG_FILE}"
${OA_SSH} "bash /usr/weaver/Resin/bin/startresin.sh 2>&1" | tee -a "${LOG_FILE}"
# Step 7: 等 Resin 恢复(最多 90 秒)
HTTP_CODE="000"
for i in $(seq 1 90); do
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null)
if [[ "${HTTP_CODE}" == "200" || "${HTTP_CODE}" == "404" || "${HTTP_CODE}" == "401" || "${HTTP_CODE}" == "500" ]]; then
echo "[${i}s] Resin 恢复 (HTTP=${HTTP_CODE})" | tee -a "${LOG_FILE}"
break
fi
sleep 1
done
# Step 8: 记录新 PID
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
END_TS=$(date '+%Y-%m-%d %H:%M:%S')
# Step 9: 钉钉结果通知
BODY="**🛠️ Resin 预防性重启完成**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| 旧 PID | \`${OLD_PID}\` |\n"
BODY="${BODY}| 新 PID | \`${NEW_PID}\` |\n"
BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n"
BODY="${BODY}| 耗时 | $(($(date +%s) - $(date -d "${START_TS}" +%s 2>/dev/null || date +%s)))秒 |\n"
BODY="${BODY}| 结束时间 | ${END_TS} |\n"
if [[ "${NEW_PID}" == "${OLD_PID}" ]]; then
BODY="${BODY}\n> ⚠️ 新旧 PID 相同,重启可能未生效,请检查!"
dingtalk_alert "P1" "Resin 预防性重启(⚠️ PID 未变)" "${BODY}"
else
dingtalk_alert "P1" "Resin 预防性重启(✅ 成功)" "${BODY}"
fi
echo "=== ${END_TS} Resin 预防性重启结束(${OLD_PID}→${NEW_PID}, HTTP=${HTTP_CODE}) ===" | tee -a "${LOG_FILE}"