Files
oa-monitor/bin/resin-watchdog.sh.bak.v3.202610281415

98 lines
4.3 KiB
Bash
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin Watchdog (v3 - message_log 入库增量判定)
# 健康依据:ecology_message_log 最近 15 分钟入库数
# - 工作时间(08:00~22:00) 入库 < 10 条 → 判定推送线程死 → 重启 Resin
# - 凌晨时段 入库少也不重启(避免误重启)
# 部署在 DSH,python paramiko 到 OA 执行重启
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
OA_HOST="39.99.236.138"
OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
MSG_WINDOW_MIN=15 # 统计最近 15 分钟
MSG_THRESHOLD=10 # 入库 < 10 条视为异常
START_HOUR=8 # 工作时间起始
END_HOUR=22 # 工作时间结束
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
# Step 1: 确保 python paramiko 可用
if ! python3 -c 'import paramiko' 2>/dev/null; then
MSG="[WATCHDOG] 🔴 python3 + paramiko 不可用!请执行: pip3 install paramiko"
log_error "${MSG}"
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
exit 1
fi
# Step 2: 写 _oa_ssh.py(仅重启时用)
cat > "${OA_SSH_PY}" << 'PYEOF'
import paramiko, sys
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=10)
stdin, stdout, stderr = ssh.exec_command(cmd)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
sys.exit(1)
PYEOF
chmod +x "${OA_SSH_PY}"
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
# Step 3: 查 message_log 最近 MSG_WINDOW_MIN 分钟入库数(DSH 直连 MySQL,走 pushlog_time 索引)
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
if [[ ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] ⚠️ 查询 message_log 失败(返回: ${MSG_COUNT}),跳过本次"
exit 0
fi
log_info "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 入库 ${MSG_COUNT} 条"
# Step 4: 判断
HOUR=$(date +%H)
if [[ ${MSG_COUNT} -ge ${MSG_THRESHOLD} ]]; then
log_info "[WATCHDOG] ✅ message_log 入库正常(${MSG_COUNT} 条 ≥ ${MSG_THRESHOLD} 阈值),推送健康"
exit 0
fi
# 入库 < 阈值
if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
log_warn "[WATCHDOG] ⚠️ 凌晨时段(${HOUR}时),message_log 仅 ${MSG_COUNT} 条 < ${MSG_THRESHOLD},仅记录不重启"
exit 0
fi
# --- 工作时间 + 入库过少 → 触发重启 ---
log_warn "[WATCHDOG] 🔴 最近 ${MSG_WINDOW_MIN} 分钟 message_log 仅 ${MSG_COUNT} 条 < ${MSG_THRESHOLD},触发 Resin 重启!"
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
BODY="**🔴 Resin 被 watchdog 自动重启**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| message_log 最近 ${MSG_WINDOW_MIN} 分钟入库 | **${MSG_COUNT} 条**(阈值 ${MSG_THRESHOLD})|\n"
BODY="${BODY}| 新 Resin PID | \`${NEW_PID}\` |\n"
BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n\n"
BODY="${BODY}> ⚠️ 推送线程静默死掉(message_log 不再入库),watchdog 自动恢复。"
BODY="${BODY}> 注:publishMessageList 日志可能仍在刷(假活跃),以 message_log 入库为准。"
dingtalk_alert "P0" "Resin watchdog 自动重启(message_log 入库 ${MSG_COUNT} 条 < ${MSG_THRESHOLD})" "${BODY}"
log_info "[WATCHDOG] ✅ Resin 重启完成,新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"