diff --git a/bin/resin-watchdog.sh b/bin/resin-watchdog.sh index 47ef64f..0ac0208 100755 --- a/bin/resin-watchdog.sh +++ b/bin/resin-watchdog.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # ============================================================================ -# OA Monitor · Resin Watchdog (v6 - 自愈前抓 jstack 现场) +# OA Monitor · Resin Watchdog (v7 - HTTP 心跳 + Threads_running 辅助) # v5: 流程驱动判定(有流程+无推送→自愈) -# v6: 触发自愈前先 jstack 抓 Java 线程现场,保存到 DSH logs/jstack_dump/ +# v6: 触发自愈前先 jstack 抓 Java 线程现场 +# v7: 加 Resin HTTP 心跳前置 + MySQL Threads_running 辅助,解决 Deadlock 时 info_read/message_log 同时零增长盲区 # 下次推送线程再死,根因直接可见,不再丢证据 # # crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh @@ -104,10 +105,39 @@ else fi # ============================================================================ -# Step 2: 流程-推送联动判定(v5 核心改动) +# Step 2: 三层判定(v7 核心改动) +# Layer 1: Resin HTTP 心跳 → Resin 活着吗? +# Layer 2: MySQL Threads_running → 有没有 Deadlock? +# Layer 3: info_read vs message_log → 推送线程活着吗? # ============================================================================ -# 2a. 最近 WINDOW_MIN 分钟新流程数(有没有人提流程?) +# 2a. Resin HTTP 心跳前置检查(任何情况下 Resin 必须能响应) +HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null) +if [[ "${HTTP_CODE}" != "200" ]]; then + log_warn "[WATCHDOG] Resin HTTP=${HTTP_CODE}, 服务不可达 → 触发自愈!" + # === 自愈逻辑(和下方重复,保持独立)=== + JSTACK_TS=$(date +%Y%m%d_%H%M%S) + JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt" + log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..." + RESIN_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null) + if [[ "${RESIN_PID}" =~ ^[0-9]+$ ]]; then + ${OA_SSH} "/usr/weaver/jdk1.8.0_151/bin/jstack -l ${RESIN_PID} > ${JSTACK_FILE}" 2>/dev/null || true + JSTACK_LINES=$(${OA_SSH} "wc -l < ${JSTACK_FILE}" 2>/dev/null | awk '{print $1}') + log_info "[WATCHDOG] ✅ jstack 已保存: ${JSTACK_LINES:-0} 行" + fi + ${OA_SSH} "/usr/weaver/Resin/bin/stopresin.sh" 2>/dev/null + sleep 5 + ${OA_SSH} "/usr/weaver/Resin/bin/startresin.sh" 2>/dev/null + sleep 20 + NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null) + NEW_HTTP=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null) + log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${NEW_HTTP}" + dingtalk_alert "P0" "Resin HTTP 不可达(${HTTP_CODE}) → 自动自愈重启" "新 PID=${NEW_PID} +HTTP=${NEW_HTTP}" + exit 0 +fi + +# 2b. 最近 WINDOW_MIN 分钟 info_read 新增(有没有流程进审批节点?) NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ -sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND ID NOT LIKE 'DUMMY_%';" 2>/dev/null) @@ -125,10 +155,19 @@ log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log # === 判定分支 === -# 分支 1: 没人提流程 → 推送线程闲着很正常,不动 -if [[ ${NEW_REQS} -eq 0 ]]; then - log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min 无新流程(${NEW_REQS}), 推送线程空闲属正常, 不动" - exit 0 +# 分支 1: info_read=0 AND message_log=0 → 可能真没人用,也可能 Deadlock 全卡死 +if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then + # 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5 + TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}') + + if [[ "${TR}" =~ ^[0-9]+$ && ${TR} -gt 5 ]]; then + # Threads_running 异常高 → 可能 Deadlock 全卡死 → 触发自愈 + log_warn "[WATCHDOG] info_read=0 + message_log=0 + MySQL Threads_running=${TR} > 5 → 疑似 Deadlock 全卡死 → 自愈!" + else + log_info "[WATCHDOG] info_read=0, message_log=0, Threads_running=${TR:-?} 正常 → 推送线程空闲属正常, 不动" + exit 0 + fi fi # 分支 2: 有推送 → 推送线程活着,不动 diff --git a/state/resin_pid.last b/state/resin_pid.last index fea0674..53c4ee3 100644 --- a/state/resin_pid.last +++ b/state/resin_pid.last @@ -1 +1 @@ -8798 207 1794013423 +23100 1422 1794278859