watchdog v7: HTTP 心跳前置 + Threads_running 辅助判定,解决 Deadlock 时 info_read/message_log 同时零增长盲区

This commit is contained in:
oa-monitor committed 2026-11-10 10:47:49 +08:00
1 parent 46ce657936
commit 875e466769
2 files changed
+47 -8

No files matched your search

+46 -7
View File
@@ -1,8 +1,9 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# ============================================================================ # ============================================================================
# OA Monitor · Resin Watchdog (v6 - 自愈前抓 jstack 现场) # OA Monitor · Resin Watchdog (v7 - HTTP 心跳 + Threads_running 辅助)
# v5: 流程驱动判定(有流程+无推送→自愈) # v5: 流程驱动判定(有流程+无推送→自愈)
# v6: 触发自愈前先 jstack 抓 Java 线程现场,保存到 DSH logs/jstack_dump/ # v6: 触发自愈前先 jstack 抓 Java 线程现场
# v7: 加 Resin HTTP 心跳前置 + MySQL Threads_running 辅助,解决 Deadlock 时 info_read/message_log 同时零增长盲区
# 下次推送线程再死,根因直接可见,不再丢证据 # 下次推送线程再死,根因直接可见,不再丢证据
# #
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh # crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
@@ -104,10 +105,39 @@ else
fi fi
# ============================================================================ # ============================================================================
# Step 2: 流程-推送联动判定(v5 核心改动) # Step 2: 三层判定(v7 核心改动)
# Layer 1: Resin HTTP 心跳 → Resin 活着吗?
# Layer 2: MySQL Threads_running → 有没有 Deadlock?
# Layer 3: info_read vs message_log → 推送线程活着吗?
# ============================================================================ # ============================================================================
# 2a. 最近 WINDOW_MIN 分钟新流程数(有没有人提流程?) # 2a. Resin HTTP 心跳前置检查(任何情况下 Resin 必须能响应)
HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null)
if [[ "${HTTP_CODE}" != "200" ]]; then
log_warn "[WATCHDOG] Resin HTTP=${HTTP_CODE}, 服务不可达 → 触发自愈!"
# === 自愈逻辑(和下方重复,保持独立)===
JSTACK_TS=$(date +%Y%m%d_%H%M%S)
JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt"
log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..."
RESIN_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
if [[ "${RESIN_PID}" =~ ^[0-9]+$ ]]; then
${OA_SSH} "/usr/weaver/jdk1.8.0_151/bin/jstack -l ${RESIN_PID} > ${JSTACK_FILE}" 2>/dev/null || true
JSTACK_LINES=$(${OA_SSH} "wc -l < ${JSTACK_FILE}" 2>/dev/null | awk '{print $1}')
log_info "[WATCHDOG] ✅ jstack 已保存: ${JSTACK_LINES:-0} 行"
fi
${OA_SSH} "/usr/weaver/Resin/bin/stopresin.sh" 2>/dev/null
sleep 5
${OA_SSH} "/usr/weaver/Resin/bin/startresin.sh" 2>/dev/null
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
NEW_HTTP=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null)
log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${NEW_HTTP}"
dingtalk_alert "P0" "Resin HTTP 不可达(${HTTP_CODE}) → 自动自愈重启" "新 PID=${NEW_PID}
HTTP=${NEW_HTTP}"
exit 0
fi
# 2b. 最近 WINDOW_MIN 分钟 info_read 新增(有没有流程进审批节点?)
NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND ID NOT LIKE 'DUMMY_%';" 2>/dev/null) -sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND ID NOT LIKE 'DUMMY_%';" 2>/dev/null)
@@ -125,10 +155,19 @@ log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log
# === 判定分支 === # === 判定分支 ===
# 分支 1: 没人提流程 → 推送线程闲着很正常,不动 # 分支 1: info_read=0 AND message_log=0 → 可能真没人用,也可能 Deadlock 全卡死
if [[ ${NEW_REQS} -eq 0 ]]; then if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min 无新流程(${NEW_REQS}), 推送线程空闲属正常, 不动" # 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5
TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}')
if [[ "${TR}" =~ ^[0-9]+$ && ${TR} -gt 5 ]]; then
# Threads_running 异常高 → 可能 Deadlock 全卡死 → 触发自愈
log_warn "[WATCHDOG] info_read=0 + message_log=0 + MySQL Threads_running=${TR} > 5 → 疑似 Deadlock 全卡死 → 自愈!"
else
log_info "[WATCHDOG] info_read=0, message_log=0, Threads_running=${TR:-?} 正常 → 推送线程空闲属正常, 不动"
exit 0 exit 0
fi
fi fi
# 分支 2: 有推送 → 推送线程活着,不动 # 分支 2: 有推送 → 推送线程活着,不动
+1 -1
View File
@@ -1 +1 @@
8798 207 1794013423 23100 1422 1794278859