watchdog v7: HTTP 心跳前置 + Threads_running 辅助判定,解决 Deadlock 时 info_read/message_log 同时零增长盲区
This commit is contained in:
1 parent
46ce657936
commit
875e466769
2 files changed
+47
-8
No files matched your search
+46
-7
@@ -1,8 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · Resin Watchdog (v6 - 自愈前抓 jstack 现场)
|
||||
# OA Monitor · Resin Watchdog (v7 - HTTP 心跳 + Threads_running 辅助)
|
||||
# v5: 流程驱动判定(有流程+无推送→自愈)
|
||||
# v6: 触发自愈前先 jstack 抓 Java 线程现场,保存到 DSH logs/jstack_dump/
|
||||
# v6: 触发自愈前先 jstack 抓 Java 线程现场
|
||||
# v7: 加 Resin HTTP 心跳前置 + MySQL Threads_running 辅助,解决 Deadlock 时 info_read/message_log 同时零增长盲区
|
||||
# 下次推送线程再死,根因直接可见,不再丢证据
|
||||
#
|
||||
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
|
||||
@@ -104,10 +105,39 @@ else
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# Step 2: 流程-推送联动判定(v5 核心改动)
|
||||
# Step 2: 三层判定(v7 核心改动)
|
||||
# Layer 1: Resin HTTP 心跳 → Resin 活着吗?
|
||||
# Layer 2: MySQL Threads_running → 有没有 Deadlock?
|
||||
# Layer 3: info_read vs message_log → 推送线程活着吗?
|
||||
# ============================================================================
|
||||
|
||||
# 2a. 最近 WINDOW_MIN 分钟新流程数(有没有人提流程?)
|
||||
# 2a. Resin HTTP 心跳前置检查(任何情况下 Resin 必须能响应)
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null)
|
||||
if [[ "${HTTP_CODE}" != "200" ]]; then
|
||||
log_warn "[WATCHDOG] Resin HTTP=${HTTP_CODE}, 服务不可达 → 触发自愈!"
|
||||
# === 自愈逻辑(和下方重复,保持独立)===
|
||||
JSTACK_TS=$(date +%Y%m%d_%H%M%S)
|
||||
JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt"
|
||||
log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..."
|
||||
RESIN_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
|
||||
if [[ "${RESIN_PID}" =~ ^[0-9]+$ ]]; then
|
||||
${OA_SSH} "/usr/weaver/jdk1.8.0_151/bin/jstack -l ${RESIN_PID} > ${JSTACK_FILE}" 2>/dev/null || true
|
||||
JSTACK_LINES=$(${OA_SSH} "wc -l < ${JSTACK_FILE}" 2>/dev/null | awk '{print $1}')
|
||||
log_info "[WATCHDOG] ✅ jstack 已保存: ${JSTACK_LINES:-0} 行"
|
||||
fi
|
||||
${OA_SSH} "/usr/weaver/Resin/bin/stopresin.sh" 2>/dev/null
|
||||
sleep 5
|
||||
${OA_SSH} "/usr/weaver/Resin/bin/startresin.sh" 2>/dev/null
|
||||
sleep 20
|
||||
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
|
||||
NEW_HTTP=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null)
|
||||
log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${NEW_HTTP}"
|
||||
dingtalk_alert "P0" "Resin HTTP 不可达(${HTTP_CODE}) → 自动自愈重启" "新 PID=${NEW_PID}
|
||||
HTTP=${NEW_HTTP}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 2b. 最近 WINDOW_MIN 分钟 info_read 新增(有没有流程进审批节点?)
|
||||
NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||||
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND ID NOT LIKE 'DUMMY_%';" 2>/dev/null)
|
||||
|
||||
@@ -125,10 +155,19 @@ log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log
|
||||
|
||||
# === 判定分支 ===
|
||||
|
||||
# 分支 1: 没人提流程 → 推送线程闲着很正常,不动
|
||||
if [[ ${NEW_REQS} -eq 0 ]]; then
|
||||
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min 无新流程(${NEW_REQS}), 推送线程空闲属正常, 不动"
|
||||
# 分支 1: info_read=0 AND message_log=0 → 可能真没人用,也可能 Deadlock 全卡死
|
||||
if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then
|
||||
# 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5
|
||||
TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||||
-sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}')
|
||||
|
||||
if [[ "${TR}" =~ ^[0-9]+$ && ${TR} -gt 5 ]]; then
|
||||
# Threads_running 异常高 → 可能 Deadlock 全卡死 → 触发自愈
|
||||
log_warn "[WATCHDOG] info_read=0 + message_log=0 + MySQL Threads_running=${TR} > 5 → 疑似 Deadlock 全卡死 → 自愈!"
|
||||
else
|
||||
log_info "[WATCHDOG] info_read=0, message_log=0, Threads_running=${TR:-?} 正常 → 推送线程空闲属正常, 不动"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# 分支 2: 有推送 → 推送线程活着,不动
|
||||
|
||||
@@ -1 +1 @@
|
||||
8798 207 1794013423
|
||||
23100 1422 1794278859
|
||||
Reference in new issue
Block a user