watchdog v5: 流程驱动判定 - 有流程+无推送才自愈,避免误伤无流程时段

This commit is contained in:
oa-monitor committed 2026-11-06 16:01:28 +08:00
1 parent cb08965cf4
commit e8fab900fa
2 files changed
+48 -25

No files matched your search

+47 -24
View File
@@ -1,11 +1,16 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin Watchdog (v4 - PID 变化检测 + message_log 入库判定)
# v3 盲区修复:Resin 被 WatchdogManager kill 后秒拉起,message_log 继续刷看起来"健康"
# v4 新增:每次巡检记录 Resin PID,PID 变化 -> 钉钉 P1 告警
# 健康依据(双重):
# 1. ecology_message_log 最近 15 分钟入库 count > 0(推送线程活着)
# 2. Resin PID 未变化(进程未被 kill 重启)
# OA Monitor · Resin Watchdog (v5 - 流程驱动判定)
# v4 盲区:15min 0 推送 ≠ 推送线程死了,可能真的没人提流程
# v5 核心逻辑:有新流程 BUT 无推送 → 推送线程才是真死了 → 自愈
#
# 判定链路(双重验证):
# Step 1: PID 变化检测(每次必跑,不依赖推送结果)
# Step 2: 流程-推送联动判定(核心改动)
# 2a. 最近 20min 新流程数 = 0 → 正常(没人提流程),直接退出
# 2b. 最近 20min message_log > 0 → 推送活着,直接退出
# 2c. 新流程 > 0 AND message_log = 0 → 推送线程死了 → 自愈!
#
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
# ============================================================================
set -u
@@ -17,7 +22,7 @@ OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
MSG_WINDOW_MIN=15
WINDOW_MIN=20 # 判定窗口(流程→推送有延迟,给足缓冲)
START_HOUR=8
END_HOUR=22
@@ -89,8 +94,7 @@ else
BODY="${BODY}上次 PID: ${LAST_PID}\n"
BODY="${BODY}新 PID: ${CUR_PID}\n"
BODY="${BODY}推算重启时间: ${RESTART_TIME}\n"
BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒\n\n"
BODY="${BODY}Resin 被 WatchdogManager kill 后拉起。message_log 可能仍在刷(假活跃), watchdog v3 会漏检。"
BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒"
dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}"
log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警"
@@ -102,45 +106,64 @@ else
fi
# ============================================================================
# Step 2: message_log 入库检测 (原 v3 逻辑)
# Step 2: 流程-推送联动判定(v5 核心改动)
# ============================================================================
# 2a. 最近 WINDOW_MIN 分钟新流程数(有没有人提流程?)
NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND DELETED=0;" 2>/dev/null)
# 2b. 最近 WINDOW_MIN 分钟 message_log 入库(推送线程活着吗?)
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
if [[ ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] 查询 message_log 失败, 跳过"
# 校验数据合法性
if [[ ! "${NEW_REQS}" =~ ^[0-9]+$ || ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] MySQL 查询失败(reqs=${NEW_REQS}, msg=${MSG_COUNT}), 跳过"
exit 0
fi
log_info "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 入库 ${MSG_COUNT} 条"
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}"
HOUR=$(date +%H)
# === 判定分支 ===
# 分支 1: 没人提流程 → 推送线程闲着很正常,不动
if [[ ${NEW_REQS} -eq 0 ]]; then
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min 无新流程(${NEW_REQS}), 推送线程空闲属正常, 不动"
exit 0
fi
# 分支 2: 有推送 → 推送线程活着,不动
if [[ ${MSG_COUNT} -gt 0 ]]; then
log_info "[WATCHDOG] message_log 入库正常(${MSG_COUNT} > 0), 推送健康"
log_info "[WATCHDOG] 有新流程(${NEW_REQS}) + 有推送(${MSG_COUNT}), 推送健康 ✅"
exit 0
fi
# 分支 3: 有流程 + 无推送 → 推送线程死了!
HOUR=$(date +%H)
if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), message_log 仅 ${MSG_COUNT} 条, 仅记录"
log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), 有流程(${NEW_REQS})但无推送(${MSG_COUNT}), 仅记录"
exit 0
fi
# --- 工作时间 + 入库过少 -> 触发重启 ---
log_warn "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log count=0, 推送线程可能已死, 触发 Resin 重启!"
# --- 触发自愈!---
log_warn "[WATCHDOG] ⚠️ 有新流程(${NEW_REQS})但 message_log=0 → 推送线程确认已死 → 触发 Resin 自愈重启!"
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
BODY="Resin watchdog 自动重启\n\n"
BODY="${BODY}message_log 最近 ${MSG_WINDOW_MIN} 分钟入库: ${MSG_COUNT} 条 (count > 0)\n"
BODY="Resin watchdog 自动重启(v5 流程驱动判定)\n\n"
BODY="${BODY}最近 ${WINDOW_MIN}min 新流程: ${NEW_REQS} 条\n"
BODY="${BODY}最近 ${WINDOW_MIN}min message_log: ${MSG_COUNT} 条\n"
BODY="${BODY}判定: 有流程但无推送 → 推送线程已死\n\n"
BODY="${BODY}新 Resin PID: ${NEW_PID}\n"
BODY="${BODY}HTTP 80: ${HTTP_CODE}\n"
dingtalk_alert "P0" "Resin watchdog 自动重启(message_log count=0)" "${BODY}"
BODY="${BODY}HTTP 80: ${HTTP_CODE}"
dingtalk_alert "P0" "Resin watchdog 自动重启(有流程但无推送)" "${BODY}"
if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then
echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}"
fi
log_info "[WATCHDOG] Resin 重启完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"
log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"