From e8fab900fa4468dba0a65690162c20b0fb8f25a8 Mon Sep 17 00:00:00 2001 From: oa-monitor Date: Fri, 6 Nov 2026 16:01:28 +0800 Subject: [PATCH] =?UTF-8?q?watchdog=20v5:=20=E6=B5=81=E7=A8=8B=E9=A9=B1?= =?UTF-8?q?=E5=8A=A8=E5=88=A4=E5=AE=9A=20-=20=E6=9C=89=E6=B5=81=E7=A8=8B+?= =?UTF-8?q?=E6=97=A0=E6=8E=A8=E9=80=81=E6=89=8D=E8=87=AA=E6=84=88=EF=BC=8C?= =?UTF-8?q?=E9=81=BF=E5=85=8D=E8=AF=AF=E4=BC=A4=E6=97=A0=E6=B5=81=E7=A8=8B?= =?UTF-8?q?=E6=97=B6=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/resin-watchdog.sh | 71 ++++++++++++++++++++++++++++--------------- state/resin_pid.last | 2 +- 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/bin/resin-watchdog.sh b/bin/resin-watchdog.sh index 870307b..6d793a5 100755 --- a/bin/resin-watchdog.sh +++ b/bin/resin-watchdog.sh @@ -1,11 +1,16 @@ #!/usr/bin/env bash # ============================================================================ -# OA Monitor · Resin Watchdog (v4 - PID 变化检测 + message_log 入库判定) -# v3 盲区修复:Resin 被 WatchdogManager kill 后秒拉起,message_log 继续刷看起来"健康" -# v4 新增:每次巡检记录 Resin PID,PID 变化 -> 钉钉 P1 告警 -# 健康依据(双重): -# 1. ecology_message_log 最近 15 分钟入库 count > 0(推送线程活着) -# 2. Resin PID 未变化(进程未被 kill 重启) +# OA Monitor · Resin Watchdog (v5 - 流程驱动判定) +# v4 盲区:15min 0 推送 ≠ 推送线程死了,可能真的没人提流程 +# v5 核心逻辑:有新流程 BUT 无推送 → 推送线程才是真死了 → 自愈 +# +# 判定链路(双重验证): +# Step 1: PID 变化检测(每次必跑,不依赖推送结果) +# Step 2: 流程-推送联动判定(核心改动) +# 2a. 最近 20min 新流程数 = 0 → 正常(没人提流程),直接退出 +# 2b. 最近 20min message_log > 0 → 推送活着,直接退出 +# 2c. 新流程 > 0 AND message_log = 0 → 推送线程死了 → 自愈! +# # crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh # ============================================================================ set -u @@ -17,7 +22,7 @@ OA_USER="root" OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM" OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1" -MSG_WINDOW_MIN=15 +WINDOW_MIN=20 # 判定窗口(流程→推送有延迟,给足缓冲) START_HOUR=8 END_HOUR=22 @@ -89,8 +94,7 @@ else BODY="${BODY}上次 PID: ${LAST_PID}\n" BODY="${BODY}新 PID: ${CUR_PID}\n" BODY="${BODY}推算重启时间: ${RESTART_TIME}\n" - BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒\n\n" - BODY="${BODY}Resin 被 WatchdogManager kill 后拉起。message_log 可能仍在刷(假活跃), watchdog v3 会漏检。" + BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒" dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}" log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警" @@ -102,45 +106,64 @@ else fi # ============================================================================ -# Step 2: message_log 入库检测 (原 v3 逻辑) +# Step 2: 流程-推送联动判定(v5 核心改动) # ============================================================================ + +# 2a. 最近 WINDOW_MIN 分钟新流程数(有没有人提流程?) +NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND DELETED=0;" 2>/dev/null) + +# 2b. 最近 WINDOW_MIN 分钟 message_log 入库(推送线程活着吗?) MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ - -sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null) + -sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null) -if [[ ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then - log_warn "[WATCHDOG] 查询 message_log 失败, 跳过" +# 校验数据合法性 +if [[ ! "${NEW_REQS}" =~ ^[0-9]+$ || ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then + log_warn "[WATCHDOG] MySQL 查询失败(reqs=${NEW_REQS}, msg=${MSG_COUNT}), 跳过" exit 0 fi -log_info "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 入库 ${MSG_COUNT} 条" +log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}" -HOUR=$(date +%H) +# === 判定分支 === + +# 分支 1: 没人提流程 → 推送线程闲着很正常,不动 +if [[ ${NEW_REQS} -eq 0 ]]; then + log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min 无新流程(${NEW_REQS}), 推送线程空闲属正常, 不动" + exit 0 +fi + +# 分支 2: 有推送 → 推送线程活着,不动 if [[ ${MSG_COUNT} -gt 0 ]]; then - log_info "[WATCHDOG] message_log 入库正常(${MSG_COUNT} > 0), 推送健康" + log_info "[WATCHDOG] 有新流程(${NEW_REQS}) + 有推送(${MSG_COUNT}), 推送健康 ✅" exit 0 fi +# 分支 3: 有流程 + 无推送 → 推送线程死了! +HOUR=$(date +%H) if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then - log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), message_log 仅 ${MSG_COUNT} 条, 仅记录" + log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), 有流程(${NEW_REQS})但无推送(${MSG_COUNT}), 仅记录" exit 0 fi -# --- 工作时间 + 入库过少 -> 触发重启 --- -log_warn "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log count=0, 推送线程可能已死, 触发 Resin 重启!" +# --- 触发自愈!--- +log_warn "[WATCHDOG] ⚠️ 有新流程(${NEW_REQS})但 message_log=0 → 推送线程确认已死 → 触发 Resin 自愈重启!" ${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}" sleep 20 NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1) HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null) -BODY="Resin watchdog 自动重启\n\n" -BODY="${BODY}message_log 最近 ${MSG_WINDOW_MIN} 分钟入库: ${MSG_COUNT} 条 (count > 0)\n" +BODY="Resin watchdog 自动重启(v5 流程驱动判定)\n\n" +BODY="${BODY}最近 ${WINDOW_MIN}min 新流程: ${NEW_REQS} 条\n" +BODY="${BODY}最近 ${WINDOW_MIN}min message_log: ${MSG_COUNT} 条\n" +BODY="${BODY}判定: 有流程但无推送 → 推送线程已死\n\n" BODY="${BODY}新 Resin PID: ${NEW_PID}\n" -BODY="${BODY}HTTP 80: ${HTTP_CODE}\n" -dingtalk_alert "P0" "Resin watchdog 自动重启(message_log count=0)" "${BODY}" +BODY="${BODY}HTTP 80: ${HTTP_CODE}" +dingtalk_alert "P0" "Resin watchdog 自动重启(有流程但无推送)" "${BODY}" if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}" fi -log_info "[WATCHDOG] Resin 重启完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}" +log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}" diff --git a/state/resin_pid.last b/state/resin_pid.last index fd948c5..3138eef 100644 --- a/state/resin_pid.last +++ b/state/resin_pid.last @@ -1 +1 @@ -1807 49791 1793951402 +1807 50429 1793952040