diff --git a/bin/resin-watchdog.sh b/bin/resin-watchdog.sh index 0ac0208..7b485ca 100755 --- a/bin/resin-watchdog.sh +++ b/bin/resin-watchdog.sh @@ -153,20 +153,39 @@ fi log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}" +# === v8 新增: BIZSTATE 积压检查(推送线程慢了但没死 → 提前告警)=== +BIZ_BACKLOG=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE BIZSTATE=0;" 2>/dev/null) +if [[ "${BIZ_BACKLOG}" =~ ^[0-9]+$ && ${BIZ_BACKLOG} -gt 1000 ]]; then + log_warn "[WATCHDOG] ⚠️ BIZSTATE=0 积压=${BIZ_BACKLOG}(推送线程追不上)→ P1 告警" + dingtalk_alert "P1" "info_read BIZSTATE 积压告警" "当前积压: ${BIZ_BACKLOG} 条 +含义: 推送线程追不上业务产生的 info_read 速度 +建议: 联系 AI 处理 UPDATE ecology_message_info_read SET BIZSTATE=1 WHERE BIZSTATE=0" +fi + # === 判定分支 === -# 分支 1: info_read=0 AND message_log=0 → 可能真没人用,也可能 Deadlock 全卡死 +# v8 新增: 先查 workflow_requestbase 最近 20min 新流程(判断业务是否真在跑) +WF_NEW=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null) + +# 分支 1: info_read=0 AND message_log=0 if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then - # 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5 - TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ - -sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}') - - if [[ "${TR}" =~ ^[0-9]+$ && ${TR} -gt 5 ]]; then - # Threads_running 异常高 → 可能 Deadlock 全卡死 → 触发自愈 - log_warn "[WATCHDOG] info_read=0 + message_log=0 + MySQL Threads_running=${TR} > 5 → 疑似 Deadlock 全卡死 → 自愈!" + # v8 Layer 4 新增: workflow_requestbase 有新流程 → 业务在跑但两张表都停了 + if [[ "${WF_NEW}" =~ ^[0-9]+$ && ${WF_NEW} -gt 0 ]]; then + log_warn "[WATCHDOG] ⚠️ info_read=0 + message_log=0 BUT workflow_requestbase 新流程=${WF_NEW} → 推送线程卡死 → 自愈!" else - log_info "[WATCHDOG] info_read=0, message_log=0, Threads_running=${TR:-?} 正常 → 推送线程空闲属正常, 不动" - exit 0 + # 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5 + TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}') + + if [[ "${TR}" =~ ^[0-9]+$ && ${TR} -gt 5 ]]; then + # Threads_running 异常高 → 可能 Deadlock 全卡死 → 触发自愈 + log_warn "[WATCHDOG] info_read=0 + message_log=0 + MySQL Threads_running=${TR} > 5 → 疑似 Deadlock 全卡死 → 自愈!" + else + log_info "[WATCHDOG] info_read=0, message_log=0, Threads_running=${TR:-?} 正常 → 推送线程空闲属正常, 不动" + exit 0 + fi fi fi diff --git a/state/resin_pid.last b/state/resin_pid.last index 53c4ee3..850caaa 100644 --- a/state/resin_pid.last +++ b/state/resin_pid.last @@ -1 +1 @@ -23100 1422 1794278859 +9822 963 1794560216