fix: 14:41 updateBizState 卡死根因修复 + watchdog v8 优化

- ALTER ecology_message_info_read ADD INDEX idx_iread_target_biz (TARGETID, USERID, BIZSTATE)
- UPDATE ecology_message_info_read SET BIZSTATE=1 WHERE BIZSTATE=0 (清 164371+3437 条积压)
- watchdog v8: Layer 4 workflow_requestbase 联动检查 + Layer 5 BIZSTATE 积压 P1 告警
- 根因: DUMMY 补 397 万 info_read 后表膨胀 20x → updateBizState 单线程全表扫追不上 → BIZSTATE=0 积压 → message_log 停写
This commit is contained in:
oa-monitor committed 2026-11-13 16:57:20 +08:00
1 parent 875e466769
commit 2e1dbb5403
2 files changed
+21 -2

No files matched your search

+20 -1
View File
@@ -153,10 +153,28 @@ fi
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}" log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}"
# === v8 新增: BIZSTATE 积压检查(推送线程慢了但没死 → 提前告警)===
BIZ_BACKLOG=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE BIZSTATE=0;" 2>/dev/null)
if [[ "${BIZ_BACKLOG}" =~ ^[0-9]+$ && ${BIZ_BACKLOG} -gt 1000 ]]; then
log_warn "[WATCHDOG] ⚠️ BIZSTATE=0 积压=${BIZ_BACKLOG}(推送线程追不上)→ P1 告警"
dingtalk_alert "P1" "info_read BIZSTATE 积压告警" "当前积压: ${BIZ_BACKLOG} 条
含义: 推送线程追不上业务产生的 info_read 速度
建议: 联系 AI 处理 UPDATE ecology_message_info_read SET BIZSTATE=1 WHERE BIZSTATE=0"
fi
# === 判定分支 === # === 判定分支 ===
# 分支 1: info_read=0 AND message_log=0 → 可能真没人用,也可能 Deadlock 全卡死 # v8 新增: 先查 workflow_requestbase 最近 20min 新流程(判断业务是否真在跑)
WF_NEW=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
# 分支 1: info_read=0 AND message_log=0
if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then
# v8 Layer 4 新增: workflow_requestbase 有新流程 → 业务在跑但两张表都停了
if [[ "${WF_NEW}" =~ ^[0-9]+$ && ${WF_NEW} -gt 0 ]]; then
log_warn "[WATCHDOG] ⚠️ info_read=0 + message_log=0 BUT workflow_requestbase 新流程=${WF_NEW} → 推送线程卡死 → 自愈!"
else
# 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5 # 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5
TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}') -sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}')
@@ -169,6 +187,7 @@ if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then
exit 0 exit 0
fi fi
fi fi
fi
# 分支 2: 有推送 → 推送线程活着,不动 # 分支 2: 有推送 → 推送线程活着,不动
if [[ ${MSG_COUNT} -gt 0 ]]; then if [[ ${MSG_COUNT} -gt 0 ]]; then
+1 -1
View File
@@ -1 +1 @@
23100 1422 1794278859 9822 963 1794560216