fix: 14:41 updateBizState 卡死根因修复 + watchdog v8 优化
- ALTER ecology_message_info_read ADD INDEX idx_iread_target_biz (TARGETID, USERID, BIZSTATE) - UPDATE ecology_message_info_read SET BIZSTATE=1 WHERE BIZSTATE=0 (清 164371+3437 条积压) - watchdog v8: Layer 4 workflow_requestbase 联动检查 + Layer 5 BIZSTATE 积压 P1 告警 - 根因: DUMMY 补 397 万 info_read 后表膨胀 20x → updateBizState 单线程全表扫追不上 → BIZSTATE=0 积压 → message_log 停写
This commit is contained in:
1 parent
875e466769
commit
2e1dbb5403
2 files changed
+21
-2
No files matched your search
+20
-1
@@ -153,10 +153,28 @@ fi
|
|||||||
|
|
||||||
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}"
|
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}"
|
||||||
|
|
||||||
|
# === v8 新增: BIZSTATE 积压检查(推送线程慢了但没死 → 提前告警)===
|
||||||
|
BIZ_BACKLOG=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||||||
|
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE BIZSTATE=0;" 2>/dev/null)
|
||||||
|
if [[ "${BIZ_BACKLOG}" =~ ^[0-9]+$ && ${BIZ_BACKLOG} -gt 1000 ]]; then
|
||||||
|
log_warn "[WATCHDOG] ⚠️ BIZSTATE=0 积压=${BIZ_BACKLOG}(推送线程追不上)→ P1 告警"
|
||||||
|
dingtalk_alert "P1" "info_read BIZSTATE 积压告警" "当前积压: ${BIZ_BACKLOG} 条
|
||||||
|
含义: 推送线程追不上业务产生的 info_read 速度
|
||||||
|
建议: 联系 AI 处理 UPDATE ecology_message_info_read SET BIZSTATE=1 WHERE BIZSTATE=0"
|
||||||
|
fi
|
||||||
|
|
||||||
# === 判定分支 ===
|
# === 判定分支 ===
|
||||||
|
|
||||||
# 分支 1: info_read=0 AND message_log=0 → 可能真没人用,也可能 Deadlock 全卡死
|
# v8 新增: 先查 workflow_requestbase 最近 20min 新流程(判断业务是否真在跑)
|
||||||
|
WF_NEW=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||||||
|
-sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
|
||||||
|
|
||||||
|
# 分支 1: info_read=0 AND message_log=0
|
||||||
if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then
|
if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then
|
||||||
|
# v8 Layer 4 新增: workflow_requestbase 有新流程 → 业务在跑但两张表都停了
|
||||||
|
if [[ "${WF_NEW}" =~ ^[0-9]+$ && ${WF_NEW} -gt 0 ]]; then
|
||||||
|
log_warn "[WATCHDOG] ⚠️ info_read=0 + message_log=0 BUT workflow_requestbase 新流程=${WF_NEW} → 推送线程卡死 → 自愈!"
|
||||||
|
else
|
||||||
# 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5
|
# 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5
|
||||||
TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||||||
-sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}')
|
-sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}')
|
||||||
@@ -169,6 +187,7 @@ if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# 分支 2: 有推送 → 推送线程活着,不动
|
# 分支 2: 有推送 → 推送线程活着,不动
|
||||||
if [[ ${MSG_COUNT} -gt 0 ]]; then
|
if [[ ${MSG_COUNT} -gt 0 ]]; then
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
23100 1422 1794278859
|
9822 963 1794560216
|
||||||
Reference in new issue
Block a user