From 2e1dbb5403bdb2e8659e3ccfef380f7fe957f341 Mon Sep 17 00:00:00 2001 From: oa-monitor Date: Fri, 13 Nov 2026 16:57:20 +0800 Subject: [PATCH] =?UTF-8?q?fix:=2014:41=20updateBizState=20=E5=8D=A1?= =?UTF-8?q?=E6=AD=BB=E6=A0=B9=E5=9B=A0=E4=BF=AE=E5=A4=8D=20+=20watchdog=20?= =?UTF-8?q?v8=20=E4=BC=98=E5=8C=96=20-=20ALTER=20ecology=5Fmessage=5Finfo?= =?UTF-8?q?=5Fread=20ADD=20INDEX=20idx=5Firead=5Ftarget=5Fbiz=20(TARGETID,?= =?UTF-8?q?=20USERID,=20BIZSTATE)=20-=20UPDATE=20ecology=5Fmessage=5Finfo?= =?UTF-8?q?=5Fread=20SET=20BIZSTATE=3D1=20WHERE=20BIZSTATE=3D0=20(?= =?UTF-8?q?=E6=B8=85=20164371+3437=20=E6=9D=A1=E7=A7=AF=E5=8E=8B)=20-=20wa?= =?UTF-8?q?tchdog=20v8:=20Layer=204=20workflow=5Frequestbase=20=E8=81=94?= =?UTF-8?q?=E5=8A=A8=E6=A3=80=E6=9F=A5=20+=20Layer=205=20BIZSTATE=20?= =?UTF-8?q?=E7=A7=AF=E5=8E=8B=20P1=20=E5=91=8A=E8=AD=A6=20-=20=E6=A0=B9?= =?UTF-8?q?=E5=9B=A0:=20DUMMY=20=E8=A1=A5=20397=20=E4=B8=87=20info=5Fread?= =?UTF-8?q?=20=E5=90=8E=E8=A1=A8=E8=86=A8=E8=83=80=2020x=20=E2=86=92=20upd?= =?UTF-8?q?ateBizState=20=E5=8D=95=E7=BA=BF=E7=A8=8B=E5=85=A8=E8=A1=A8?= =?UTF-8?q?=E6=89=AB=E8=BF=BD=E4=B8=8D=E4=B8=8A=20=E2=86=92=20BIZSTATE=3D0?= =?UTF-8?q?=20=E7=A7=AF=E5=8E=8B=20=E2=86=92=20message=5Flog=20=E5=81=9C?= =?UTF-8?q?=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/resin-watchdog.sh | 39 +++++++++++++++++++++++++++++---------- state/resin_pid.last | 2 +- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/bin/resin-watchdog.sh b/bin/resin-watchdog.sh index 0ac0208..7b485ca 100755 --- a/bin/resin-watchdog.sh +++ b/bin/resin-watchdog.sh @@ -153,20 +153,39 @@ fi log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}" +# === v8 新增: BIZSTATE 积压检查(推送线程慢了但没死 → 提前告警)=== +BIZ_BACKLOG=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE BIZSTATE=0;" 2>/dev/null) +if [[ "${BIZ_BACKLOG}" =~ ^[0-9]+$ && ${BIZ_BACKLOG} -gt 1000 ]]; then + log_warn "[WATCHDOG] ⚠️ BIZSTATE=0 积压=${BIZ_BACKLOG}(推送线程追不上)→ P1 告警" + dingtalk_alert "P1" "info_read BIZSTATE 积压告警" "当前积压: ${BIZ_BACKLOG} 条 +含义: 推送线程追不上业务产生的 info_read 速度 +建议: 联系 AI 处理 UPDATE ecology_message_info_read SET BIZSTATE=1 WHERE BIZSTATE=0" +fi + # === 判定分支 === -# 分支 1: info_read=0 AND message_log=0 → 可能真没人用,也可能 Deadlock 全卡死 +# v8 新增: 先查 workflow_requestbase 最近 20min 新流程(判断业务是否真在跑) +WF_NEW=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null) + +# 分支 1: info_read=0 AND message_log=0 if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then - # 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5 - TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ - -sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}') - - if [[ "${TR}" =~ ^[0-9]+$ && ${TR} -gt 5 ]]; then - # Threads_running 异常高 → 可能 Deadlock 全卡死 → 触发自愈 - log_warn "[WATCHDOG] info_read=0 + message_log=0 + MySQL Threads_running=${TR} > 5 → 疑似 Deadlock 全卡死 → 自愈!" + # v8 Layer 4 新增: workflow_requestbase 有新流程 → 业务在跑但两张表都停了 + if [[ "${WF_NEW}" =~ ^[0-9]+$ && ${WF_NEW} -gt 0 ]]; then + log_warn "[WATCHDOG] ⚠️ info_read=0 + message_log=0 BUT workflow_requestbase 新流程=${WF_NEW} → 推送线程卡死 → 自愈!" else - log_info "[WATCHDOG] info_read=0, message_log=0, Threads_running=${TR:-?} 正常 → 推送线程空闲属正常, 不动" - exit 0 + # 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5 + TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}') + + if [[ "${TR}" =~ ^[0-9]+$ && ${TR} -gt 5 ]]; then + # Threads_running 异常高 → 可能 Deadlock 全卡死 → 触发自愈 + log_warn "[WATCHDOG] info_read=0 + message_log=0 + MySQL Threads_running=${TR} > 5 → 疑似 Deadlock 全卡死 → 自愈!" + else + log_info "[WATCHDOG] info_read=0, message_log=0, Threads_running=${TR:-?} 正常 → 推送线程空闲属正常, 不动" + exit 0 + fi fi fi diff --git a/state/resin_pid.last b/state/resin_pid.last index 53c4ee3..850caaa 100644 --- a/state/resin_pid.last +++ b/state/resin_pid.last @@ -1 +1 @@ -23100 1422 1794278859 +9822 963 1794560216