From cc3184fd3e44afdc0ba406362096ff7398f34dba Mon Sep 17 00:00:00 2001 From: oa-monitor Date: Sat, 7 Nov 2026 08:55:07 +0800 Subject: [PATCH] =?UTF-8?q?check-alert=20v3:=20=E6=94=B9=E6=88=90=20MySQL?= =?UTF-8?q?=20=E6=B5=81=E7=A8=8B=E9=A9=B1=E5=8A=A8=E5=88=A4=E5=AE=9A?= =?UTF-8?q?=EF=BC=8C=E5=92=8C=20watchdog=20v5=20=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E5=8F=A3=E5=BE=84=EF=BC=8C=E5=91=A8=E6=9C=AB=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E8=AF=AF=E6=8A=A5=E6=8E=A8=E9=80=81=E6=96=AD=E6=A1=A3=20P0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/check-alert.sh | 64 ++++++++++++++++++++++---------------------- state/resin_pid.last | 2 +- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/bin/check-alert.sh b/bin/check-alert.sh index 73244c3..056ea9c 100755 --- a/bin/check-alert.sh +++ b/bin/check-alert.sh @@ -91,61 +91,61 @@ if [[ ${FAIL_COUNT} -gt 0 ]]; then sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');" fi -# ============================================================================ -# 3. 业务层检测(被 BIZ_FRESH 门禁) +# 3. 业务层检测(v3 修复:直接查 MySQL 流程驱动判定,和 watchdog v5 统一口径) # ============================================================================ if [[ "${BIZ_FRESH}" != "true" ]]; then log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)" else - # v3 修复:显式调用 sqlite3 + 检查退出码 + 正则校验,区分"查询失败"和"结果为 0" - MSG_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null) - MSG_RC=$? - ONGOING_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null) - ONGOING_RC=$? + # v3 修复:不再用 SQLite ongoing_requests(累积流转数会在周末误报) + # 直接查 MySQL 最近 20min 新流程 + message_log,和 watchdog v5 逻辑统一 + ALERT_WINDOW_MIN=20 + NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${ALERT_WINDOW_MIN} MINUTE, '%H:%i:%s') AND DELETED=0;" 2>/dev/null) + MSG_20MIN=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${ALERT_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null) - # 双重校验:退出码必须为 0,且返回值非空且为纯数字 + # 三重校验:非空 + 纯数字 BIZ_QUERY_OK=true - if [[ ${MSG_RC} -ne 0 ]] || [[ -z "${MSG_RAW}" ]] || [[ ! "${MSG_RAW}" =~ ^[0-9]+$ ]]; then - log_warn "[ALERT] SQLite msg_log_hourly 查询异常 rc=${MSG_RC} raw='${MSG_RAW}',跳过业务层(防假阳性)" + if [[ -z "${NEW_REQS}" || ! "${NEW_REQS}" =~ ^[0-9]+$ ]]; then + log_warn "[ALERT] MySQL 新流程查询异常 reqs='${NEW_REQS}',跳过业务层" BIZ_QUERY_OK=false fi - if [[ ${ONGOING_RC} -ne 0 ]] || [[ -z "${ONGOING_RAW}" ]] || [[ ! "${ONGOING_RAW}" =~ ^[0-9]+$ ]]; then - log_warn "[ALERT] SQLite ongoing_requests 查询异常 rc=${ONGOING_RC} raw='${ONGOING_RAW}',跳过业务层(防假阳性)" + if [[ -z "${MSG_20MIN}" || ! "${MSG_20MIN}" =~ ^[0-9]+$ ]]; then + log_warn "[ALERT] MySQL message_log 查询异常 msg='${MSG_20MIN}',跳过业务层" BIZ_QUERY_OK=false fi if [[ "${BIZ_QUERY_OK}" != "true" ]]; then log_warn "[ALERT] ⏸️ 业务层数据查询异常,跳过钉钉推送断档 P0 检测" else - HOUR_MSG=${MSG_RAW} - ONGOING=${ONGOING_RAW} - - CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal + CURR_HOUR=$((10#$(date +%H))) WORK_HOURS_ENABLED=false if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then WORK_HOURS_ENABLED=true fi - if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then - if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then - BODY="**🔴 P0 钉钉推送完全断档!**\n\n" - BODY="${BODY}| 指标 | 值 |\n|---|---|\n" - BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n" - BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n" - BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n" - BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n" - BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n" - BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n" - BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**" - dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}" - sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');" - fi + # === 和 watchdog v5 完全统一的判定逻辑 === + # 分支 1: 没人提流程 → 推送线程闲着正常,不动(周末必走这里) + if [[ ${NEW_REQS} -eq 0 ]]; then + log_info "[ALERT] 最近 ${ALERT_WINDOW_MIN}min 无新流程(reqs=${NEW_REQS}),推送线程空闲属正常" + # 分支 2: 有流程 + 有推送 → 健康 + elif [[ ${MSG_20MIN} -gt 0 ]]; then + log_info "[ALERT] 有新流程(${NEW_REQS}) + 有推送(${MSG_20MIN}),推送健康 ✅" + # 分支 3: 有流程 + 无推送 → 推送线程死了! + elif [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then + BODY="**🔴 P0 钉钉推送完全断档!**\n\n" + BODY="${BODY}| 指标 | 值 |\n|---|---|\n" + BODY="${BODY}| 最近 ${ALERT_WINDOW_MIN}min 新流程 | **${NEW_REQS} 条** |\n" + BODY="${BODY}| 最近 ${ALERT_WINDOW_MIN}min message_log | **${MSG_20MIN} 条** |\n" + BODY="${BODY}| 判定 | **有流程但无推送 → 推送线程已死** |\n\n" + BODY="${BODY}> ⚠️ 推送线程卡死!watchdog v6 会触发自愈重启\n" + dingtalk_alert "P0" "钉钉推送完全断档(有流程但无推送)" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, ${NOW}, 'P0', 'business', '钉钉推送完全断档', 'reqs=${NEW_REQS} msg=${MSG_20MIN}')" else - log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警" + log_warn "[ALERT] 非工作时间(${CURR_HOUR}:00),有流程(${NEW_REQS})但无推送(${MSG_20MIN}),仅记录" fi fi fi - # ============================================================================ # 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信) # ============================================================================ diff --git a/state/resin_pid.last b/state/resin_pid.last index b0e3c16..5af8c21 100644 --- a/state/resin_pid.last +++ b/state/resin_pid.last @@ -1 +1 @@ -31753 2602 1793960923 +9155 32063 1794012902