check-alert v3: 改成 MySQL 流程驱动判定,和 watchdog v5 统一口径,周末不再误报推送断档 P0

This commit is contained in:
oa-monitor committed 2026-11-07 08:55:07 +08:00
1 parent 689303f5d7
commit cc3184fd3e
2 files changed
+31 -31

No files matched your search

+30 -30
View File
@@ -91,61 +91,61 @@ if [[ ${FAIL_COUNT} -gt 0 ]]; then
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');"
fi fi
# ============================================================================ # 3. 业务层检测(v3 修复:直接查 MySQL 流程驱动判定,和 watchdog v5 统一口径)
# 3. 业务层检测(被 BIZ_FRESH 门禁)
# ============================================================================ # ============================================================================
if [[ "${BIZ_FRESH}" != "true" ]]; then if [[ "${BIZ_FRESH}" != "true" ]]; then
log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)" log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)"
else else
# v3 修复:显式调用 sqlite3 + 检查退出码 + 正则校验,区分"查询失败"和"结果为 0" # v3 修复:不再用 SQLite ongoing_requests(累积流转数会在周末误报)
MSG_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null) # 直接查 MySQL 最近 20min 新流程 + message_log,和 watchdog v5 逻辑统一
MSG_RC=$? ALERT_WINDOW_MIN=20
ONGOING_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null) NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
ONGOING_RC=$? -sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${ALERT_WINDOW_MIN} MINUTE, '%H:%i:%s') AND DELETED=0;" 2>/dev/null)
MSG_20MIN=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${ALERT_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
# 双重校验:退出码必须为 0,且返回值非空且为纯数字 # 三重校验:非空 + 纯数字
BIZ_QUERY_OK=true BIZ_QUERY_OK=true
if [[ ${MSG_RC} -ne 0 ]] || [[ -z "${MSG_RAW}" ]] || [[ ! "${MSG_RAW}" =~ ^[0-9]+$ ]]; then if [[ -z "${NEW_REQS}" || ! "${NEW_REQS}" =~ ^[0-9]+$ ]]; then
log_warn "[ALERT] SQLite msg_log_hourly 查询异常 rc=${MSG_RC} raw='${MSG_RAW}',跳过业务层(防假阳性)" log_warn "[ALERT] MySQL 新流程查询异常 reqs='${NEW_REQS}',跳过业务层"
BIZ_QUERY_OK=false BIZ_QUERY_OK=false
fi fi
if [[ ${ONGOING_RC} -ne 0 ]] || [[ -z "${ONGOING_RAW}" ]] || [[ ! "${ONGOING_RAW}" =~ ^[0-9]+$ ]]; then if [[ -z "${MSG_20MIN}" || ! "${MSG_20MIN}" =~ ^[0-9]+$ ]]; then
log_warn "[ALERT] SQLite ongoing_requests 查询异常 rc=${ONGOING_RC} raw='${ONGOING_RAW}',跳过业务层(防假阳性)" log_warn "[ALERT] MySQL message_log 查询异常 msg='${MSG_20MIN}',跳过业务层"
BIZ_QUERY_OK=false BIZ_QUERY_OK=false
fi fi
if [[ "${BIZ_QUERY_OK}" != "true" ]]; then if [[ "${BIZ_QUERY_OK}" != "true" ]]; then
log_warn "[ALERT] ⏸️ 业务层数据查询异常,跳过钉钉推送断档 P0 检测" log_warn "[ALERT] ⏸️ 业务层数据查询异常,跳过钉钉推送断档 P0 检测"
else else
HOUR_MSG=${MSG_RAW} CURR_HOUR=$((10#$(date +%H)))
ONGOING=${ONGOING_RAW}
CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal
WORK_HOURS_ENABLED=false WORK_HOURS_ENABLED=false
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
WORK_HOURS_ENABLED=true WORK_HOURS_ENABLED=true
fi fi
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then # === 和 watchdog v5 完全统一的判定逻辑 ===
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then # 分支 1: 没人提流程 → 推送线程闲着正常,不动(周末必走这里)
if [[ ${NEW_REQS} -eq 0 ]]; then
log_info "[ALERT] 最近 ${ALERT_WINDOW_MIN}min 无新流程(reqs=${NEW_REQS}),推送线程空闲属正常"
# 分支 2: 有流程 + 有推送 → 健康
elif [[ ${MSG_20MIN} -gt 0 ]]; then
log_info "[ALERT] 有新流程(${NEW_REQS}) + 有推送(${MSG_20MIN}),推送健康 ✅"
# 分支 3: 有流程 + 无推送 → 推送线程死了!
elif [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
BODY="**🔴 P0 钉钉推送完全断档!**\n\n" BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n" BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n" BODY="${BODY}| 最近 ${ALERT_WINDOW_MIN}min 新流程 | **${NEW_REQS} 条** |\n"
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n" BODY="${BODY}| 最近 ${ALERT_WINDOW_MIN}min message_log | **${MSG_20MIN} 条** |\n"
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n" BODY="${BODY}| 判定 | **有流程但无推送 → 推送线程已死** |\n\n"
BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n" BODY="${BODY}> ⚠️ 推送线程卡死!watchdog v6 会触发自愈重启\n"
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n" dingtalk_alert "P0" "钉钉推送完全断档(有流程但无推送)" "${BODY}"
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n" sqlite_exec "INSERT INTO alerts (ts, ${NOW}, 'P0', 'business', '钉钉推送完全断档', 'reqs=${NEW_REQS} msg=${MSG_20MIN}')"
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**"
dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
fi
else else
log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警" log_warn "[ALERT] 非工作时间(${CURR_HOUR}:00),有流程(${NEW_REQS})但无推送(${MSG_20MIN}),仅记录"
fi fi
fi fi
fi fi
# ============================================================================ # ============================================================================
# 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信) # 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信)
# ============================================================================ # ============================================================================
+1 -1
View File
@@ -1 +1 @@
31753 2602 1793960923 9155 32063 1794012902