#!/usr/bin/env bash # ============================================================================ # OA Monitor · 告警检测(每 1 分钟) # v2 修复(2026-10-28): # - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警 # - Step 3 业务层检测被 Step 0 结果门禁 # v3 修复(2026-11-04): # - 业务层 P0 推送断档假阳性:sqlite_exec 吞错误(2>/dev/null)→ 查询失败返回空串 → :-0 默认为 0 → 误报 # - 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,查询失败跳过检测并 warn # ============================================================================ set -u SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "${SCRIPT_DIR}/../lib/common.sh" NOW=$(date +%s) log_info "[ALERT] ========== 告警检测开始 ==========" # ============================================================================ # Step 0: 数据新鲜度前置检查(防假阳性的第一层) # ============================================================================ # business_metrics 由 collect-business.sh 每 15min 写一次 # db_metrics 由 collect-db.sh 每 30s 写一次 # 如果最新 business_metrics ts 距现在 > 30min → 采集脚本挂了 → 告警+跳过业务层 BIZ_TS=$(sqlite_exec "SELECT ts FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1) DB_TS=$(sqlite_exec "SELECT ts FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1) BIZ_FRESH=true BIZ_AGE=0 if [[ ! "${BIZ_TS}" =~ ^[0-9]+$ ]]; then log_error "[ALERT] business_metrics 表无数据或采集从未运行" BIZ_FRESH=false BIZ_AGE=999999 elif [[ ! "${DB_TS}" =~ ^[0-9]+$ ]]; then log_warn "[ALERT] db_metrics 表无数据(可能 MySQL 断了)" else BIZ_AGE=$((NOW - BIZ_TS)) DB_AGE=$((NOW - DB_TS)) if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟 BIZ_FRESH=false HUMAN_AGE=$((BIZ_AGE / 60)) BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec "SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec "SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警" dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}" log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)" else log_info "[ALERT] 数据新鲜:biz_age=${BIZ_AGE}s db_age=${DB_AGE}s" fi fi # ============================================================================ # 1. DB 层检测(db_metrics 每分钟更新,数据可信) # ============================================================================ TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1) if [[ -n "${TR}" ]] && [[ ${TR} =~ ^[0-9]+$ ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)" dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');" fi # 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的) LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;") if [[ -n "${LONG_QUERIES}" ]]; then COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ') DETAIL="" while IFS='|' read -r pid time user state info; do DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`" done <<< "${LONG_QUERIES}" BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理" dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');" fi # ============================================================================ # 2. HTTP 端点检测(http_metrics 每分钟更新,数据可信) # ============================================================================ FAIL_COUNT=0 FAIL_DETAIL="" for endpoint in "${OA_ENDPOINTS[@]}"; do FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;" | head -1) FAIL_HERE=${FAIL_HERE:-0} if [[ "${FAIL_HERE}" =~ ^[0-9]+$ ]] && [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then FAIL_COUNT=$((FAIL_COUNT + 1)) LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;" | head -1) FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}" fi done if [[ ${FAIL_COUNT} -gt 0 ]]; then BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态" dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');" fi # ============================================================================ # 3. 业务层检测(被 BIZ_FRESH 门禁) # ============================================================================ if [[ "${BIZ_FRESH}" != "true" ]]; then log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)" else # v3 修复:显式调用 sqlite3 + 检查退出码 + 正则校验,区分"查询失败"和"结果为 0" MSG_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null) MSG_RC=$? ONGOING_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null) ONGOING_RC=$? # 双重校验:退出码必须为 0,且返回值非空且为纯数字 BIZ_QUERY_OK=true if [[ ${MSG_RC} -ne 0 ]] || [[ -z "${MSG_RAW}" ]] || [[ ! "${MSG_RAW}" =~ ^[0-9]+$ ]]; then log_warn "[ALERT] SQLite msg_log_hourly 查询异常 rc=${MSG_RC} raw='${MSG_RAW}',跳过业务层(防假阳性)" BIZ_QUERY_OK=false fi if [[ ${ONGOING_RC} -ne 0 ]] || [[ -z "${ONGOING_RAW}" ]] || [[ ! "${ONGOING_RAW}" =~ ^[0-9]+$ ]]; then log_warn "[ALERT] SQLite ongoing_requests 查询异常 rc=${ONGOING_RC} raw='${ONGOING_RAW}',跳过业务层(防假阳性)" BIZ_QUERY_OK=false fi if [[ "${BIZ_QUERY_OK}" != "true" ]]; then log_warn "[ALERT] ⏸️ 业务层数据查询异常,跳过钉钉推送断档 P0 检测" else HOUR_MSG=${MSG_RAW} ONGOING=${ONGOING_RAW} CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal WORK_HOURS_ENABLED=false if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then WORK_HOURS_ENABLED=true fi if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then BODY="**🔴 P0 钉钉推送完全断档!**\n\n" BODY="${BODY}| 指标 | 值 |\n|---|---|\n" BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n" BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n" BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n" BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n" BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n" BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n" BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**" dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');" fi else log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警" fi fi fi # ============================================================================ # 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信) # ============================================================================ DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1) PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;" | head -1) if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> 请立即检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`" dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');" fi CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1) AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;" | head -1) if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]] && [[ "${CURR_COUNT}" =~ ^[0-9]+$ ]]; then AVG_INT=${AVG_COUNT%%.*} DELTA=$(( CURR_COUNT - AVG_INT )) ABS_DELTA=${DELTA#-} if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a" dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');" fi fi DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1) if [[ -n "${DISK_PCT}" ]] && [[ "${DISK_PCT}" =~ ^[0-9.]+$ ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)" dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');" fi INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1) if [[ -n "${INODE_PCT}" ]] && [[ "${INODE_PCT}" =~ ^[0-9.]+$ ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)" dingtalk_alert "P1" "DSH inode 告警" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');" fi log_info "[ALERT] ========== 告警检测完成 =========="