Files
oa-monitor/bin/check-alert.sh
T
gsadmin 90be249a72 fix: 修复 check-alert.sh 假阳性 + collect-business.sh 窗口对齐
- check-alert.sh 新增 Step0 数据新鲜度前置检查:business_metrics 超 30min 过期 →
  先告警采集脚本挂了,跳过业务层检测,防止读旧数据乱报推送断档
- collect-business.sh msg_log_hourly 窗口从 1h 改为 15min(和 crontab 频率对齐)
- CURR_HOUR 用 10# 算术强制十进制,防 08/09 被解析成 octal
2026-10-28 18:38:13 +08:00

167 lines
10 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 告警检测(每 1 分钟)
# v2 修复(2026-10-28):
# - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警
# - Step 3 业务层检测被 Step 0 结果门禁
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
NOW=$(date +%s)
log_info "[ALERT] ========== 告警检测开始 =========="
# ============================================================================
# Step 0: 数据新鲜度前置检查(防假阳性的第一层)
# ============================================================================
# business_metrics 由 collect-business.sh 每 15min 写一次
# db_metrics 由 collect-db.sh 每 30s 写一次
# 如果最新 business_metrics ts 距现在 > 30min → 采集脚本挂了 → 告警+跳过业务层
BIZ_TS=$(sqlite_exec "SELECT ts FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
DB_TS=$(sqlite_exec "SELECT ts FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
BIZ_FRESH=true
BIZ_AGE=0
if [[ ! "${BIZ_TS}" =~ ^[0-9]+$ ]]; then
log_error "[ALERT] business_metrics 表无数据或采集从未运行"
BIZ_FRESH=false
BIZ_AGE=999999
elif [[ ! "${DB_TS}" =~ ^[0-9]+$ ]]; then
log_warn "[ALERT] db_metrics 表无数据(可能 MySQL 断了)"
else
BIZ_AGE=$((NOW - BIZ_TS))
DB_AGE=$((NOW - DB_TS))
if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟
BIZ_FRESH=false
HUMAN_AGE=$((BIZ_AGE / 60))
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec \"SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')\") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec \"SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')\") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}"
log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)"
else
log_info "[ALERT] 数据新鲜:biz_age=${BIZ_AGE}s db_age=${DB_AGE}s"
fi
fi
# ============================================================================
# 1. DB 层检测(db_metrics 每分钟更新,数据可信)
# ============================================================================
TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
if [[ -n "${TR}" ]] && [[ ${TR} =~ ^[0-9]+$ ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then
BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)"
dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');"
fi
# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的)
LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;")
if [[ -n "${LONG_QUERIES}" ]]; then
COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ')
DETAIL=""
while IFS='|' read -r pid time user state info; do
DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`"
done <<< "${LONG_QUERIES}"
BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理"
dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');"
fi
# ============================================================================
# 2. HTTP 端点检测(http_metrics 每分钟更新,数据可信)
# ============================================================================
FAIL_COUNT=0
FAIL_DETAIL=""
for endpoint in "${OA_ENDPOINTS[@]}"; do
FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;" | head -1)
FAIL_HERE=${FAIL_HERE:-0}
if [[ "${FAIL_HERE}" =~ ^[0-9]+$ ]] && [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then
FAIL_COUNT=$((FAIL_COUNT + 1))
LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;" | head -1)
FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}"
fi
done
if [[ ${FAIL_COUNT} -gt 0 ]]; then
BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态"
dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');"
fi
# ============================================================================
# 3. 业务层检测(被 BIZ_FRESH 门禁)
# ============================================================================
if [[ "${BIZ_FRESH}" != "true" ]]; then
log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)"
else
# 数据新鲜 → 正常业务层判断
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
HOUR_MSG=${HOUR_MSG:-0}
ONGOING=${ONGOING:-0}
CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal
WORK_HOURS_ENABLED=false
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
WORK_HOURS_ENABLED=true
fi
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n"
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n"
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n"
BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n"
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n"
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n"
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**"
dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
fi
else
log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警"
fi
fi
# ============================================================================
# 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信)
# ============================================================================
DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;" | head -1)
if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then
BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> 请立即检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`"
dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');"
fi
CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;" | head -1)
if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]] && [[ "${CURR_COUNT}" =~ ^[0-9]+$ ]]; then
AVG_INT=${AVG_COUNT%%.*}
DELTA=$(( CURR_COUNT - AVG_INT ))
ABS_DELTA=${DELTA#-}
if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then
BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a"
dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');"
fi
fi
DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
if [[ -n "${DISK_PCT}" ]] && [[ "${DISK_PCT}" =~ ^[0-9.]+$ ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then
BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)"
dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');"
fi
INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
if [[ -n "${INODE_PCT}" ]] && [[ "${INODE_PCT}" =~ ^[0-9.]+$ ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then
BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)"
dingtalk_alert "P1" "DSH inode 告警" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');"
fi
log_info "[ALERT] ========== 告警检测完成 =========="