Files

188 lines
12 KiB
Bash
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 告警检测(每 1 分钟)
# v2 修复(2026-10-28):
# - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警
# - Step 3 业务层检测被 Step 0 结果门禁
# v3 修复(2026-11-04):
# - 业务层 P0 推送断档假阳性:sqlite_exec 吞错误(2>/dev/null)→ 查询失败返回空串 → :-0 默认为 0 → 误报
# - 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,查询失败跳过检测并 warn
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
NOW=$(date +%s)
log_info "[ALERT] ========== 告警检测开始 =========="
# ============================================================================
# Step 0: 数据新鲜度前置检查(防假阳性的第一层)
# ============================================================================
# business_metrics 由 collect-business.sh 每 15min 写一次
# db_metrics 由 collect-db.sh 每 30s 写一次
# 如果最新 business_metrics ts 距现在 > 30min → 采集脚本挂了 → 告警+跳过业务层
BIZ_TS=$(sqlite_exec "SELECT ts FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
DB_TS=$(sqlite_exec "SELECT ts FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
BIZ_FRESH=true
BIZ_AGE=0
if [[ ! "${BIZ_TS}" =~ ^[0-9]+$ ]]; then
log_error "[ALERT] business_metrics 表无数据或采集从未运行"
BIZ_FRESH=false
BIZ_AGE=999999
elif [[ ! "${DB_TS}" =~ ^[0-9]+$ ]]; then
log_warn "[ALERT] db_metrics 表无数据(可能 MySQL 断了)"
else
BIZ_AGE=$((NOW - BIZ_TS))
DB_AGE=$((NOW - DB_TS))
if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟
BIZ_FRESH=false
HUMAN_AGE=$((BIZ_AGE / 60))
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec "SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec "SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}"
log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)"
else
log_info "[ALERT] 数据新鲜:biz_age=${BIZ_AGE}s db_age=${DB_AGE}s"
fi
fi
# ============================================================================
# 1. DB 层检测(db_metrics 每分钟更新,数据可信)
# ============================================================================
TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
if [[ -n "${TR}" ]] && [[ ${TR} =~ ^[0-9]+$ ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then
BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)"
dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');"
fi
# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的)
LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;")
if [[ -n "${LONG_QUERIES}" ]]; then
COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ')
DETAIL=""
while IFS='|' read -r pid time user state info; do
DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`"
done <<< "${LONG_QUERIES}"
BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理"
dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');"
fi
# ============================================================================
# 2. HTTP 端点检测(http_metrics 每分钟更新,数据可信)
# ============================================================================
FAIL_COUNT=0
FAIL_DETAIL=""
for endpoint in "${OA_ENDPOINTS[@]}"; do
FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;" | head -1)
FAIL_HERE=${FAIL_HERE:-0}
if [[ "${FAIL_HERE}" =~ ^[0-9]+$ ]] && [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then
FAIL_COUNT=$((FAIL_COUNT + 1))
LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;" | head -1)
FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}"
fi
done
if [[ ${FAIL_COUNT} -gt 0 ]]; then
BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态"
dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');"
fi
# 3. 业务层检测(v3 修复:直接查 MySQL 流程驱动判定,和 watchdog v5 统一口径)
# ============================================================================
if [[ "${BIZ_FRESH}" != "true" ]]; then
log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)"
else
# v3 修复:不再用 SQLite ongoing_requests(累积流转数会在周末误报)
# 直接查 MySQL 最近 20min 新流程 + message_log,和 watchdog v5 逻辑统一
ALERT_WINDOW_MIN=20
NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${ALERT_WINDOW_MIN} MINUTE, '%H:%i:%s') AND ID NOT LIKE 'DUMMY_%';" 2>/dev/null)
MSG_20MIN=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${ALERT_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
# 三重校验:非空 + 纯数字
BIZ_QUERY_OK=true
if [[ -z "${NEW_REQS}" || ! "${NEW_REQS}" =~ ^[0-9]+$ ]]; then
log_warn "[ALERT] MySQL 新流程查询异常 reqs='${NEW_REQS}',跳过业务层"
BIZ_QUERY_OK=false
fi
if [[ -z "${MSG_20MIN}" || ! "${MSG_20MIN}" =~ ^[0-9]+$ ]]; then
log_warn "[ALERT] MySQL message_log 查询异常 msg='${MSG_20MIN}',跳过业务层"
BIZ_QUERY_OK=false
fi
if [[ "${BIZ_QUERY_OK}" != "true" ]]; then
log_warn "[ALERT] ⏸️ 业务层数据查询异常,跳过钉钉推送断档 P0 检测"
else
CURR_HOUR=$((10#$(date +%H)))
WORK_HOURS_ENABLED=false
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
WORK_HOURS_ENABLED=true
fi
# === 和 watchdog v5 完全统一的判定逻辑 ===
# 分支 1: 没人提流程 → 推送线程闲着正常,不动(周末必走这里)
if [[ ${NEW_REQS} -eq 0 ]]; then
log_info "[ALERT] 最近 ${ALERT_WINDOW_MIN}min 无新流程(reqs=${NEW_REQS}),推送线程空闲属正常"
# 分支 2: 有流程 + 有推送 → 健康
elif [[ ${MSG_20MIN} -gt 0 ]]; then
log_info "[ALERT] 有新流程(${NEW_REQS}) + 有推送(${MSG_20MIN}),推送健康 ✅"
# 分支 3: 有流程 + 无推送 → 推送线程死了!
elif [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| 最近 ${ALERT_WINDOW_MIN}min 新流程 | **${NEW_REQS} 条** |\n"
BODY="${BODY}| 最近 ${ALERT_WINDOW_MIN}min message_log | **${MSG_20MIN} 条** |\n"
BODY="${BODY}| 判定 | **有流程但无推送 → 推送线程已死** |\n\n"
BODY="${BODY}> ⚠️ 推送线程卡死!watchdog v6 会触发自愈重启\n"
dingtalk_alert "P0" "钉钉推送完全断档(有流程但无推送)" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, ${NOW}, 'P0', 'business', '钉钉推送完全断档', 'reqs=${NEW_REQS} msg=${MSG_20MIN}')"
else
log_warn "[ALERT] 非工作时间(${CURR_HOUR}:00),有流程(${NEW_REQS})但无推送(${MSG_20MIN}),仅记录"
fi
fi
fi
# ============================================================================
# 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信)
# ============================================================================
DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;" | head -1)
if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then
BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> 请立即检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`"
dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');"
fi
CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;" | head -1)
if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]] && [[ "${CURR_COUNT}" =~ ^[0-9]+$ ]]; then
AVG_INT=${AVG_COUNT%%.*}
DELTA=$(( CURR_COUNT - AVG_INT ))
ABS_DELTA=${DELTA#-}
if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then
BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a"
dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');"
fi
fi
DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
if [[ -n "${DISK_PCT}" ]] && [[ "${DISK_PCT}" =~ ^[0-9.]+$ ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then
BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)"
dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');"
fi
INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
if [[ -n "${INODE_PCT}" ]] && [[ "${INODE_PCT}" =~ ^[0-9.]+$ ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then
BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)"
dingtalk_alert "P1" "DSH inode 告警" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');"
fi
log_info "[ALERT] ========== 告警检测完成 =========="