188 lines
12 KiB
Bash
Executable File
188 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# ============================================================================
|
||
# OA Monitor · 告警检测(每 1 分钟)
|
||
# v2 修复(2026-10-28):
|
||
# - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警
|
||
# - Step 3 业务层检测被 Step 0 结果门禁
|
||
# v3 修复(2026-11-04):
|
||
# - 业务层 P0 推送断档假阳性:sqlite_exec 吞错误(2>/dev/null)→ 查询失败返回空串 → :-0 默认为 0 → 误报
|
||
# - 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,查询失败跳过检测并 warn
|
||
# ============================================================================
|
||
set -u
|
||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||
|
||
NOW=$(date +%s)
|
||
log_info "[ALERT] ========== 告警检测开始 =========="
|
||
|
||
# ============================================================================
|
||
# Step 0: 数据新鲜度前置检查(防假阳性的第一层)
|
||
# ============================================================================
|
||
# business_metrics 由 collect-business.sh 每 15min 写一次
|
||
# db_metrics 由 collect-db.sh 每 30s 写一次
|
||
# 如果最新 business_metrics ts 距现在 > 30min → 采集脚本挂了 → 告警+跳过业务层
|
||
BIZ_TS=$(sqlite_exec "SELECT ts FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||
DB_TS=$(sqlite_exec "SELECT ts FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||
BIZ_FRESH=true
|
||
BIZ_AGE=0
|
||
|
||
if [[ ! "${BIZ_TS}" =~ ^[0-9]+$ ]]; then
|
||
log_error "[ALERT] business_metrics 表无数据或采集从未运行"
|
||
BIZ_FRESH=false
|
||
BIZ_AGE=999999
|
||
elif [[ ! "${DB_TS}" =~ ^[0-9]+$ ]]; then
|
||
log_warn "[ALERT] db_metrics 表无数据(可能 MySQL 断了)"
|
||
else
|
||
BIZ_AGE=$((NOW - BIZ_TS))
|
||
DB_AGE=$((NOW - DB_TS))
|
||
|
||
if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟
|
||
BIZ_FRESH=false
|
||
HUMAN_AGE=$((BIZ_AGE / 60))
|
||
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec "SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec "SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
|
||
dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}"
|
||
log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)"
|
||
else
|
||
log_info "[ALERT] 数据新鲜:biz_age=${BIZ_AGE}s db_age=${DB_AGE}s"
|
||
fi
|
||
fi
|
||
|
||
# ============================================================================
|
||
# 1. DB 层检测(db_metrics 每分钟更新,数据可信)
|
||
# ============================================================================
|
||
TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||
if [[ -n "${TR}" ]] && [[ ${TR} =~ ^[0-9]+$ ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then
|
||
BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)"
|
||
dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}"
|
||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');"
|
||
fi
|
||
|
||
# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的)
|
||
LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;")
|
||
if [[ -n "${LONG_QUERIES}" ]]; then
|
||
COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ')
|
||
DETAIL=""
|
||
while IFS='|' read -r pid time user state info; do
|
||
DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`"
|
||
done <<< "${LONG_QUERIES}"
|
||
BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理"
|
||
dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}"
|
||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');"
|
||
fi
|
||
|
||
# ============================================================================
|
||
# 2. HTTP 端点检测(http_metrics 每分钟更新,数据可信)
|
||
# ============================================================================
|
||
FAIL_COUNT=0
|
||
FAIL_DETAIL=""
|
||
for endpoint in "${OA_ENDPOINTS[@]}"; do
|
||
FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;" | head -1)
|
||
FAIL_HERE=${FAIL_HERE:-0}
|
||
if [[ "${FAIL_HERE}" =~ ^[0-9]+$ ]] && [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then
|
||
FAIL_COUNT=$((FAIL_COUNT + 1))
|
||
LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;" | head -1)
|
||
FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}"
|
||
fi
|
||
done
|
||
|
||
if [[ ${FAIL_COUNT} -gt 0 ]]; then
|
||
BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态"
|
||
dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}"
|
||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');"
|
||
fi
|
||
|
||
# 3. 业务层检测(v3 修复:直接查 MySQL 流程驱动判定,和 watchdog v5 统一口径)
|
||
# ============================================================================
|
||
if [[ "${BIZ_FRESH}" != "true" ]]; then
|
||
log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)"
|
||
else
|
||
# v3 修复:不再用 SQLite ongoing_requests(累积流转数会在周末误报)
|
||
# 直接查 MySQL 最近 20min 新流程 + message_log,和 watchdog v5 逻辑统一
|
||
ALERT_WINDOW_MIN=20
|
||
NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${ALERT_WINDOW_MIN} MINUTE, '%H:%i:%s') AND ID NOT LIKE 'DUMMY_%';" 2>/dev/null)
|
||
MSG_20MIN=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${ALERT_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
|
||
|
||
# 三重校验:非空 + 纯数字
|
||
BIZ_QUERY_OK=true
|
||
if [[ -z "${NEW_REQS}" || ! "${NEW_REQS}" =~ ^[0-9]+$ ]]; then
|
||
log_warn "[ALERT] MySQL 新流程查询异常 reqs='${NEW_REQS}',跳过业务层"
|
||
BIZ_QUERY_OK=false
|
||
fi
|
||
if [[ -z "${MSG_20MIN}" || ! "${MSG_20MIN}" =~ ^[0-9]+$ ]]; then
|
||
log_warn "[ALERT] MySQL message_log 查询异常 msg='${MSG_20MIN}',跳过业务层"
|
||
BIZ_QUERY_OK=false
|
||
fi
|
||
|
||
if [[ "${BIZ_QUERY_OK}" != "true" ]]; then
|
||
log_warn "[ALERT] ⏸️ 业务层数据查询异常,跳过钉钉推送断档 P0 检测"
|
||
else
|
||
CURR_HOUR=$((10#$(date +%H)))
|
||
WORK_HOURS_ENABLED=false
|
||
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
|
||
WORK_HOURS_ENABLED=true
|
||
fi
|
||
|
||
# === 和 watchdog v5 完全统一的判定逻辑 ===
|
||
# 分支 1: 没人提流程 → 推送线程闲着正常,不动(周末必走这里)
|
||
if [[ ${NEW_REQS} -eq 0 ]]; then
|
||
log_info "[ALERT] 最近 ${ALERT_WINDOW_MIN}min 无新流程(reqs=${NEW_REQS}),推送线程空闲属正常"
|
||
# 分支 2: 有流程 + 有推送 → 健康
|
||
elif [[ ${MSG_20MIN} -gt 0 ]]; then
|
||
log_info "[ALERT] 有新流程(${NEW_REQS}) + 有推送(${MSG_20MIN}),推送健康 ✅"
|
||
# 分支 3: 有流程 + 无推送 → 推送线程死了!
|
||
elif [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
|
||
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
|
||
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
||
BODY="${BODY}| 最近 ${ALERT_WINDOW_MIN}min 新流程 | **${NEW_REQS} 条** |\n"
|
||
BODY="${BODY}| 最近 ${ALERT_WINDOW_MIN}min message_log | **${MSG_20MIN} 条** |\n"
|
||
BODY="${BODY}| 判定 | **有流程但无推送 → 推送线程已死** |\n\n"
|
||
BODY="${BODY}> ⚠️ 推送线程卡死!watchdog v6 会触发自愈重启\n"
|
||
dingtalk_alert "P0" "钉钉推送完全断档(有流程但无推送)" "${BODY}"
|
||
sqlite_exec "INSERT INTO alerts (ts, ${NOW}, 'P0', 'business', '钉钉推送完全断档', 'reqs=${NEW_REQS} msg=${MSG_20MIN}')"
|
||
else
|
||
log_warn "[ALERT] 非工作时间(${CURR_HOUR}:00),有流程(${NEW_REQS})但无推送(${MSG_20MIN}),仅记录"
|
||
fi
|
||
fi
|
||
fi
|
||
# ============================================================================
|
||
# 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信)
|
||
# ============================================================================
|
||
DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||
PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;" | head -1)
|
||
if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then
|
||
BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> 请立即检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`"
|
||
dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}"
|
||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');"
|
||
fi
|
||
|
||
CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||
AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;" | head -1)
|
||
if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]] && [[ "${CURR_COUNT}" =~ ^[0-9]+$ ]]; then
|
||
AVG_INT=${AVG_COUNT%%.*}
|
||
DELTA=$(( CURR_COUNT - AVG_INT ))
|
||
ABS_DELTA=${DELTA#-}
|
||
if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then
|
||
BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a"
|
||
dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}"
|
||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');"
|
||
fi
|
||
fi
|
||
|
||
DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||
if [[ -n "${DISK_PCT}" ]] && [[ "${DISK_PCT}" =~ ^[0-9.]+$ ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then
|
||
BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)"
|
||
dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}"
|
||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');"
|
||
fi
|
||
|
||
INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||
if [[ -n "${INODE_PCT}" ]] && [[ "${INODE_PCT}" =~ ^[0-9.]+$ ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then
|
||
BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)"
|
||
dingtalk_alert "P1" "DSH inode 告警" "${BODY}"
|
||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');"
|
||
fi
|
||
|
||
log_info "[ALERT] ========== 告警检测完成 =========="
|