feat: watchdog v4 + check-alert v2 + collect-business v2 + DUMMY补数据脚本
This commit is contained in:
commit
7ab75364b1
22 files changed
+1795
No files matched your search
Executable
+166
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 告警检测(每 1 分钟)
|
||||
# v2 修复(2026-10-28):
|
||||
# - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警
|
||||
# - Step 3 业务层检测被 Step 0 结果门禁
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
log_info "[ALERT] ========== 告警检测开始 =========="
|
||||
|
||||
# ============================================================================
|
||||
# Step 0: 数据新鲜度前置检查(防假阳性的第一层)
|
||||
# ============================================================================
|
||||
# business_metrics 由 collect-business.sh 每 15min 写一次
|
||||
# db_metrics 由 collect-db.sh 每 30s 写一次
|
||||
# 如果最新 business_metrics ts 距现在 > 30min → 采集脚本挂了 → 告警+跳过业务层
|
||||
BIZ_TS=$(sqlite_exec "SELECT ts FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
DB_TS=$(sqlite_exec "SELECT ts FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
BIZ_FRESH=true
|
||||
BIZ_AGE=0
|
||||
|
||||
if [[ ! "${BIZ_TS}" =~ ^[0-9]+$ ]]; then
|
||||
log_error "[ALERT] business_metrics 表无数据或采集从未运行"
|
||||
BIZ_FRESH=false
|
||||
BIZ_AGE=999999
|
||||
elif [[ ! "${DB_TS}" =~ ^[0-9]+$ ]]; then
|
||||
log_warn "[ALERT] db_metrics 表无数据(可能 MySQL 断了)"
|
||||
else
|
||||
BIZ_AGE=$((NOW - BIZ_TS))
|
||||
DB_AGE=$((NOW - DB_TS))
|
||||
|
||||
if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟
|
||||
BIZ_FRESH=false
|
||||
HUMAN_AGE=$((BIZ_AGE / 60))
|
||||
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec \"SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')\") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec \"SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')\") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
|
||||
dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}"
|
||||
log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)"
|
||||
else
|
||||
log_info "[ALERT] 数据新鲜:biz_age=${BIZ_AGE}s db_age=${DB_AGE}s"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 1. DB 层检测(db_metrics 每分钟更新,数据可信)
|
||||
# ============================================================================
|
||||
TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
if [[ -n "${TR}" ]] && [[ ${TR} =~ ^[0-9]+$ ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then
|
||||
BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)"
|
||||
dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');"
|
||||
fi
|
||||
|
||||
# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的)
|
||||
LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;")
|
||||
if [[ -n "${LONG_QUERIES}" ]]; then
|
||||
COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ')
|
||||
DETAIL=""
|
||||
while IFS='|' read -r pid time user state info; do
|
||||
DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`"
|
||||
done <<< "${LONG_QUERIES}"
|
||||
BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理"
|
||||
dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 2. HTTP 端点检测(http_metrics 每分钟更新,数据可信)
|
||||
# ============================================================================
|
||||
FAIL_COUNT=0
|
||||
FAIL_DETAIL=""
|
||||
for endpoint in "${OA_ENDPOINTS[@]}"; do
|
||||
FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;" | head -1)
|
||||
FAIL_HERE=${FAIL_HERE:-0}
|
||||
if [[ "${FAIL_HERE}" =~ ^[0-9]+$ ]] && [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then
|
||||
FAIL_COUNT=$((FAIL_COUNT + 1))
|
||||
LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${FAIL_COUNT} -gt 0 ]]; then
|
||||
BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态"
|
||||
dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 3. 业务层检测(被 BIZ_FRESH 门禁)
|
||||
# ============================================================================
|
||||
if [[ "${BIZ_FRESH}" != "true" ]]; then
|
||||
log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)"
|
||||
else
|
||||
# 数据新鲜 → 正常业务层判断
|
||||
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
HOUR_MSG=${HOUR_MSG:-0}
|
||||
ONGOING=${ONGOING:-0}
|
||||
|
||||
CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal
|
||||
WORK_HOURS_ENABLED=false
|
||||
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
|
||||
WORK_HOURS_ENABLED=true
|
||||
fi
|
||||
|
||||
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
|
||||
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then
|
||||
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
|
||||
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
||||
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n"
|
||||
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n"
|
||||
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n"
|
||||
BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n"
|
||||
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n"
|
||||
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n"
|
||||
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**"
|
||||
dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
|
||||
fi
|
||||
else
|
||||
log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信)
|
||||
# ============================================================================
|
||||
DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;" | head -1)
|
||||
if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then
|
||||
BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> 请立即检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`"
|
||||
dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');"
|
||||
fi
|
||||
|
||||
CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;" | head -1)
|
||||
if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]] && [[ "${CURR_COUNT}" =~ ^[0-9]+$ ]]; then
|
||||
AVG_INT=${AVG_COUNT%%.*}
|
||||
DELTA=$(( CURR_COUNT - AVG_INT ))
|
||||
ABS_DELTA=${DELTA#-}
|
||||
if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then
|
||||
BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a"
|
||||
dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');"
|
||||
fi
|
||||
fi
|
||||
|
||||
DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
if [[ -n "${DISK_PCT}" ]] && [[ "${DISK_PCT}" =~ ^[0-9.]+$ ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then
|
||||
BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)"
|
||||
dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');"
|
||||
fi
|
||||
|
||||
INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
if [[ -n "${INODE_PCT}" ]] && [[ "${INODE_PCT}" =~ ^[0-9.]+$ ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then
|
||||
BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)"
|
||||
dingtalk_alert "P1" "DSH inode 告警" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');"
|
||||
fi
|
||||
|
||||
log_info "[ALERT] ========== 告警检测完成 =========="
|
||||
Reference in new issue
Block a user