Files
oa-monitor/check-alert.sh
T
oa-monitor 20fa48fb34 fix: check-alert v3 修复钉钉推送断档 P0 假阳性
根因:sqlite_exec 吞掉 stderr(2>/dev/null),查询失败返回空串,
0 默认为 0 导致误报

修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,
查询失败跳过检测并 warn 日志
2026-11-04 19:41:18 +08:00

121 lines
8.0 KiB
Bash
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 告警检测(每 1 分钟)
# 遍历所有指标,超过阈值触发钉钉告警
# 告警冷却由 common.sh 的 alert_cooldown 表处理(10 分钟同类型不重复)
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
NOW=$(date +%s)
log_info "[ALERT] ========== 告警检测开始 =========="
# ============================================================================
# 1. DB 层检测
# ============================================================================
# Threads_running 过高
TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${TR}" ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then
BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;')\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;')"
dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');"
fi
# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的)
LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;")
if [[ -n "${LONG_QUERIES}" ]]; then
COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ')
DETAIL=""
while IFS='|' read -r pid time user state info; do
DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`"
done <<< "${LONG_QUERIES}"
BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理"
dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');"
fi
# ============================================================================
# 2. HTTP 端点检测
# ============================================================================
# 最近 3 次的失败次数
FAIL_COUNT=0
FAIL_DETAIL=""
for endpoint in "${OA_ENDPOINTS[@]}"; do
FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;")
FAIL_HERE=${FAIL_HERE:-0}
if [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then
FAIL_COUNT=$((FAIL_COUNT + 1))
LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;")
FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}"
fi
done
if [[ ${FAIL_COUNT} -gt 0 ]]; then
BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态"
dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');"
fi
# ============================================================================
# 3. 业务层检测
# ============================================================================
# message_log 小时增量过低(钉钉推送可能断了)
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${HOUR_MSG}" ]] && [[ ${HOUR_MSG} -lt ${ALERT_MSG_LOG_HOUR_MIN} ]] && [[ ${HOUR_MSG} -gt 0 ]]; then
BODY="**ecology_message_log 小时增量仅 ${HOUR_MSG} 条**(阈值 ${ALERT_MSG_LOG_HOUR_MIN})\n\n> 钉钉推送可能阻塞。泛微云桥 e-Bridge 为黑盒线程,需登录 OA 后台检查「消息推送」日志或用 SHOW PROCESSLIST 看 active 线程"
dingtalk_alert "P1" "钉钉推送疑似停滞" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'business', '钉钉推送疑似停滞', 'msg_hourly=${HOUR_MSG}');"
fi
# ============================================================================
# 4. 基础设施层检测
# ============================================================================
# Docker daemon 挂了
DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;")
if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then
BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> ⚠️ 今天的 rm -rf /ipaas 事故中 Docker daemon 重启过。请立即登录 DSH 检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`"
dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');"
fi
# 容器数突变(对比最近 30 分钟的平均数)
CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;")
if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]]; then
DELTA=$(( CURR_COUNT - ${AVG_COUNT%%.*} ))
ABS_DELTA=${DELTA#-}
if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then
BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a"
dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');"
fi
fi
# 磁盘
DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${DISK_PCT}" ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then
BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)\n\n> 请检查:df -h → du -sh /var/lib/docker/* → pm2 logs 清理 → 旧 SQLite 数据归档"
dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');"
fi
# inode
INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${INODE_PCT}" ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then
BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)\n\n> 可能有大量小文件,检查 find / -type f | wc -l"
dingtalk_alert "P1" "DSH inode 告警" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');"
fi
# Docker root 写入突增
ROOT_WRITE=$(sqlite_exec "SELECT docker_root_write_mb FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${ROOT_WRITE}" ]] && (( ${ROOT_WRITE%%.*} >= ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN )); then
BODY="**Docker data-root 1 分钟内写入 ${ROOT_WRITE}MB**(阈值 ${ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN}MB)\n\n> 可能有容器在疯狂写数据,检查:docker stats → docker logs"
dingtalk_alert "P2" "Docker data-root 写入突增" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P2', 'infra', 'Docker root 写入突增', 'write_mb=${ROOT_WRITE}');"
fi
log_info "[ALERT] ========== 告警检测完成 =========="