#!/usr/bin/env bash # ============================================================================ # OA Monitor · 告警检测(每 1 分钟) # 遍历所有指标,超过阈值触发钉钉告警 # 告警冷却由 common.sh 的 alert_cooldown 表处理(10 分钟同类型不重复) # ============================================================================ set -u SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "${SCRIPT_DIR}/../lib/common.sh" NOW=$(date +%s) log_info "[ALERT] ========== 告警检测开始 ==========" # ============================================================================ # 1. DB 层检测 # ============================================================================ # Threads_running 过高 TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;") if [[ -n "${TR}" ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;')\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;')" dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');" fi # 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的) LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;") if [[ -n "${LONG_QUERIES}" ]]; then COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ') DETAIL="" while IFS='|' read -r pid time user state info; do DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`" done <<< "${LONG_QUERIES}" BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理" dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');" fi # ============================================================================ # 2. HTTP 端点检测 # ============================================================================ # 最近 3 次的失败次数 FAIL_COUNT=0 FAIL_DETAIL="" for endpoint in "${OA_ENDPOINTS[@]}"; do FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;") FAIL_HERE=${FAIL_HERE:-0} if [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then FAIL_COUNT=$((FAIL_COUNT + 1)) LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;") FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}" fi done if [[ ${FAIL_COUNT} -gt 0 ]]; then BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态" dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');" fi # ============================================================================ # 3. 业务层检测 # ============================================================================ # message_log 小时增量过低(钉钉推送可能断了) HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;") if [[ -n "${HOUR_MSG}" ]] && [[ ${HOUR_MSG} -lt ${ALERT_MSG_LOG_HOUR_MIN} ]] && [[ ${HOUR_MSG} -gt 0 ]]; then BODY="**ecology_message_log 小时增量仅 ${HOUR_MSG} 条**(阈值 ${ALERT_MSG_LOG_HOUR_MIN})\n\n> 钉钉推送可能阻塞。泛微云桥 e-Bridge 为黑盒线程,需登录 OA 后台检查「消息推送」日志或用 SHOW PROCESSLIST 看 active 线程" dingtalk_alert "P1" "钉钉推送疑似停滞" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'business', '钉钉推送疑似停滞', 'msg_hourly=${HOUR_MSG}');" fi # ============================================================================ # 4. 基础设施层检测 # ============================================================================ # Docker daemon 挂了 DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;") PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;") if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> ⚠️ 今天的 rm -rf /ipaas 事故中 Docker daemon 重启过。请立即登录 DSH 检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`" dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');" fi # 容器数突变(对比最近 30 分钟的平均数) CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;") AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;") if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]]; then DELTA=$(( CURR_COUNT - ${AVG_COUNT%%.*} )) ABS_DELTA=${DELTA#-} if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a" dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');" fi fi # 磁盘 DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;") if [[ -n "${DISK_PCT}" ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)\n\n> 请检查:df -h → du -sh /var/lib/docker/* → pm2 logs 清理 → 旧 SQLite 数据归档" dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');" fi # inode INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;") if [[ -n "${INODE_PCT}" ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)\n\n> 可能有大量小文件,检查 find / -type f | wc -l" dingtalk_alert "P1" "DSH inode 告警" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');" fi # Docker root 写入突增 ROOT_WRITE=$(sqlite_exec "SELECT docker_root_write_mb FROM infra_metrics ORDER BY ts DESC LIMIT 1;") if [[ -n "${ROOT_WRITE}" ]] && (( ${ROOT_WRITE%%.*} >= ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN )); then BODY="**Docker data-root 1 分钟内写入 ${ROOT_WRITE}MB**(阈值 ${ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN}MB)\n\n> 可能有容器在疯狂写数据,检查:docker stats → docker logs" dingtalk_alert "P2" "Docker data-root 写入突增" "${BODY}" sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P2', 'infra', 'Docker root 写入突增', 'write_mb=${ROOT_WRITE}');" fi log_info "[ALERT] ========== 告警检测完成 =========="