Files
oa-monitor/bin/check-alert.sh.bak.v1.202610281838

156 lines
9.6 KiB
Bash
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 告警检测(每 1 分钟)
# 遍历所有指标,超过阈值触发钉钉告警
# 告警冷却由 common.sh 的 alert_cooldown 表处理(10 分钟同类型不重复)
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
NOW=$(date +%s)
log_info "[ALERT] ========== 告警检测开始 =========="
# ============================================================================
# 1. DB 层检测
# ============================================================================
# Threads_running 过高
TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${TR}" ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then
BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;')\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;')"
dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');"
fi
# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的)
LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;")
if [[ -n "${LONG_QUERIES}" ]]; then
COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ')
DETAIL=""
while IFS='|' read -r pid time user state info; do
DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`"
done <<< "${LONG_QUERIES}"
BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理"
dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');"
fi
# ============================================================================
# 2. HTTP 端点检测
# ============================================================================
# 最近 3 次的失败次数
FAIL_COUNT=0
FAIL_DETAIL=""
for endpoint in "${OA_ENDPOINTS[@]}"; do
FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;")
FAIL_HERE=${FAIL_HERE:-0}
if [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then
FAIL_COUNT=$((FAIL_COUNT + 1))
LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;")
FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}"
fi
done
if [[ ${FAIL_COUNT} -gt 0 ]]; then
BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态"
dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');"
fi
# ============================================================================
# 3. 业务层检测
# ============================================================================
# 凌晨没人操作 OA 时 msg=0 本来正常——加时间窗口和业务活跃双重判断
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;")
ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;")
HOUR_MSG=${HOUR_MSG:-0}
ONGOING=${ONGOING:-0}
# 当前小时数(用于判断是否工作时间)
CURR_HOUR=$(date +%H | sed 's/^0//') # 去掉前导零
WORK_HOURS_ENABLED=false
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
WORK_HOURS_ENABLED=true # 08:00~22:00 才算工作时间
fi
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
# 只保留 P0:工作时间内,有业务在跑,但消息一条都没推出来 → 推送链路完全断裂
# P1 已砍掉(噪音太大:上午刚开始上班 msg=99 也会触发,实际推送是正常的)
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n"
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n"
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n\n"
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n"
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n"
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 `ps aux | grep Resin` → `bash /usr/weaver/Resin/bin/stopresin.sh && startresin.sh`**"
if dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"; then
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
fi
fi
else
log_info "[BIZ] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警检查"
fi
# ============================================================================
# 4. 基础设施层检测
# ============================================================================
# Docker daemon 挂了
DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;")
if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then
BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> ⚠️ 今天的 rm -rf /ipaas 事故中 Docker daemon 重启过。请立即登录 DSH 检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`"
dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');"
fi
# 容器数突变(对比最近 30 分钟的平均数)
CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;")
if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]]; then
DELTA=$(( CURR_COUNT - ${AVG_COUNT%%.*} ))
ABS_DELTA=${DELTA#-}
if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then
BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a"
dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');"
fi
fi
# 磁盘
DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${DISK_PCT}" ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then
BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)\n\n> 请检查:df -h → du -sh /var/lib/docker/* → pm2 logs 清理 → 旧 SQLite 数据归档"
dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');"
fi
# inode
INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${INODE_PCT}" ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then
BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)\n\n> 可能有大量小文件,检查 find / -type f | wc -l"
dingtalk_alert "P1" "DSH inode 告警" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');"
fi
# Docker root 写入突增(读最新一条和上一条的差值)
ROOT_NOW=$(sqlite_exec "SELECT docker_root_write_mb FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
ROOT_PREV=$(sqlite_exec "SELECT docker_root_write_mb FROM infra_metrics ORDER BY ts DESC LIMIT 1 OFFSET 1;")
ROOT_DELTA=0
if [[ -n "${ROOT_NOW}" ]] && [[ -n "${ROOT_PREV}" ]]; then
# 截断小数,bash $(()) 只认整数
ROOT_NOW_INT=${ROOT_NOW%%.*}
ROOT_PREV_INT=${ROOT_PREV%%.*}
if [[ "${ROOT_NOW_INT}" =~ ^[0-9]+$ ]] && [[ "${ROOT_PREV_INT}" =~ ^[0-9]+$ ]]; then
ROOT_DELTA=$(( ROOT_NOW_INT - ROOT_PREV_INT ))
fi
fi
if [[ ${ROOT_DELTA} -ge ${ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN} ]]; then
BODY="**Docker overlay2 增长 ${ROOT_DELTA}MB**(阈值 ${ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN}MB)\n\n> 可能有容器在疯狂写数据,检查:docker stats && docker logs"
dingtalk_alert "P2" "Docker data-root 写入突增" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P2', 'infra', 'Docker root 写入突增', 'delta_mb=${ROOT_DELTA}');"
fi
log_info "[ALERT] ========== 告警检测完成 =========="