feat: watchdog v4 + check-alert v2 + collect-business v2 + DUMMY补数据脚本
This commit is contained in:
commit
7ab75364b1
22 files changed
+1795
No files matched your search
Executable
+155
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 告警检测(每 1 分钟)
|
||||
# 遍历所有指标,超过阈值触发钉钉告警
|
||||
# 告警冷却由 common.sh 的 alert_cooldown 表处理(10 分钟同类型不重复)
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
log_info "[ALERT] ========== 告警检测开始 =========="
|
||||
|
||||
# ============================================================================
|
||||
# 1. DB 层检测
|
||||
# ============================================================================
|
||||
# Threads_running 过高
|
||||
TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
if [[ -n "${TR}" ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then
|
||||
BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;')\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;')"
|
||||
dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');"
|
||||
fi
|
||||
|
||||
# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的)
|
||||
LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;")
|
||||
if [[ -n "${LONG_QUERIES}" ]]; then
|
||||
COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ')
|
||||
DETAIL=""
|
||||
while IFS='|' read -r pid time user state info; do
|
||||
DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`"
|
||||
done <<< "${LONG_QUERIES}"
|
||||
BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理"
|
||||
dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 2. HTTP 端点检测
|
||||
# ============================================================================
|
||||
# 最近 3 次的失败次数
|
||||
FAIL_COUNT=0
|
||||
FAIL_DETAIL=""
|
||||
for endpoint in "${OA_ENDPOINTS[@]}"; do
|
||||
FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;")
|
||||
FAIL_HERE=${FAIL_HERE:-0}
|
||||
if [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then
|
||||
FAIL_COUNT=$((FAIL_COUNT + 1))
|
||||
LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;")
|
||||
FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${FAIL_COUNT} -gt 0 ]]; then
|
||||
BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态"
|
||||
dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 3. 业务层检测
|
||||
# ============================================================================
|
||||
# 凌晨没人操作 OA 时 msg=0 本来正常——加时间窗口和业务活跃双重判断
|
||||
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
HOUR_MSG=${HOUR_MSG:-0}
|
||||
ONGOING=${ONGOING:-0}
|
||||
|
||||
# 当前小时数(用于判断是否工作时间)
|
||||
CURR_HOUR=$(date +%H | sed 's/^0//') # 去掉前导零
|
||||
WORK_HOURS_ENABLED=false
|
||||
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
|
||||
WORK_HOURS_ENABLED=true # 08:00~22:00 才算工作时间
|
||||
fi
|
||||
|
||||
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
|
||||
# 只保留 P0:工作时间内,有业务在跑,但消息一条都没推出来 → 推送链路完全断裂
|
||||
# P1 已砍掉(噪音太大:上午刚开始上班 msg=99 也会触发,实际推送是正常的)
|
||||
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then
|
||||
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
|
||||
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
||||
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n"
|
||||
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n"
|
||||
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n\n"
|
||||
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n"
|
||||
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n"
|
||||
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 `ps aux | grep Resin` → `bash /usr/weaver/Resin/bin/stopresin.sh && startresin.sh`**"
|
||||
if dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"; then
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log_info "[BIZ] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警检查"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 4. 基础设施层检测
|
||||
# ============================================================================
|
||||
# Docker daemon 挂了
|
||||
DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;")
|
||||
if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then
|
||||
BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> ⚠️ 今天的 rm -rf /ipaas 事故中 Docker daemon 重启过。请立即登录 DSH 检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`"
|
||||
dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');"
|
||||
fi
|
||||
|
||||
# 容器数突变(对比最近 30 分钟的平均数)
|
||||
CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;")
|
||||
if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]]; then
|
||||
DELTA=$(( CURR_COUNT - ${AVG_COUNT%%.*} ))
|
||||
ABS_DELTA=${DELTA#-}
|
||||
if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then
|
||||
BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a"
|
||||
dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 磁盘
|
||||
DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
if [[ -n "${DISK_PCT}" ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then
|
||||
BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)\n\n> 请检查:df -h → du -sh /var/lib/docker/* → pm2 logs 清理 → 旧 SQLite 数据归档"
|
||||
dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');"
|
||||
fi
|
||||
|
||||
# inode
|
||||
INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
if [[ -n "${INODE_PCT}" ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then
|
||||
BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)\n\n> 可能有大量小文件,检查 find / -type f | wc -l"
|
||||
dingtalk_alert "P1" "DSH inode 告警" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');"
|
||||
fi
|
||||
|
||||
# Docker root 写入突增(读最新一条和上一条的差值)
|
||||
ROOT_NOW=$(sqlite_exec "SELECT docker_root_write_mb FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
ROOT_PREV=$(sqlite_exec "SELECT docker_root_write_mb FROM infra_metrics ORDER BY ts DESC LIMIT 1 OFFSET 1;")
|
||||
ROOT_DELTA=0
|
||||
if [[ -n "${ROOT_NOW}" ]] && [[ -n "${ROOT_PREV}" ]]; then
|
||||
# 截断小数,bash $(()) 只认整数
|
||||
ROOT_NOW_INT=${ROOT_NOW%%.*}
|
||||
ROOT_PREV_INT=${ROOT_PREV%%.*}
|
||||
if [[ "${ROOT_NOW_INT}" =~ ^[0-9]+$ ]] && [[ "${ROOT_PREV_INT}" =~ ^[0-9]+$ ]]; then
|
||||
ROOT_DELTA=$(( ROOT_NOW_INT - ROOT_PREV_INT ))
|
||||
fi
|
||||
fi
|
||||
if [[ ${ROOT_DELTA} -ge ${ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN} ]]; then
|
||||
BODY="**Docker overlay2 增长 ${ROOT_DELTA}MB**(阈值 ${ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN}MB)\n\n> 可能有容器在疯狂写数据,检查:docker stats && docker logs"
|
||||
dingtalk_alert "P2" "Docker data-root 写入突增" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P2', 'infra', 'Docker root 写入突增', 'delta_mb=${ROOT_DELTA}');"
|
||||
fi
|
||||
|
||||
log_info "[ALERT] ========== 告警检测完成 =========="
|
||||
Reference in new issue
Block a user