fix: 修复 check-alert.sh 假阳性 + collect-business.sh 窗口对齐

- check-alert.sh 新增 Step0 数据新鲜度前置检查:business_metrics 超 30min 过期 →
  先告警采集脚本挂了,跳过业务层检测,防止读旧数据乱报推送断档
- collect-business.sh msg_log_hourly 窗口从 1h 改为 15min(和 crontab 频率对齐)
- CURR_HOUR 用 10# 算术强制十进制,防 08/09 被解析成 octal
This commit is contained in:
gsadmin authored and trae-solo-agent committed 2026-10-28 18:38:13 +08:00
1 parent 2e7504baf8
commit 90be249a72
10 files changed
+829

No files matched your search

+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 长查询检测 dry-run KILL(每 10 分钟)
# 默认 DRY_RUN=1(只报告,不真 KILL)。--execute 才真的 KILL
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
DRY_RUN=1
for arg in "$@"; do
[[ "${arg}" == "--execute" ]] && DRY_RUN=0
[[ "${arg}" == "--dry-run" ]] && DRY_RUN=1
done
NOW=$(date +%s)
MYSQL_RUN() {
MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology "$@" 2>/dev/null
}
log_info "[LONG-CHECK] 开始(DRY_RUN=${DRY_RUN},阈值 ${ALERT_LONG_QUERY_SEC}s)"
RESULT=$(MYSQL_RUN "SHOW FULL PROCESSLIST;" | awk -F'\t' -v t="${ALERT_LONG_QUERY_SEC}" '
NR==1 { next }
NF < 10 { next }
{
pid=$1; user=$2; state=$6; time=$7; info=$9
if (time+0 >= t+0 && time+0 > 0) {
gsub(/\t/, " ", info)
printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300)
}
}
')
if [[ -z "${RESULT}" ]]; then
log_info "[LONG-CHECK] ✅ 无长查询"
exit 0
fi
COUNT=$(echo "${RESULT}" | wc -l | tr -d ' ')
log_warn "[LONG-CHECK] ⚠️ ${COUNT} 条长查询"
DETAIL=""
while IFS=$'\t' read -r pid time user state info; do
[[ -z "${pid}" ]] && continue
DETAIL="${DETAIL}\n- PID \`${pid}\` **${time}s** \`${user}\` ${state}"
DETAIL="${DETAIL}\n \`\`\`\n ${info}\n \`\`\`"
done <<< "${RESULT}"
MODE_LABEL=$([[ ${DRY_RUN} -eq 1 ]] && echo "🔒 DRY-RUN" || echo "🚨 执行 KILL")
BODY="**${COUNT} 条长查询**(${ALERT_LONG_QUERY_SEC}s 阈值)· ${MODE_LABEL}\n\n${DETAIL}"
dingtalk_alert "P0" "长查询阻塞需处理" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询', '${COUNT} 条, dry_run=${DRY_RUN}');"
if [[ ${DRY_RUN} -eq 0 ]]; then
log_warn "[LONG-CHECK] 执行 KILL!"
while IFS=$'\t' read -r pid time user state info; do
[[ -z "${pid}" ]] && continue
MYSQL_RUN "KILL ${pid};"
log_warn "[LONG-CHECK] KILL ${pid} (${time}s, ${user})"
done <<< "${RESULT}"
else
log_info "[LONG-CHECK] DRY-RUN,不执行 KILL"
fi