fix: 修复 check-alert.sh 假阳性 + collect-business.sh 窗口对齐
- check-alert.sh 新增 Step0 数据新鲜度前置检查:business_metrics 超 30min 过期 → 先告警采集脚本挂了,跳过业务层检测,防止读旧数据乱报推送断档 - collect-business.sh msg_log_hourly 窗口从 1h 改为 15min(和 crontab 频率对齐) - CURR_HOUR 用 10# 算术强制十进制,防 08/09 被解析成 octal
This commit is contained in:
1 parent
2e7504baf8
commit
90be249a72
10 files changed
+829
No files matched your search
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 长查询检测 dry-run KILL(每 10 分钟)
|
||||
# 默认 DRY_RUN=1(只报告,不真 KILL)。--execute 才真的 KILL
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
DRY_RUN=1
|
||||
for arg in "$@"; do
|
||||
[[ "${arg}" == "--execute" ]] && DRY_RUN=0
|
||||
[[ "${arg}" == "--dry-run" ]] && DRY_RUN=1
|
||||
done
|
||||
|
||||
NOW=$(date +%s)
|
||||
|
||||
MYSQL_RUN() {
|
||||
MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology "$@" 2>/dev/null
|
||||
}
|
||||
|
||||
log_info "[LONG-CHECK] 开始(DRY_RUN=${DRY_RUN},阈值 ${ALERT_LONG_QUERY_SEC}s)"
|
||||
|
||||
RESULT=$(MYSQL_RUN "SHOW FULL PROCESSLIST;" | awk -F'\t' -v t="${ALERT_LONG_QUERY_SEC}" '
|
||||
NR==1 { next }
|
||||
NF < 10 { next }
|
||||
{
|
||||
pid=$1; user=$2; state=$6; time=$7; info=$9
|
||||
if (time+0 >= t+0 && time+0 > 0) {
|
||||
gsub(/\t/, " ", info)
|
||||
printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300)
|
||||
}
|
||||
}
|
||||
')
|
||||
|
||||
if [[ -z "${RESULT}" ]]; then
|
||||
log_info "[LONG-CHECK] ✅ 无长查询"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
COUNT=$(echo "${RESULT}" | wc -l | tr -d ' ')
|
||||
log_warn "[LONG-CHECK] ⚠️ ${COUNT} 条长查询"
|
||||
|
||||
DETAIL=""
|
||||
while IFS=$'\t' read -r pid time user state info; do
|
||||
[[ -z "${pid}" ]] && continue
|
||||
DETAIL="${DETAIL}\n- PID \`${pid}\` **${time}s** \`${user}\` ${state}"
|
||||
DETAIL="${DETAIL}\n \`\`\`\n ${info}\n \`\`\`"
|
||||
done <<< "${RESULT}"
|
||||
|
||||
MODE_LABEL=$([[ ${DRY_RUN} -eq 1 ]] && echo "🔒 DRY-RUN" || echo "🚨 执行 KILL")
|
||||
BODY="**${COUNT} 条长查询**(${ALERT_LONG_QUERY_SEC}s 阈值)· ${MODE_LABEL}\n\n${DETAIL}"
|
||||
dingtalk_alert "P0" "长查询阻塞需处理" "${BODY}"
|
||||
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询', '${COUNT} 条, dry_run=${DRY_RUN}');"
|
||||
|
||||
if [[ ${DRY_RUN} -eq 0 ]]; then
|
||||
log_warn "[LONG-CHECK] 执行 KILL!"
|
||||
while IFS=$'\t' read -r pid time user state info; do
|
||||
[[ -z "${pid}" ]] && continue
|
||||
MYSQL_RUN "KILL ${pid};"
|
||||
log_warn "[LONG-CHECK] KILL ${pid} (${time}s, ${user})"
|
||||
done <<< "${RESULT}"
|
||||
else
|
||||
log_info "[LONG-CHECK] DRY-RUN,不执行 KILL"
|
||||
fi
|
||||
Reference in new issue
Block a user