Files
oa-monitor/bin/run-long-check.sh
T
gsadmin 90be249a72 fix: 修复 check-alert.sh 假阳性 + collect-business.sh 窗口对齐
- check-alert.sh 新增 Step0 数据新鲜度前置检查:business_metrics 超 30min 过期 →
  先告警采集脚本挂了,跳过业务层检测,防止读旧数据乱报推送断档
- collect-business.sh msg_log_hourly 窗口从 1h 改为 15min(和 crontab 频率对齐)
- CURR_HOUR 用 10# 算术强制十进制,防 08/09 被解析成 octal
2026-10-28 18:38:13 +08:00

67 lines
2.2 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 长查询检测 dry-run KILL(每 10 分钟)
# 默认 DRY_RUN=1(只报告,不真 KILL)。--execute 才真的 KILL
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
DRY_RUN=1
for arg in "$@"; do
[[ "${arg}" == "--execute" ]] && DRY_RUN=0
[[ "${arg}" == "--dry-run" ]] && DRY_RUN=1
done
NOW=$(date +%s)
MYSQL_RUN() {
MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology "$@" 2>/dev/null
}
log_info "[LONG-CHECK] 开始(DRY_RUN=${DRY_RUN},阈值 ${ALERT_LONG_QUERY_SEC}s)"
RESULT=$(MYSQL_RUN "SHOW FULL PROCESSLIST;" | awk -F'\t' -v t="${ALERT_LONG_QUERY_SEC}" '
NR==1 { next }
NF < 10 { next }
{
pid=$1; user=$2; state=$6; time=$7; info=$9
if (time+0 >= t+0 && time+0 > 0) {
gsub(/\t/, " ", info)
printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300)
}
}
')
if [[ -z "${RESULT}" ]]; then
log_info "[LONG-CHECK] ✅ 无长查询"
exit 0
fi
COUNT=$(echo "${RESULT}" | wc -l | tr -d ' ')
log_warn "[LONG-CHECK] ⚠️ ${COUNT} 条长查询"
DETAIL=""
while IFS=$'\t' read -r pid time user state info; do
[[ -z "${pid}" ]] && continue
DETAIL="${DETAIL}\n- PID \`${pid}\` **${time}s** \`${user}\` ${state}"
DETAIL="${DETAIL}\n \`\`\`\n ${info}\n \`\`\`"
done <<< "${RESULT}"
MODE_LABEL=$([[ ${DRY_RUN} -eq 1 ]] && echo "🔒 DRY-RUN" || echo "🚨 执行 KILL")
BODY="**${COUNT} 条长查询**(${ALERT_LONG_QUERY_SEC}s 阈值)· ${MODE_LABEL}\n\n${DETAIL}"
dingtalk_alert "P0" "长查询阻塞需处理" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询', '${COUNT} 条, dry_run=${DRY_RUN}');"
if [[ ${DRY_RUN} -eq 0 ]]; then
log_warn "[LONG-CHECK] 执行 KILL!"
while IFS=$'\t' read -r pid time user state info; do
[[ -z "${pid}" ]] && continue
MYSQL_RUN "KILL ${pid};"
log_warn "[LONG-CHECK] KILL ${pid} (${time}s, ${user})"
done <<< "${RESULT}"
else
log_info "[LONG-CHECK] DRY-RUN,不执行 KILL"
fi