fix: check-alert v3 修复钉钉推送断档 P0 假阳性

根因:sqlite_exec 吞掉 stderr(2>/dev/null),查询失败返回空串,
0 默认为 0 导致误报

修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,
查询失败跳过检测并 warn 日志
This commit is contained in:
oa-monitor committed 2026-11-04 19:41:18 +08:00
1 parent 7ab75364b1
commit 20fa48fb34
17 files changed
+1747 -26

No files matched your search

+106
View File
@@ -0,0 +1,106 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 全局配置与公共函数
# 所有采集/告警脚本 source 此文件,只修改这里的变量即可调整行为
# ============================================================================
set -u # 未定义变量报错
# 注意:不写 set -e,采集脚本单条命令失败不应整体中断
# ---- 路径 ----
export MONITOR_ROOT="/opt/oa-monitor"
export MONITOR_DATA="${MONITOR_ROOT}/data"
export MONITOR_LOGS="${MONITOR_ROOT}/logs"
export MONITOR_DB="${MONITOR_DATA}/oa-monitor.db"
mkdir -p "${MONITOR_DATA}" "${MONITOR_LOGS}"
# ---- 工具 ----
export MYSQL_CMD="mysql -h39.100.72.102 -P3306 -uecology"
export DB_NAME="ecology"
export DB_PASS='Z!BYT+jSDDyYHMdX+qFSLzJCYXW'
export SSH_DSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i ~/.ssh/id_ed25519 root@47.108.21.167"
# ---- OA 端点(DSH 上访问 OA 应用服务器内网 IP,更稳)----
OA_HTTP="http://39.99.236.138"
OA_ENDPOINTS=(
"${OA_HTTP}:80/wui/main/Login.jsp"
"${OA_HTTP}:8444/api/portal/portalinfo"
"${OA_HTTP}:8444/api/portal/sso/getLoginInfo.jsp"
"${OA_HTTP}/api/workflow/getRequestList"
)
# ---- 钉钉 Webhook(群 723387268,走 DSH 本地转发)----
export DINGTALK_WEBHOOK="https://dsh.gs-robot.com/dingtalk/webhook"
export DINGTALK_SECRET=""
# ---- 告警阈值 ----
export ALERT_THREADS_RUNNING_HIGH=50 # 当前活跃线程 > 50 告警
export ALERT_SLOW_QUERY_SEC=30 # 慢查询 > 30s 告警
export ALERT_SLOW_QUERY_COUNT=5 # 同进程号出现 5 次以上告警
export ALERT_HTTP_FAIL_COUNT=3 # HTTP 端点连续失败 3 次告警
export ALERT_MSG_LOG_HOUR_MIN=100 # 1 小时消息增量 < 100 条告警(可能钉钉推送断了)
export ALERT_DOCKER_CONTAINER_DELTA=3 # 容器数突变 > ±3 告警
export ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN=1024 # Docker data-root 1 分钟写 > 1GB 告警
export ALERT_DISK_USAGE_PCT=85 # 磁盘使用率 > 85% 告警
export ALERT_INODE_USAGE_PCT=80 # inode 使用率 > 80% 告警
export ALERT_DOCKER_DAEMON_FAIL_COUNT=2 # Docker daemon 连续失败 2 次告警
export ALERT_LONG_QUERY_SEC=300 # 长查询 > 300s 触发 KILL(dry-run 默认)
# ---- 告警抑制 ----
export ALERT_COOLDOWN_SEC=600 # 同一告警类型 10 分钟内不重复发
# ---- SQLite 工具函数 ----
sqlite_exec() {
sqlite3 "${MONITOR_DB}" "$@" 2>/dev/null
}
# ---- 日志函数 ----
log() {
local level="$1"; shift
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MONITOR_LOGS}/oa-monitor.log"
}
log_info() { log "INFO" "$@"; }
log_warn() { log "WARN" "$@"; }
log_error() { log "ERROR" "$@"; }
# ---- 钉钉告警(markdown 格式)----
# 用法:dingtalk_alert <level> <title> <markdown_body>
# level: P0 / P1 / P2
dingtalk_alert() {
local level="$1" title="$2" body="$3"
local timestamp
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
# 告警冷却检查
local key="${level}:${title}"
local last_ts
last_ts=$(sqlite_exec "SELECT COALESCE(last_alert_ts, 0) FROM alert_cooldown WHERE alert_key='${key}';")
local now_epoch
now_epoch=$(date +%s)
if [[ "${last_ts}" =~ ^[0-9]+$ ]] && (( now_epoch - last_ts < ALERT_COOLDOWN_SEC )); then
log_info "告警抑制:${key}(距上次 ${now_epoch}s < ${ALERT_COOLDOWN_SEC}s)"
return 0
fi
# 构造 JSON
local json
json=$(cat <<JSON
{"msgtype":"markdown","markdown":{"title":"[OA监控] ${level} ${title}","text":"### [OA监控] **${level}** ${title}\n\n**时间**:${timestamp}\n\n**主机**:DSH (47.108.21.167)\n\n---\n\n${body}\n\n---\n\n> OA 运维监控平台 · $(basename "$0")"}}
JSON
)
# 发送
local http_code
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Content-Type: application/json" \
-d "${json}" "${DINGTALK_WEBHOOK}" 2>/dev/null)
if [[ "${http_code}" == "200" ]]; then
log_info "钉钉告警发送成功:[${level}] ${title}"
sqlite_exec "INSERT OR REPLACE INTO alert_cooldown VALUES ('${key}', ${now_epoch});"
else
log_error "钉钉告警发送失败(HTTP ${http_code}):[${level}] ${title}"
fi
}