Files

114 lines
4.5 KiB
Bash
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 全局配置与公共函数
# 所有采集/告警脚本 source 此文件,只修改这里的变量即可调整行为
# ============================================================================
set -u # 未定义变量报错
# 注意:不写 set -e,采集脚本单条命令失败不应整体中断
# ---- 路径 ----
export MONITOR_ROOT="/opt/oa-monitor"
export MONITOR_DATA="${MONITOR_ROOT}/data"
export MONITOR_LOGS="${MONITOR_ROOT}/logs"
export MONITOR_DB="${MONITOR_DATA}/oa-monitor.db"
mkdir -p "${MONITOR_DATA}" "${MONITOR_LOGS}"
# ---- 工具 ----
export MYSQL_CMD="mysql -h39.100.72.102 -P3306 -uecology"
export DB_NAME="ecology"
export DB_PASS='Z!BYT+jSDDyYHMdX+qFSLzJCYXW'
export SSH_DSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i ~/.ssh/id_ed25519 root@47.108.21.167"
# ---- OA 端点(DSH 上访问 OA 应用服务器内网 IP,更稳)----
OA_HTTP="http://39.99.236.138"
OA_ENDPOINTS=(
"${OA_HTTP}:80/wui/main/Login.jsp"
"${OA_HTTP}:8444/api/portal/portalinfo"
"${OA_HTTP}:8444/api/portal/sso/getLoginInfo.jsp"
"${OA_HTTP}/api/workflow/getRequestList"
)
# ---- 钉钉 Webhook(群机器人,2026-10-22)----
export DINGTALK_WEBHOOK="https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249"
export DINGTALK_SECRET=""
# ---- 告警阈值 ----
export ALERT_THREADS_RUNNING_HIGH=50 # 当前活跃线程 > 50 告警
export ALERT_SLOW_QUERY_SEC=30 # 慢查询 > 30s 告警
export ALERT_SLOW_QUERY_COUNT=5 # 同进程号出现 5 次以上告警
export ALERT_HTTP_FAIL_COUNT=3 # HTTP 端点连续失败 3 次告警
export ALERT_MSG_LOG_HOUR_MIN=100 # 1 小时消息增量 < 100 条告警(可能钉钉推送断了)
export ALERT_DOCKER_CONTAINER_DELTA=3 # 容器数突变 > ±3 告警
export ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN=1024 # Docker data-root 1 分钟写 > 1GB 告警
export ALERT_DISK_USAGE_PCT=85 # 磁盘使用率 > 85% 告警
export ALERT_INODE_USAGE_PCT=80 # inode 使用率 > 80% 告警
export ALERT_DOCKER_DAEMON_FAIL_COUNT=2 # Docker daemon 连续失败 2 次告警
export ALERT_LONG_QUERY_SEC=300 # 长查询 > 300s 触发 KILL(dry-run 默认)
# ---- 告警抑制 ----
export ALERT_COOLDOWN_SEC=600 # 同一告警类型 10 分钟内不重复发
# ---- SQLite 工具函数 ----
sqlite_exec() {
sqlite3 "${MONITOR_DB}" "$@" 2>/dev/null
}
# ---- 日志函数 ----
log() {
local level="$1"; shift
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MONITOR_LOGS}/oa-monitor.log"
}
log_info() { log "INFO" "$@"; }
log_warn() { log "WARN" "$@"; }
log_error() { log "ERROR" "$@"; }
# ---- 钉钉告警(markdown 格式)----
# 用法:dingtalk_alert <level> <title> <markdown_body>
# level: P0 / P1 / P2
dingtalk_alert() {
local level="$1" title="$2" body="$3"
local timestamp
# v2 降级(2026-11-05):只放行 P0,拦截 P1/P2 降噪
# 放行:watchdog count=0 自愈、DB 长查询、Docker 挂、推送断档
# 拦截:PID 异常、预防性重启、HTTP 故障、磁盘告警
if [[ "${level}" != "P0" ]]; then
log_info "[ALERT-LEVEL] 降级模式:拦截 ${level} ${title}"
return 0
fi
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
# 告警冷却检查
local key="${level}:${title}"
local last_ts
last_ts=$(sqlite_exec "SELECT COALESCE(last_alert_ts, 0) FROM alert_cooldown WHERE alert_key='${key}';")
local now_epoch
now_epoch=$(date +%s)
if [[ "${last_ts}" =~ ^[0-9]+$ ]] && (( now_epoch - last_ts < ALERT_COOLDOWN_SEC )); then
log_info "告警抑制:${key}(距上次 ${now_epoch}s < ${ALERT_COOLDOWN_SEC}s)"
return 0
fi
# 构造 JSON
local json
json=$(cat <<JSON
{"msgtype":"markdown","markdown":{"title":"[OA监控] ${level} ${title}","text":"### [OA监控] **${level}** ${title}\n\n**时间**:${timestamp}\n\n**主机**:DSH (47.108.21.167)\n\n---\n\n${body}\n\n---\n\n> OA 运维监控平台 · $(basename "$0")"}}
JSON
)
# 发送
local http_code
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Content-Type: application/json" \
-d "${json}" "${DINGTALK_WEBHOOK}" 2>/dev/null)
if [[ "${http_code}" == "200" ]]; then
log_info "钉钉告警发送成功:[${level}] ${title}"
sqlite_exec "INSERT OR REPLACE INTO alert_cooldown VALUES ('${key}', ${now_epoch});"
else
log_error "钉钉告警发送失败(HTTP ${http_code}):[${level}] ${title}"
fi
}