114 lines
4.5 KiB
Bash
Executable File
114 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# ============================================================================
|
||
# OA Monitor · 全局配置与公共函数
|
||
# 所有采集/告警脚本 source 此文件,只修改这里的变量即可调整行为
|
||
# ============================================================================
|
||
set -u # 未定义变量报错
|
||
# 注意:不写 set -e,采集脚本单条命令失败不应整体中断
|
||
|
||
# ---- 路径 ----
|
||
export MONITOR_ROOT="/opt/oa-monitor"
|
||
export MONITOR_DATA="${MONITOR_ROOT}/data"
|
||
export MONITOR_LOGS="${MONITOR_ROOT}/logs"
|
||
export MONITOR_DB="${MONITOR_DATA}/oa-monitor.db"
|
||
|
||
mkdir -p "${MONITOR_DATA}" "${MONITOR_LOGS}"
|
||
|
||
# ---- 工具 ----
|
||
export MYSQL_CMD="mysql -h39.100.72.102 -P3306 -uecology"
|
||
export DB_NAME="ecology"
|
||
export DB_PASS='Z!BYT+jSDDyYHMdX+qFSLzJCYXW'
|
||
|
||
export SSH_DSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i ~/.ssh/id_ed25519 root@47.108.21.167"
|
||
|
||
# ---- OA 端点(DSH 上访问 OA 应用服务器内网 IP,更稳)----
|
||
OA_HTTP="http://39.99.236.138"
|
||
OA_ENDPOINTS=(
|
||
"${OA_HTTP}:80/wui/main/Login.jsp"
|
||
"${OA_HTTP}:8444/api/portal/portalinfo"
|
||
"${OA_HTTP}:8444/api/portal/sso/getLoginInfo.jsp"
|
||
"${OA_HTTP}/api/workflow/getRequestList"
|
||
)
|
||
|
||
# ---- 钉钉 Webhook(群机器人,2026-10-22)----
|
||
export DINGTALK_WEBHOOK="https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249"
|
||
export DINGTALK_SECRET=""
|
||
|
||
# ---- 告警阈值 ----
|
||
export ALERT_THREADS_RUNNING_HIGH=50 # 当前活跃线程 > 50 告警
|
||
export ALERT_SLOW_QUERY_SEC=30 # 慢查询 > 30s 告警
|
||
export ALERT_SLOW_QUERY_COUNT=5 # 同进程号出现 5 次以上告警
|
||
export ALERT_HTTP_FAIL_COUNT=3 # HTTP 端点连续失败 3 次告警
|
||
export ALERT_MSG_LOG_HOUR_MIN=100 # 1 小时消息增量 < 100 条告警(可能钉钉推送断了)
|
||
export ALERT_DOCKER_CONTAINER_DELTA=3 # 容器数突变 > ±3 告警
|
||
export ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN=1024 # Docker data-root 1 分钟写 > 1GB 告警
|
||
export ALERT_DISK_USAGE_PCT=85 # 磁盘使用率 > 85% 告警
|
||
export ALERT_INODE_USAGE_PCT=80 # inode 使用率 > 80% 告警
|
||
export ALERT_DOCKER_DAEMON_FAIL_COUNT=2 # Docker daemon 连续失败 2 次告警
|
||
export ALERT_LONG_QUERY_SEC=300 # 长查询 > 300s 触发 KILL(dry-run 默认)
|
||
|
||
# ---- 告警抑制 ----
|
||
export ALERT_COOLDOWN_SEC=600 # 同一告警类型 10 分钟内不重复发
|
||
|
||
# ---- SQLite 工具函数 ----
|
||
sqlite_exec() {
|
||
sqlite3 "${MONITOR_DB}" "$@" 2>/dev/null
|
||
}
|
||
|
||
# ---- 日志函数 ----
|
||
log() {
|
||
local level="$1"; shift
|
||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MONITOR_LOGS}/oa-monitor.log"
|
||
}
|
||
|
||
log_info() { log "INFO" "$@"; }
|
||
log_warn() { log "WARN" "$@"; }
|
||
log_error() { log "ERROR" "$@"; }
|
||
|
||
# ---- 钉钉告警(markdown 格式)----
|
||
# 用法:dingtalk_alert <level> <title> <markdown_body>
|
||
# level: P0 / P1 / P2
|
||
dingtalk_alert() {
|
||
local level="$1" title="$2" body="$3"
|
||
local timestamp
|
||
# v2 降级(2026-11-05):只放行 P0,拦截 P1/P2 降噪
|
||
# 放行:watchdog count=0 自愈、DB 长查询、Docker 挂、推送断档
|
||
# 拦截:PID 异常、预防性重启、HTTP 故障、磁盘告警
|
||
if [[ "${level}" != "P0" ]]; then
|
||
log_info "[ALERT-LEVEL] 降级模式:拦截 ${level} ${title}"
|
||
return 0
|
||
fi
|
||
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
# 告警冷却检查
|
||
local key="${level}:${title}"
|
||
local last_ts
|
||
last_ts=$(sqlite_exec "SELECT COALESCE(last_alert_ts, 0) FROM alert_cooldown WHERE alert_key='${key}';")
|
||
local now_epoch
|
||
now_epoch=$(date +%s)
|
||
if [[ "${last_ts}" =~ ^[0-9]+$ ]] && (( now_epoch - last_ts < ALERT_COOLDOWN_SEC )); then
|
||
log_info "告警抑制:${key}(距上次 ${now_epoch}s < ${ALERT_COOLDOWN_SEC}s)"
|
||
return 0
|
||
fi
|
||
|
||
# 构造 JSON
|
||
local json
|
||
json=$(cat <<JSON
|
||
{"msgtype":"markdown","markdown":{"title":"[OA监控] ${level} ${title}","text":"### [OA监控] **${level}** ${title}\n\n**时间**:${timestamp}\n\n**主机**:DSH (47.108.21.167)\n\n---\n\n${body}\n\n---\n\n> OA 运维监控平台 · $(basename "$0")"}}
|
||
JSON
|
||
)
|
||
|
||
# 发送
|
||
local http_code
|
||
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||
-H "Content-Type: application/json" \
|
||
-d "${json}" "${DINGTALK_WEBHOOK}" 2>/dev/null)
|
||
|
||
if [[ "${http_code}" == "200" ]]; then
|
||
log_info "钉钉告警发送成功:[${level}] ${title}"
|
||
sqlite_exec "INSERT OR REPLACE INTO alert_cooldown VALUES ('${key}', ${now_epoch});"
|
||
else
|
||
log_error "钉钉告警发送失败(HTTP ${http_code}):[${level}] ${title}"
|
||
fi
|
||
}
|