#!/usr/bin/env bash # ============================================================================ # OA Monitor · 全局配置与公共函数 # 所有采集/告警脚本 source 此文件,只修改这里的变量即可调整行为 # ============================================================================ set -u # 未定义变量报错 # 注意:不写 set -e,采集脚本单条命令失败不应整体中断 # ---- 路径 ---- export MONITOR_ROOT="/opt/oa-monitor" export MONITOR_DATA="${MONITOR_ROOT}/data" export MONITOR_LOGS="${MONITOR_ROOT}/logs" export MONITOR_DB="${MONITOR_DATA}/oa-monitor.db" mkdir -p "${MONITOR_DATA}" "${MONITOR_LOGS}" # ---- 工具 ---- export MYSQL_CMD="mysql -h39.100.72.102 -P3306 -uecology" export DB_NAME="ecology" export DB_PASS='Z!BYT+jSDDyYHMdX+qFSLzJCYXW' export SSH_DSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i ~/.ssh/id_ed25519 root@47.108.21.167" # ---- OA 端点(DSH 上访问 OA 应用服务器内网 IP,更稳)---- OA_HTTP="http://39.99.236.138" OA_ENDPOINTS=( "${OA_HTTP}:80/wui/main/Login.jsp" "${OA_HTTP}:8444/api/portal/portalinfo" "${OA_HTTP}:8444/api/portal/sso/getLoginInfo.jsp" "${OA_HTTP}/api/workflow/getRequestList" ) # ---- 钉钉 Webhook(群 723387268,走 DSH 本地转发)---- export DINGTALK_WEBHOOK="https://dsh.gs-robot.com/dingtalk/webhook" export DINGTALK_SECRET="" # ---- 告警阈值 ---- export ALERT_THREADS_RUNNING_HIGH=50 # 当前活跃线程 > 50 告警 export ALERT_SLOW_QUERY_SEC=30 # 慢查询 > 30s 告警 export ALERT_SLOW_QUERY_COUNT=5 # 同进程号出现 5 次以上告警 export ALERT_HTTP_FAIL_COUNT=3 # HTTP 端点连续失败 3 次告警 export ALERT_MSG_LOG_HOUR_MIN=100 # 1 小时消息增量 < 100 条告警(可能钉钉推送断了) export ALERT_DOCKER_CONTAINER_DELTA=3 # 容器数突变 > ±3 告警 export ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN=1024 # Docker data-root 1 分钟写 > 1GB 告警 export ALERT_DISK_USAGE_PCT=85 # 磁盘使用率 > 85% 告警 export ALERT_INODE_USAGE_PCT=80 # inode 使用率 > 80% 告警 export ALERT_DOCKER_DAEMON_FAIL_COUNT=2 # Docker daemon 连续失败 2 次告警 export ALERT_LONG_QUERY_SEC=300 # 长查询 > 300s 触发 KILL(dry-run 默认) # ---- 告警抑制 ---- export ALERT_COOLDOWN_SEC=600 # 同一告警类型 10 分钟内不重复发 # ---- SQLite 工具函数 ---- sqlite_exec() { sqlite3 "${MONITOR_DB}" "$@" 2>/dev/null } # ---- 日志函数 ---- log() { local level="$1"; shift echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MONITOR_LOGS}/oa-monitor.log" } log_info() { log "INFO" "$@"; } log_warn() { log "WARN" "$@"; } log_error() { log "ERROR" "$@"; } # ---- 钉钉告警(markdown 格式)---- # 用法:dingtalk_alert <markdown_body> # level: P0 / P1 / P2 dingtalk_alert() { local level="$1" title="$2" body="$3" local timestamp timestamp=$(date '+%Y-%m-%d %H:%M:%S') # 告警冷却检查 local key="${level}:${title}" local last_ts last_ts=$(sqlite_exec "SELECT COALESCE(last_alert_ts, 0) FROM alert_cooldown WHERE alert_key='${key}';") local now_epoch now_epoch=$(date +%s) if [[ "${last_ts}" =~ ^[0-9]+$ ]] && (( now_epoch - last_ts < ALERT_COOLDOWN_SEC )); then log_info "告警抑制:${key}(距上次 ${now_epoch}s < ${ALERT_COOLDOWN_SEC}s)" return 0 fi # 构造 JSON local json json=$(cat <<JSON {"msgtype":"markdown","markdown":{"title":"[OA监控] ${level} ${title}","text":"### [OA监控] **${level}** ${title}\n\n**时间**:${timestamp}\n\n**主机**:DSH (47.108.21.167)\n\n---\n\n${body}\n\n---\n\n> OA 运维监控平台 · $(basename "$0")"}} JSON ) # 发送 local http_code http_code=$(curl -s -o /dev/null -w "%{http_code}" \ -H "Content-Type: application/json" \ -d "${json}" "${DINGTALK_WEBHOOK}" 2>/dev/null) if [[ "${http_code}" == "200" ]]; then log_info "钉钉告警发送成功:[${level}] ${title}" sqlite_exec "INSERT OR REPLACE INTO alert_cooldown VALUES ('${key}', ${now_epoch});" else log_error "钉钉告警发送失败(HTTP ${http_code}):[${level}] ${title}" fi }