commit 2e7504baf8417a6419c1cc716bd113da3b99d5f3 Author: gsadmin Date: Wed Oct 28 18:36:28 2026 +0800 chore: 初始化 - 拉取 DSH 生产监控脚本(单一事实源) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7ecfd60 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +# 敏感/运行时产物 +_state/ +logs/ +*.log +*_bak.* +*.bak.* +_oa_ssh.py + +# 凭证 +.env +*secret* +*credential* + +# 构建产物 +data/ +*.db +*.db-shm +*.db-wal diff --git a/lib/common.sh b/lib/common.sh new file mode 100755 index 0000000..a26fbdc --- /dev/null +++ b/lib/common.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 全局配置与公共函数 +# 所有采集/告警脚本 source 此文件,只修改这里的变量即可调整行为 +# ============================================================================ +set -u # 未定义变量报错 +# 注意:不写 set -e,采集脚本单条命令失败不应整体中断 + +# ---- 路径 ---- +export MONITOR_ROOT="/opt/oa-monitor" +export MONITOR_DATA="${MONITOR_ROOT}/data" +export MONITOR_LOGS="${MONITOR_ROOT}/logs" +export MONITOR_DB="${MONITOR_DATA}/oa-monitor.db" + +mkdir -p "${MONITOR_DATA}" "${MONITOR_LOGS}" + +# ---- 工具 ---- +export MYSQL_CMD="mysql -h39.100.72.102 -P3306 -uecology" +export DB_NAME="ecology" +export DB_PASS='Z!BYT+jSDDyYHMdX+qFSLzJCYXW' + +export SSH_DSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i ~/.ssh/id_ed25519 root@47.108.21.167" + +# ---- OA 端点(DSH 上访问 OA 应用服务器内网 IP,更稳)---- +OA_HTTP="http://39.99.236.138" +OA_ENDPOINTS=( + "${OA_HTTP}:80/wui/main/Login.jsp" + "${OA_HTTP}:8444/api/portal/portalinfo" + "${OA_HTTP}:8444/api/portal/sso/getLoginInfo.jsp" + "${OA_HTTP}/api/workflow/getRequestList" +) + +# ---- 钉钉 Webhook(群机器人,2026-10-22)---- +export DINGTALK_WEBHOOK="https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249" +export DINGTALK_SECRET="" + +# ---- 告警阈值 ---- +export ALERT_THREADS_RUNNING_HIGH=50 # 当前活跃线程 > 50 告警 +export ALERT_SLOW_QUERY_SEC=30 # 慢查询 > 30s 告警 +export ALERT_SLOW_QUERY_COUNT=5 # 同进程号出现 5 次以上告警 +export ALERT_HTTP_FAIL_COUNT=3 # HTTP 端点连续失败 3 次告警 +export ALERT_MSG_LOG_HOUR_MIN=100 # 1 小时消息增量 < 100 条告警(可能钉钉推送断了) +export ALERT_DOCKER_CONTAINER_DELTA=3 # 容器数突变 > ±3 告警 +export ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN=1024 # Docker data-root 1 分钟写 > 1GB 告警 +export ALERT_DISK_USAGE_PCT=85 # 磁盘使用率 > 85% 告警 +export ALERT_INODE_USAGE_PCT=80 # inode 使用率 > 80% 告警 +export ALERT_DOCKER_DAEMON_FAIL_COUNT=2 # Docker daemon 连续失败 2 次告警 +export ALERT_LONG_QUERY_SEC=300 # 长查询 > 300s 触发 KILL(dry-run 默认) + +# ---- 告警抑制 ---- +export ALERT_COOLDOWN_SEC=600 # 同一告警类型 10 分钟内不重复发 + +# ---- SQLite 工具函数 ---- +sqlite_exec() { + sqlite3 "${MONITOR_DB}" "$@" 2>/dev/null +} + +# ---- 日志函数 ---- +log() { + local level="$1"; shift + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MONITOR_LOGS}/oa-monitor.log" +} + +log_info() { log "INFO" "$@"; } +log_warn() { log "WARN" "$@"; } +log_error() { log "ERROR" "$@"; } + +# ---- 钉钉告警(markdown 格式)---- +# 用法:dingtalk_alert <markdown_body> +# level: P0 / P1 / P2 +dingtalk_alert() { + local level="$1" title="$2" body="$3" + local timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + + # 告警冷却检查 + local key="${level}:${title}" + local last_ts + last_ts=$(sqlite_exec "SELECT COALESCE(last_alert_ts, 0) FROM alert_cooldown WHERE alert_key='${key}';") + local now_epoch + now_epoch=$(date +%s) + if [[ "${last_ts}" =~ ^[0-9]+$ ]] && (( now_epoch - last_ts < ALERT_COOLDOWN_SEC )); then + log_info "告警抑制:${key}(距上次 ${now_epoch}s < ${ALERT_COOLDOWN_SEC}s)" + return 0 + fi + + # 构造 JSON + local json + json=$(cat <<JSON +{"msgtype":"markdown","markdown":{"title":"[OA监控] ${level} ${title}","text":"### [OA监控] **${level}** ${title}\n\n**时间**:${timestamp}\n\n**主机**:DSH (47.108.21.167)\n\n---\n\n${body}\n\n---\n\n> OA 运维监控平台 · $(basename "$0")"}} +JSON +) + + # 发送 + local http_code + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Content-Type: application/json" \ + -d "${json}" "${DINGTALK_WEBHOOK}" 2>/dev/null) + + if [[ "${http_code}" == "200" ]]; then + log_info "钉钉告警发送成功:[${level}] ${title}" + sqlite_exec "INSERT OR REPLACE INTO alert_cooldown VALUES ('${key}', ${now_epoch});" + else + log_error "钉钉告警发送失败(HTTP ${http_code}):[${level}] ${title}" + fi +}