chore: 初始化 - 拉取 DSH 生产监控脚本(单一事实源)
This commit is contained in:
commit
2e7504baf8
2 files changed
+124
No files matched your search
+18
@@ -0,0 +1,18 @@
|
|||||||
|
# 敏感/运行时产物
|
||||||
|
_state/
|
||||||
|
logs/
|
||||||
|
*.log
|
||||||
|
*_bak.*
|
||||||
|
*.bak.*
|
||||||
|
_oa_ssh.py
|
||||||
|
|
||||||
|
# 凭证
|
||||||
|
.env
|
||||||
|
*secret*
|
||||||
|
*credential*
|
||||||
|
|
||||||
|
# 构建产物
|
||||||
|
data/
|
||||||
|
*.db
|
||||||
|
*.db-shm
|
||||||
|
*.db-wal
|
||||||
Executable
+106
@@ -0,0 +1,106 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# ============================================================================
|
||||||
|
# OA Monitor · 全局配置与公共函数
|
||||||
|
# 所有采集/告警脚本 source 此文件,只修改这里的变量即可调整行为
|
||||||
|
# ============================================================================
|
||||||
|
set -u # 未定义变量报错
|
||||||
|
# 注意:不写 set -e,采集脚本单条命令失败不应整体中断
|
||||||
|
|
||||||
|
# ---- 路径 ----
|
||||||
|
export MONITOR_ROOT="/opt/oa-monitor"
|
||||||
|
export MONITOR_DATA="${MONITOR_ROOT}/data"
|
||||||
|
export MONITOR_LOGS="${MONITOR_ROOT}/logs"
|
||||||
|
export MONITOR_DB="${MONITOR_DATA}/oa-monitor.db"
|
||||||
|
|
||||||
|
mkdir -p "${MONITOR_DATA}" "${MONITOR_LOGS}"
|
||||||
|
|
||||||
|
# ---- 工具 ----
|
||||||
|
export MYSQL_CMD="mysql -h39.100.72.102 -P3306 -uecology"
|
||||||
|
export DB_NAME="ecology"
|
||||||
|
export DB_PASS='Z!BYT+jSDDyYHMdX+qFSLzJCYXW'
|
||||||
|
|
||||||
|
export SSH_DSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i ~/.ssh/id_ed25519 root@47.108.21.167"
|
||||||
|
|
||||||
|
# ---- OA 端点(DSH 上访问 OA 应用服务器内网 IP,更稳)----
|
||||||
|
OA_HTTP="http://39.99.236.138"
|
||||||
|
OA_ENDPOINTS=(
|
||||||
|
"${OA_HTTP}:80/wui/main/Login.jsp"
|
||||||
|
"${OA_HTTP}:8444/api/portal/portalinfo"
|
||||||
|
"${OA_HTTP}:8444/api/portal/sso/getLoginInfo.jsp"
|
||||||
|
"${OA_HTTP}/api/workflow/getRequestList"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---- 钉钉 Webhook(群机器人,2026-10-22)----
|
||||||
|
export DINGTALK_WEBHOOK="https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249"
|
||||||
|
export DINGTALK_SECRET=""
|
||||||
|
|
||||||
|
# ---- 告警阈值 ----
|
||||||
|
export ALERT_THREADS_RUNNING_HIGH=50 # 当前活跃线程 > 50 告警
|
||||||
|
export ALERT_SLOW_QUERY_SEC=30 # 慢查询 > 30s 告警
|
||||||
|
export ALERT_SLOW_QUERY_COUNT=5 # 同进程号出现 5 次以上告警
|
||||||
|
export ALERT_HTTP_FAIL_COUNT=3 # HTTP 端点连续失败 3 次告警
|
||||||
|
export ALERT_MSG_LOG_HOUR_MIN=100 # 1 小时消息增量 < 100 条告警(可能钉钉推送断了)
|
||||||
|
export ALERT_DOCKER_CONTAINER_DELTA=3 # 容器数突变 > ±3 告警
|
||||||
|
export ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN=1024 # Docker data-root 1 分钟写 > 1GB 告警
|
||||||
|
export ALERT_DISK_USAGE_PCT=85 # 磁盘使用率 > 85% 告警
|
||||||
|
export ALERT_INODE_USAGE_PCT=80 # inode 使用率 > 80% 告警
|
||||||
|
export ALERT_DOCKER_DAEMON_FAIL_COUNT=2 # Docker daemon 连续失败 2 次告警
|
||||||
|
export ALERT_LONG_QUERY_SEC=300 # 长查询 > 300s 触发 KILL(dry-run 默认)
|
||||||
|
|
||||||
|
# ---- 告警抑制 ----
|
||||||
|
export ALERT_COOLDOWN_SEC=600 # 同一告警类型 10 分钟内不重复发
|
||||||
|
|
||||||
|
# ---- SQLite 工具函数 ----
|
||||||
|
sqlite_exec() {
|
||||||
|
sqlite3 "${MONITOR_DB}" "$@" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---- 日志函数 ----
|
||||||
|
log() {
|
||||||
|
local level="$1"; shift
|
||||||
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MONITOR_LOGS}/oa-monitor.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_info() { log "INFO" "$@"; }
|
||||||
|
log_warn() { log "WARN" "$@"; }
|
||||||
|
log_error() { log "ERROR" "$@"; }
|
||||||
|
|
||||||
|
# ---- 钉钉告警(markdown 格式)----
|
||||||
|
# 用法:dingtalk_alert <level> <title> <markdown_body>
|
||||||
|
# level: P0 / P1 / P2
|
||||||
|
dingtalk_alert() {
|
||||||
|
local level="$1" title="$2" body="$3"
|
||||||
|
local timestamp
|
||||||
|
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||||
|
|
||||||
|
# 告警冷却检查
|
||||||
|
local key="${level}:${title}"
|
||||||
|
local last_ts
|
||||||
|
last_ts=$(sqlite_exec "SELECT COALESCE(last_alert_ts, 0) FROM alert_cooldown WHERE alert_key='${key}';")
|
||||||
|
local now_epoch
|
||||||
|
now_epoch=$(date +%s)
|
||||||
|
if [[ "${last_ts}" =~ ^[0-9]+$ ]] && (( now_epoch - last_ts < ALERT_COOLDOWN_SEC )); then
|
||||||
|
log_info "告警抑制:${key}(距上次 ${now_epoch}s < ${ALERT_COOLDOWN_SEC}s)"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 构造 JSON
|
||||||
|
local json
|
||||||
|
json=$(cat <<JSON
|
||||||
|
{"msgtype":"markdown","markdown":{"title":"[OA监控] ${level} ${title}","text":"### [OA监控] **${level}** ${title}\n\n**时间**:${timestamp}\n\n**主机**:DSH (47.108.21.167)\n\n---\n\n${body}\n\n---\n\n> OA 运维监控平台 · $(basename "$0")"}}
|
||||||
|
JSON
|
||||||
|
)
|
||||||
|
|
||||||
|
# 发送
|
||||||
|
local http_code
|
||||||
|
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "${json}" "${DINGTALK_WEBHOOK}" 2>/dev/null)
|
||||||
|
|
||||||
|
if [[ "${http_code}" == "200" ]]; then
|
||||||
|
log_info "钉钉告警发送成功:[${level}] ${title}"
|
||||||
|
sqlite_exec "INSERT OR REPLACE INTO alert_cooldown VALUES ('${key}', ${now_epoch});"
|
||||||
|
else
|
||||||
|
log_error "钉钉告警发送失败(HTTP ${http_code}):[${level}] ${title}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
Reference in new issue
Block a user