feat: watchdog v4 + check-alert v2 + collect-business v2 + DUMMY补数据脚本

This commit is contained in:
oa-monitor committed 2026-11-04 17:25:42 +08:00
commit 7ab75364b1
22 files changed
+1795

No files matched your search

+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin Watchdog (v2 - python paramiko 版)
# 监控 Util_Message 线程活跃度(publishMessageList),超过 30 分钟无活动则自动重启 Resin
# 部署在 DSH 上,通过 python paramiko 到 OA 服务器(39.99.236.138)执行
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
OA_HOST="39.99.236.138"
OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
OA_LOG="/usr/weaver/Resin/log/stdout.log"
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
MAX_IDLE_MIN=30
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
NOW=$(date +%s)
# Step 1: 确保 python paramiko 可用
if ! python3 -c 'import paramiko' 2>/dev/null; then
MSG="[WATCHDOG] 🔴 python3 + paramiko 不可用!请执行: pip3 install paramiko"
log_error "${MSG}"
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
exit 1
fi
# Step 2: 写 _oa_ssh.py(paramiko + errors=replace 处理 GBK 乱码)
cat > "${OA_SSH_PY}" << 'PYEOF'
import paramiko, sys
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=10)
stdin, stdout, stderr = ssh.exec_command(cmd)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
sys.exit(1)
PYEOF
chmod +x "${OA_SSH_PY}"
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
# Step 3: 提取最后一个 publishMessageList 时间(tail -5000 ≈ 10 分钟覆盖)
LAST_RAW=$(${OA_SSH} "tail -5000 ${OA_LOG} 2>/dev/null | grep publishMessageList | tail -1" 2>/dev/null)
LAST_TS=$(echo "${LAST_RAW}" | grep -oE '[0-9]{4}\.[0-9]{2}\.[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}' | head -1)
if [[ -z "${LAST_TS}" ]]; then
log_warn "[WATCHDOG] ⚠️ stdout.log 里没找到 publishMessageList 记录,跳过本次"
exit 0
fi
# Step 4: 解析时间戳 → epoch
LAST_EPOCH=$(date -d "$(echo ${LAST_TS} | sed "s|\.|-|g")" +%s 2>/dev/null)
if [[ -z "${LAST_EPOCH}" ]]; then
log_warn "[WATCHDOG] ⚠️ 无法解析日志时间: ${LAST_TS},跳过本次"
exit 0
fi
IDLE_MIN=$(( (NOW - LAST_EPOCH) / 60 ))
log_info "[WATCHDOG] 最后 publishMessageList: ${LAST_TS},已空闲 ${IDLE_MIN} 分钟(阈值 ${MAX_IDLE_MIN})"
# Step 5: 判断是否需要重启
if [[ ${IDLE_MIN} -lt ${MAX_IDLE_MIN} ]]; then
log_info "[WATCHDOG] ✅ publishMessageList 活跃(最近 ${IDLE_MIN} 分钟前),Resin 健康"
exit 0
fi
# --- 触发重启 ---
log_warn "[WATCHDOG] 🔴 空闲 ${IDLE_MIN} 分钟 ≥ ${MAX_IDLE_MIN} 分钟!触发 Resin 重启!"
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
BODY="**🔴 Resin 被 watchdog 自动重启**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| publishMessageList 最后活动 | \`${LAST_TS}\` |\n"
BODY="${BODY}| 空闲时长 | **${IDLE_MIN} 分钟** |\n"
BODY="${BODY}| 新 Resin PID | \`${NEW_PID}\` |\n"
BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n\n"
BODY="${BODY}> ⚠️ 推送线程静默死掉,watchdog 自动恢复。"
BODY="${BODY}> 建议:泛微需要检查 Util_Message 线程为什么会静默挂掉。"
dingtalk_alert "P0" "Resin watchdog 自动重启(推送线程空闲 ${IDLE_MIN} 分钟)" "${BODY}"
log_info "[WATCHDOG] ✅ Resin 重启完成,新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"