Files
oa-monitor/bin/resin-watchdog.sh.bak.v2.202610261128
T

97 lines
4.0 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin Watchdog (v2 - python paramiko 版)
# 监控 Util_Message 线程活跃度(publishMessageList),超过 30 分钟无活动则自动重启 Resin
# 部署在 DSH 上,通过 python paramiko 到 OA 服务器(39.99.236.138)执行
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
OA_HOST="39.99.236.138"
OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
OA_LOG="/usr/weaver/Resin/log/stdout.log"
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
MAX_IDLE_MIN=30
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
NOW=$(date +%s)
# Step 1: 确保 python paramiko 可用
if ! python3 -c 'import paramiko' 2>/dev/null; then
MSG="[WATCHDOG] 🔴 python3 + paramiko 不可用!请执行: pip3 install paramiko"
log_error "${MSG}"
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
exit 1
fi
# Step 2: 写 _oa_ssh.py(paramiko + errors=replace 处理 GBK 乱码)
cat > "${OA_SSH_PY}" << 'PYEOF'
import paramiko, sys
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=10)
stdin, stdout, stderr = ssh.exec_command(cmd)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
sys.exit(1)
PYEOF
chmod +x "${OA_SSH_PY}"
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
# Step 3: 提取最后一个 publishMessageList 时间(tail -5000 ≈ 10 分钟覆盖)
LAST_RAW=$(${OA_SSH} "tail -5000 ${OA_LOG} 2>/dev/null | grep publishMessageList | tail -1" 2>/dev/null)
LAST_TS=$(echo "${LAST_RAW}" | grep -oE '[0-9]{4}\.[0-9]{2}\.[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}' | head -1)
if [[ -z "${LAST_TS}" ]]; then
log_warn "[WATCHDOG] ⚠️ stdout.log 里没找到 publishMessageList 记录,跳过本次"
exit 0
fi
# Step 4: 解析时间戳 → epoch
LAST_EPOCH=$(date -d "$(echo ${LAST_TS} | sed "s|\.|-|g")" +%s 2>/dev/null)
if [[ -z "${LAST_EPOCH}" ]]; then
log_warn "[WATCHDOG] ⚠️ 无法解析日志时间: ${LAST_TS},跳过本次"
exit 0
fi
IDLE_MIN=$(( (NOW - LAST_EPOCH) / 60 ))
log_info "[WATCHDOG] 最后 publishMessageList: ${LAST_TS},已空闲 ${IDLE_MIN} 分钟(阈值 ${MAX_IDLE_MIN})"
# Step 5: 判断是否需要重启
if [[ ${IDLE_MIN} -lt ${MAX_IDLE_MIN} ]]; then
log_info "[WATCHDOG] ✅ publishMessageList 活跃(最近 ${IDLE_MIN} 分钟前),Resin 健康"
exit 0
fi
# --- 触发重启 ---
log_warn "[WATCHDOG] 🔴 空闲 ${IDLE_MIN} 分钟 ≥ ${MAX_IDLE_MIN} 分钟!触发 Resin 重启!"
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
BODY="**🔴 Resin 被 watchdog 自动重启**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| publishMessageList 最后活动 | \`${LAST_TS}\` |\n"
BODY="${BODY}| 空闲时长 | **${IDLE_MIN} 分钟** |\n"
BODY="${BODY}| 新 Resin PID | \`${NEW_PID}\` |\n"
BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n\n"
BODY="${BODY}> ⚠️ 推送线程静默死掉,watchdog 自动恢复。"
BODY="${BODY}> 建议:泛微需要检查 Util_Message 线程为什么会静默挂掉。"
dingtalk_alert "P0" "Resin watchdog 自动重启(推送线程空闲 ${IDLE_MIN} 分钟)" "${BODY}"
log_info "[WATCHDOG] ✅ Resin 重启完成,新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"