85 lines
4.4 KiB
Bash
Executable File
85 lines
4.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ============================================================================
|
|
# OA Monitor · Resin Watchdog
|
|
# 监控 Util_Message 线程活跃度(publishMessageList),超过 30 分钟无活动则自动重启 Resin
|
|
# 部署在 DSH 上,通过 sshpass 到 OA 服务器(39.99.236.138)执行
|
|
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
|
|
# ============================================================================
|
|
set -u
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
source "${SCRIPT_DIR}/../lib/common.sh"
|
|
|
|
OA_HOST="39.99.236.138"
|
|
OA_USER="root"
|
|
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
|
|
OA_LOG="/usr/weaver/Resin/log/stdout.log"
|
|
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
|
|
MAX_IDLE_MIN=30 # 30 分钟没 publishMessageList 活动就重启
|
|
RESIN_PATTERN="com.cloudstore.dev.api.util.Util_Message.*publishMessageList is start"
|
|
|
|
NOW=$(date +%s)
|
|
|
|
# --- 1. sshpass 可用性检查 ---
|
|
if ! command -v sshpass &>/dev/null; then
|
|
log_warn "[WATCHDOG] sshpass 未安装,尝试安装..."
|
|
ssh root@${OA_HOST} "which sshpass 2>/dev/null" && SSHPASS_OK=true || SSHPASS_OK=false
|
|
fi
|
|
|
|
# --- 2. ssh 到 OA 服务器查 publishMessageList 最后活动时间 ---
|
|
LAST_TS=$(sshpass -p "${OA_PASS}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${OA_USER}@${OA_HOST} "
|
|
# 从 stdout.log 最后 500 行找 publishMessageList
|
|
tail -5000 ${OA_LOG} 2>/dev/null | grep -E \"${RESIN_PATTERN}\" | tail -1 | grep -oP '\\d{4}\\.\\d{2}\\.\\d{2} \\d{2}:\\d{2}:\\d{2}' | head -1
|
|
" 2>/dev/null)
|
|
|
|
# 如果 stdout.log 没有,查 yesterday.log
|
|
if [[ -z "${LAST_TS}" ]]; then
|
|
LAST_TS=$(sshpass -p "${OA_PASS}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${OA_USER}@${OA_HOST} "
|
|
find /usr/weaver/Resin/log -name '*.log*' -newer ${OA_LOG}.yesterday 2>/dev/null | head -3 | xargs grep -lE '${RESIN_PATTERN}' 2>/dev/null | head -1 | xargs grep -E '${RESIN_PATTERN}' 2>/dev/null | tail -1 | grep -oP '\\d{4}\\.\\d{2}\\.\\d{2} \\d{2}:\\d{2}:\\d{2}' | head -1
|
|
" 2>/dev/null)
|
|
fi
|
|
|
|
# --- 3. 判断是否需要重启 ---
|
|
if [[ -z "${LAST_TS}" ]]; then
|
|
# 完全没找到 publishMessageList 记录——可能是新部署,不要贸然重启
|
|
log_warn "[WATCHDOG] ⚠️ stdout.log 里没找到 publishMessageList 记录,可能是新部署。跳过本次。"
|
|
exit 0
|
|
fi
|
|
|
|
# 把日志时间转成 epoch
|
|
LAST_EPOCH=$(date -j -f "%Y.%m.%d %H:%M:%S" "${LAST_TS}" +%s 2>/dev/null || date -d "${LAST_TS}" +%s 2>/dev/null)
|
|
if [[ -z "${LAST_EPOCH}" ]]; then
|
|
log_warn "[WATCHDOG] ⚠️ 无法解析日志时间: ${LAST_TS},跳过本次。"
|
|
exit 0
|
|
fi
|
|
|
|
IDLE_MIN=$(( (NOW - LAST_EPOCH) / 60 ))
|
|
|
|
log_info "[WATCHDOG] 最后 publishMessageList: ${LAST_TS},已空闲 ${IDLE_MIN} 分钟(阈值 ${MAX_IDLE_MIN} 分钟)"
|
|
|
|
if [[ ${IDLE_MIN} -ge ${MAX_IDLE_MIN} ]]; then
|
|
log_warn "[WATCHDOG] 🔴 空闲 ${IDLE_MIN} 分钟 ≥ ${MAX_IDLE_MIN} 分钟!触发 Resin 重启!"
|
|
|
|
# --- 4. 执行重启 ---
|
|
sshpass -p "${OA_PASS}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=15 ${OA_USER}@${OA_HOST} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
|
|
|
|
sleep 20
|
|
|
|
# --- 5. 验证新 Resin PID ---
|
|
NEW_PID=$(sshpass -p "${OA_PASS}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${OA_USER}@${OA_HOST} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null)
|
|
HTTP_CODE=$(sshpass -p "${OA_PASS}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${OA_USER}@${OA_HOST} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp 2>/dev/null" 2>/dev/null)
|
|
|
|
BODY="**🔴 Resin 被 watchdog 自动重启**\n\n"
|
|
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
|
BODY="${BODY}| publishMessageList 最后活动 | \`${LAST_TS}\` |\n"
|
|
BODY="${BODY}| 空闲时长 | **${IDLE_MIN} 分钟** |\n"
|
|
BODY="${BODY}| 新 Resin PID | \`${NEW_PID}\` |\n"
|
|
BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n\n"
|
|
BODY="${BODY}> ⚠️ 推送线程静默死掉,watchdog 自动恢复。\n"
|
|
BODY="${BODY}> 建议:泛微需要检查 Util_Message 线程为什么会静默挂掉。"
|
|
dingtalk_alert "P0" "Resin watchdog 自动重启(推送线程空闲 ${IDLE_MIN} 分钟)" "${BODY}"
|
|
|
|
log_info "[WATCHDOG] ✅ Resin 重启完成,新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"
|
|
else
|
|
log_info "[WATCHDOG] ✅ publishMessageList 活跃(最近 ${IDLE_MIN} 分钟前),Resin 健康"
|
|
fi
|