Files
oa-monitor/bin/resin-watchdog.sh
T

147 lines
6.1 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin Watchdog (v4 - PID 变化检测 + message_log 入库判定)
# v3 盲区修复:Resin 被 WatchdogManager kill 后秒拉起,message_log 继续刷看起来"健康"
# v4 新增:每次巡检记录 Resin PID,PID 变化 -> 钉钉 P1 告警
# 健康依据(双重):
# 1. ecology_message_log 最近 15 分钟入库 count > 0(推送线程活着)
# 2. Resin PID 未变化(进程未被 kill 重启)
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
OA_HOST="39.99.236.138"
OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
MSG_WINDOW_MIN=15
START_HOUR=8
END_HOUR=22
PID_FILE="${MONITOR_ROOT}/state/resin_pid.last"
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
# --- 依赖检查 ---
if ! python3 -c 'import paramiko' 2>/dev/null; then
MSG="[WATCHDOG] paramiko 不可用!"
log_error "${MSG}"
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
exit 1
fi
# --- SSH helper ---
cat > "${OA_SSH_PY}" << 'PYEOF'
import paramiko, sys
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=10)
stdin, stdout, stderr = ssh.exec_command(cmd)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
sys.exit(1)
PYEOF
chmod +x "${OA_SSH_PY}"
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
# ============================================================================
# Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果)
# ============================================================================
mkdir -p "$(dirname "${PID_FILE}")"
PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null)
CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}')
CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}')
if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测"
else
LAST_PID=""
LAST_ETIMES=""
if [[ -f "${PID_FILE}" ]]; then
read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true
fi
if [[ -z "${LAST_PID}" ]]; then
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s"
elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then
NOW_HOUR=$(date +%H)
NOW_MIN=$(date +%M)
IS_PREVENTIVE=0
if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then
IS_PREVENTIVE=1
fi
if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then
log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常"
else
RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S')
BODY="Resin PID 异常变化(非预期重启)\n\n"
BODY="${BODY}上次 PID: ${LAST_PID}\n"
BODY="${BODY}新 PID: ${CUR_PID}\n"
BODY="${BODY}推算重启时间: ${RESTART_TIME}\n"
BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒\n\n"
BODY="${BODY}Resin 被 WatchdogManager kill 后拉起。message_log 可能仍在刷(假活跃), watchdog v3 会漏检。"
dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}"
log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警"
fi
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
else
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
fi
fi
# ============================================================================
# Step 2: message_log 入库检测 (原 v3 逻辑)
# ============================================================================
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
if [[ ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] 查询 message_log 失败, 跳过"
exit 0
fi
log_info "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 入库 ${MSG_COUNT} 条"
HOUR=$(date +%H)
if [[ ${MSG_COUNT} -gt 0 ]]; then
log_info "[WATCHDOG] message_log 入库正常(${MSG_COUNT} > 0), 推送健康"
exit 0
fi
if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), message_log 仅 ${MSG_COUNT} 条, 仅记录"
exit 0
fi
# --- 工作时间 + 入库过少 -> 触发重启 ---
log_warn "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log count=0, 推送线程可能已死, 触发 Resin 重启!"
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
BODY="Resin watchdog 自动重启\n\n"
BODY="${BODY}message_log 最近 ${MSG_WINDOW_MIN} 分钟入库: ${MSG_COUNT} 条 (count > 0)\n"
BODY="${BODY}新 Resin PID: ${NEW_PID}\n"
BODY="${BODY}HTTP 80: ${HTTP_CODE}\n"
dingtalk_alert "P0" "Resin watchdog 自动重启(message_log count=0)" "${BODY}"
if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then
echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}"
fi
log_info "[WATCHDOG] Resin 重启完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"