#!/usr/bin/env bash # ============================================================================ # OA Monitor · Resin Watchdog (v5 - 流程驱动判定) # v4 盲区:15min 0 推送 ≠ 推送线程死了,可能真的没人提流程 # v5 核心逻辑:有新流程 BUT 无推送 → 推送线程才是真死了 → 自愈 # # 判定链路(双重验证): # Step 1: PID 变化检测(每次必跑,不依赖推送结果) # Step 2: 流程-推送联动判定(核心改动) # 2a. 最近 20min 新流程数 = 0 → 正常(没人提流程),直接退出 # 2b. 最近 20min message_log > 0 → 推送活着,直接退出 # 2c. 新流程 > 0 AND message_log = 0 → 推送线程死了 → 自愈! # # crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh # ============================================================================ set -u SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "${SCRIPT_DIR}/../lib/common.sh" OA_HOST="39.99.236.138" OA_USER="root" OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM" OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1" WINDOW_MIN=20 # 判定窗口(流程→推送有延迟,给足缓冲) START_HOUR=8 END_HOUR=22 PID_FILE="${MONITOR_ROOT}/state/resin_pid.last" LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log" OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py" # --- 依赖检查 --- if ! python3 -c 'import paramiko' 2>/dev/null; then MSG="[WATCHDOG] paramiko 不可用!" log_error "${MSG}" dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}" exit 1 fi # --- SSH helper --- cat > "${OA_SSH_PY}" << 'PYEOF' import paramiko, sys host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: ssh.connect(host, username=user, password=password, timeout=10) stdin, stdout, stderr = ssh.exec_command(cmd) sys.stdout.write(stdout.read().decode("utf-8", errors="replace")) ssh.close() except Exception as e: print(f"SSH_ERROR: {e}", file=sys.stderr) sys.exit(1) PYEOF chmod +x "${OA_SSH_PY}" OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}" # ============================================================================ # Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果) # ============================================================================ mkdir -p "$(dirname "${PID_FILE}")" PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null) CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}') CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}') if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测" else LAST_PID="" LAST_ETIMES="" if [[ -f "${PID_FILE}" ]]; then read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true fi if [[ -z "${LAST_PID}" ]]; then echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s" elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then NOW_HOUR=$(date +%H) NOW_MIN=$(date +%M) IS_PREVENTIVE=0 if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then IS_PREVENTIVE=1 fi if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常" else RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S') BODY="Resin PID 异常变化(非预期重启)\n\n" BODY="${BODY}上次 PID: ${LAST_PID}\n" BODY="${BODY}新 PID: ${CUR_PID}\n" BODY="${BODY}推算重启时间: ${RESTART_TIME}\n" BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒" dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}" log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警" fi echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" else echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" fi fi # ============================================================================ # Step 2: 流程-推送联动判定(v5 核心改动) # ============================================================================ # 2a. 最近 WINDOW_MIN 分钟新流程数(有没有人提流程?) NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ -sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND DELETED=0;" 2>/dev/null) # 2b. 最近 WINDOW_MIN 分钟 message_log 入库(推送线程活着吗?) MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ -sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null) # 校验数据合法性 if [[ ! "${NEW_REQS}" =~ ^[0-9]+$ || ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then log_warn "[WATCHDOG] MySQL 查询失败(reqs=${NEW_REQS}, msg=${MSG_COUNT}), 跳过" exit 0 fi log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}" # === 判定分支 === # 分支 1: 没人提流程 → 推送线程闲着很正常,不动 if [[ ${NEW_REQS} -eq 0 ]]; then log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min 无新流程(${NEW_REQS}), 推送线程空闲属正常, 不动" exit 0 fi # 分支 2: 有推送 → 推送线程活着,不动 if [[ ${MSG_COUNT} -gt 0 ]]; then log_info "[WATCHDOG] 有新流程(${NEW_REQS}) + 有推送(${MSG_COUNT}), 推送健康 ✅" exit 0 fi # 分支 3: 有流程 + 无推送 → 推送线程死了! HOUR=$(date +%H) if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), 有流程(${NEW_REQS})但无推送(${MSG_COUNT}), 仅记录" exit 0 fi # --- 触发自愈!--- log_warn "[WATCHDOG] ⚠️ 有新流程(${NEW_REQS})但 message_log=0 → 推送线程确认已死 → 触发 Resin 自愈重启!" ${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}" sleep 20 NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1) HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null) BODY="Resin watchdog 自动重启(v5 流程驱动判定)\n\n" BODY="${BODY}最近 ${WINDOW_MIN}min 新流程: ${NEW_REQS} 条\n" BODY="${BODY}最近 ${WINDOW_MIN}min message_log: ${MSG_COUNT} 条\n" BODY="${BODY}判定: 有流程但无推送 → 推送线程已死\n\n" BODY="${BODY}新 Resin PID: ${NEW_PID}\n" BODY="${BODY}HTTP 80: ${HTTP_CODE}" dingtalk_alert "P0" "Resin watchdog 自动重启(有流程但无推送)" "${BODY}" if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}" fi log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"