Files
oa-monitor/bin/resin-watchdog.sh
T

170 lines
7.2 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin Watchdog (v5 - 流程驱动判定)
# v4 盲区:15min 0 推送 ≠ 推送线程死了,可能真的没人提流程
# v5 核心逻辑:有新流程 BUT 无推送 → 推送线程才是真死了 → 自愈
#
# 判定链路(双重验证):
# Step 1: PID 变化检测(每次必跑,不依赖推送结果)
# Step 2: 流程-推送联动判定(核心改动)
# 2a. 最近 20min 新流程数 = 0 → 正常(没人提流程),直接退出
# 2b. 最近 20min message_log > 0 → 推送活着,直接退出
# 2c. 新流程 > 0 AND message_log = 0 → 推送线程死了 → 自愈!
#
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
OA_HOST="39.99.236.138"
OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
WINDOW_MIN=20 # 判定窗口(流程→推送有延迟,给足缓冲)
START_HOUR=8
END_HOUR=22
PID_FILE="${MONITOR_ROOT}/state/resin_pid.last"
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
# --- 依赖检查 ---
if ! python3 -c 'import paramiko' 2>/dev/null; then
MSG="[WATCHDOG] paramiko 不可用!"
log_error "${MSG}"
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
exit 1
fi
# --- SSH helper ---
cat > "${OA_SSH_PY}" << 'PYEOF'
import paramiko, sys
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=10)
stdin, stdout, stderr = ssh.exec_command(cmd)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
sys.exit(1)
PYEOF
chmod +x "${OA_SSH_PY}"
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
# ============================================================================
# Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果)
# ============================================================================
mkdir -p "$(dirname "${PID_FILE}")"
PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null)
CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}')
CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}')
if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测"
else
LAST_PID=""
LAST_ETIMES=""
if [[ -f "${PID_FILE}" ]]; then
read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true
fi
if [[ -z "${LAST_PID}" ]]; then
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s"
elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then
NOW_HOUR=$(date +%H)
NOW_MIN=$(date +%M)
IS_PREVENTIVE=0
if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then
IS_PREVENTIVE=1
fi
if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then
log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常"
else
RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S')
BODY="Resin PID 异常变化(非预期重启)\n\n"
BODY="${BODY}上次 PID: ${LAST_PID}\n"
BODY="${BODY}新 PID: ${CUR_PID}\n"
BODY="${BODY}推算重启时间: ${RESTART_TIME}\n"
BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒"
dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}"
log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警"
fi
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
else
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
fi
fi
# ============================================================================
# Step 2: 流程-推送联动判定(v5 核心改动)
# ============================================================================
# 2a. 最近 WINDOW_MIN 分钟新流程数(有没有人提流程?)
NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND DELETED=0;" 2>/dev/null)
# 2b. 最近 WINDOW_MIN 分钟 message_log 入库(推送线程活着吗?)
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
# 校验数据合法性
if [[ ! "${NEW_REQS}" =~ ^[0-9]+$ || ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] MySQL 查询失败(reqs=${NEW_REQS}, msg=${MSG_COUNT}), 跳过"
exit 0
fi
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}"
# === 判定分支 ===
# 分支 1: 没人提流程 → 推送线程闲着很正常,不动
if [[ ${NEW_REQS} -eq 0 ]]; then
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min 无新流程(${NEW_REQS}), 推送线程空闲属正常, 不动"
exit 0
fi
# 分支 2: 有推送 → 推送线程活着,不动
if [[ ${MSG_COUNT} -gt 0 ]]; then
log_info "[WATCHDOG] 有新流程(${NEW_REQS}) + 有推送(${MSG_COUNT}), 推送健康 ✅"
exit 0
fi
# 分支 3: 有流程 + 无推送 → 推送线程死了!
HOUR=$(date +%H)
if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), 有流程(${NEW_REQS})但无推送(${MSG_COUNT}), 仅记录"
exit 0
fi
# --- 触发自愈!---
log_warn "[WATCHDOG] ⚠️ 有新流程(${NEW_REQS})但 message_log=0 → 推送线程确认已死 → 触发 Resin 自愈重启!"
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
BODY="Resin watchdog 自动重启(v5 流程驱动判定)\n\n"
BODY="${BODY}最近 ${WINDOW_MIN}min 新流程: ${NEW_REQS} 条\n"
BODY="${BODY}最近 ${WINDOW_MIN}min message_log: ${MSG_COUNT} 条\n"
BODY="${BODY}判定: 有流程但无推送 → 推送线程已死\n\n"
BODY="${BODY}新 Resin PID: ${NEW_PID}\n"
BODY="${BODY}HTTP 80: ${HTTP_CODE}"
dingtalk_alert "P0" "Resin watchdog 自动重启(有流程但无推送)" "${BODY}"
if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then
echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}"
fi
log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"