diff --git a/bin/_oa_ssh.py b/bin/_oa_ssh.py index 44e0b21..8c6c717 100755 --- a/bin/_oa_ssh.py +++ b/bin/_oa_ssh.py @@ -3,9 +3,10 @@ host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: - ssh.connect(host, username=user, password=password, timeout=10) - stdin, stdout, stderr = ssh.exec_command(cmd) + ssh.connect(host, username=user, password=password, timeout=30) + stdin, stdout, stderr = ssh.exec_command(cmd, timeout=30) sys.stdout.write(stdout.read().decode("utf-8", errors="replace")) + sys.stderr.write(stderr.read().decode("utf-8", errors="replace")) ssh.close() except Exception as e: print(f"SSH_ERROR: {e}", file=sys.stderr) diff --git a/bin/resin-watchdog.sh b/bin/resin-watchdog.sh index 6d793a5..19b28e1 100755 --- a/bin/resin-watchdog.sh +++ b/bin/resin-watchdog.sh @@ -1,15 +1,9 @@ #!/usr/bin/env bash # ============================================================================ -# OA Monitor · Resin Watchdog (v5 - 流程驱动判定) -# v4 盲区:15min 0 推送 ≠ 推送线程死了,可能真的没人提流程 -# v5 核心逻辑:有新流程 BUT 无推送 → 推送线程才是真死了 → 自愈 -# -# 判定链路(双重验证): -# Step 1: PID 变化检测(每次必跑,不依赖推送结果) -# Step 2: 流程-推送联动判定(核心改动) -# 2a. 最近 20min 新流程数 = 0 → 正常(没人提流程),直接退出 -# 2b. 最近 20min message_log > 0 → 推送活着,直接退出 -# 2c. 新流程 > 0 AND message_log = 0 → 推送线程死了 → 自愈! +# OA Monitor · Resin Watchdog (v6 - 自愈前抓 jstack 现场) +# v5: 流程驱动判定(有流程+无推送→自愈) +# v6: 触发自愈前先 jstack 抓 Java 线程现场,保存到 DSH logs/jstack_dump/ +# 下次推送线程再死,根因直接可见,不再丢证据 # # crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh # ============================================================================ @@ -28,8 +22,11 @@ END_HOUR=22 PID_FILE="${MONITOR_ROOT}/state/resin_pid.last" LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log" +JSTACK_DIR="${MONITOR_LOGS}/jstack_dump" OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py" +mkdir -p "${JSTACK_DIR}" + # --- 依赖检查 --- if ! python3 -c 'import paramiko' 2>/dev/null; then MSG="[WATCHDOG] paramiko 不可用!" @@ -45,9 +42,10 @@ host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: - ssh.connect(host, username=user, password=password, timeout=10) - stdin, stdout, stderr = ssh.exec_command(cmd) + ssh.connect(host, username=user, password=password, timeout=30) + stdin, stdout, stderr = ssh.exec_command(cmd, timeout=30) sys.stdout.write(stdout.read().decode("utf-8", errors="replace")) + sys.stderr.write(stderr.read().decode("utf-8", errors="replace")) ssh.close() except Exception as e: print(f"SSH_ERROR: {e}", file=sys.stderr) @@ -146,6 +144,27 @@ if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then exit 0 fi +# --- v6 新增: 自愈前抓 jstack 现场(推送线程死锁证据)--- +JSTACK_TS=$(date +%Y%m%d_%H%M%S) +JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt" + +log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..." +JSTACK_CMD="RESIN_PID=\$(ps -eo pid,cmd | grep java.*resin.xml | grep -v grep | head -1 | awk '{print \$1}'); \ + if [ -n \"\$RESIN_PID\" ]; then \ + /usr/weaver/jdk1.8.0_151/bin/jstack -l \$RESIN_PID 2>&1; \ + else \ + echo 'ERROR: cannot find Resin PID'; \ + fi" + +${OA_SSH} "${JSTACK_CMD}" > "${JSTACK_FILE}" 2>/dev/null +JSTACK_LINES=$(wc -l < "${JSTACK_FILE}" 2>/dev/null || echo 0) + +if [[ ${JSTACK_LINES} -gt 0 ]]; then + log_info "[WATCHDOG] ✅ jstack 现场已保存: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)" +else + log_warn "[WATCHDOG] ❌ jstack 抓取失败" +fi + # --- 触发自愈!--- log_warn "[WATCHDOG] ⚠️ 有新流程(${NEW_REQS})但 message_log=0 → 推送线程确认已死 → 触发 Resin 自愈重启!" ${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}" @@ -154,12 +173,24 @@ sleep 20 NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1) HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null) -BODY="Resin watchdog 自动重启(v5 流程驱动判定)\n\n" +BODY="Resin watchdog 自动重启(v6 流程驱动判定)\n\n" BODY="${BODY}最近 ${WINDOW_MIN}min 新流程: ${NEW_REQS} 条\n" BODY="${BODY}最近 ${WINDOW_MIN}min message_log: ${MSG_COUNT} 条\n" BODY="${BODY}判定: 有流程但无推送 → 推送线程已死\n\n" BODY="${BODY}新 Resin PID: ${NEW_PID}\n" -BODY="${BODY}HTTP 80: ${HTTP_CODE}" +BODY="${BODY}HTTP 80: ${HTTP_CODE}\n" +BODY="${BODY}jstack 现场: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)" + +# 钉钉附加关键线程片段(BLOCKED / info_read / updateBizState / 推送相关) +if [[ ${JSTACK_LINES} -gt 0 ]]; then + DEADLOCK_LINES=$(grep -A20 "Found one Java-level deadlock\|Found [0-9]* deadlock" "${JSTACK_FILE}" 2>/dev/null | head -25) + BLOCKED_COUNT=$(grep -c "java.lang.Thread.State: BLOCKED" "${JSTACK_FILE}" 2>/dev/null || echo 0) + BODY="${BODY}\n\n🧵 jstack 快览: BLOCKED=${BLOCKED_COUNT}" + if [[ -n "${DEADLOCK_LINES}" ]]; then + BODY="${BODY}\n🔥 DEADLOCK 发现:\n${DEADLOCK_LINES}" + fi +fi + dingtalk_alert "P0" "Resin watchdog 自动重启(有流程但无推送)" "${BODY}" if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then diff --git a/state/resin_pid.last b/state/resin_pid.last index 3138eef..b0e3c16 100644 --- a/state/resin_pid.last +++ b/state/resin_pid.last @@ -1 +1 @@ -1807 50429 1793952040 +31753 2602 1793960923