watchdog v6: 自愈前先抓 jstack 现场,保存到 logs/jstack_dump/ 下次推送线程死锁根因可追溯

This commit is contained in:
oa-monitor committed 2026-11-06 18:28:51 +08:00
1 parent e8fab900fa
commit 689303f5d7
3 files changed
+49 -17

No files matched your search

+3 -2
View File
@@ -3,9 +3,10 @@ host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=10)
stdin, stdout, stderr = ssh.exec_command(cmd)
ssh.connect(host, username=user, password=password, timeout=30)
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=30)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
sys.stderr.write(stderr.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
+45 -14
View File
@@ -1,15 +1,9 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin Watchdog (v5 - 流程驱动判定)
# v4 盲区:15min 0 推送 ≠ 推送线程死了,可能真的没人提流程
# v5 核心逻辑:有新流程 BUT 无推送 → 推送线程才是真死了 → 自愈
#
# 判定链路(双重验证):
# Step 1: PID 变化检测(每次必跑,不依赖推送结果)
# Step 2: 流程-推送联动判定(核心改动)
# 2a. 最近 20min 新流程数 = 0 → 正常(没人提流程),直接退出
# 2b. 最近 20min message_log > 0 → 推送活着,直接退出
# 2c. 新流程 > 0 AND message_log = 0 → 推送线程死了 → 自愈!
# OA Monitor · Resin Watchdog (v6 - 自愈前抓 jstack 现场)
# v5: 流程驱动判定(有流程+无推送→自愈)
# v6: 触发自愈前先 jstack 抓 Java 线程现场,保存到 DSH logs/jstack_dump/
# 下次推送线程再死,根因直接可见,不再丢证据
#
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
# ============================================================================
@@ -28,8 +22,11 @@ END_HOUR=22
PID_FILE="${MONITOR_ROOT}/state/resin_pid.last"
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
JSTACK_DIR="${MONITOR_LOGS}/jstack_dump"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
mkdir -p "${JSTACK_DIR}"
# --- 依赖检查 ---
if ! python3 -c 'import paramiko' 2>/dev/null; then
MSG="[WATCHDOG] paramiko 不可用!"
@@ -45,9 +42,10 @@ host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=10)
stdin, stdout, stderr = ssh.exec_command(cmd)
ssh.connect(host, username=user, password=password, timeout=30)
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=30)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
sys.stderr.write(stderr.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
@@ -146,6 +144,27 @@ if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
exit 0
fi
# --- v6 新增: 自愈前抓 jstack 现场(推送线程死锁证据)---
JSTACK_TS=$(date +%Y%m%d_%H%M%S)
JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt"
log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..."
JSTACK_CMD="RESIN_PID=\$(ps -eo pid,cmd | grep java.*resin.xml | grep -v grep | head -1 | awk '{print \$1}'); \
if [ -n \"\$RESIN_PID\" ]; then \
/usr/weaver/jdk1.8.0_151/bin/jstack -l \$RESIN_PID 2>&1; \
else \
echo 'ERROR: cannot find Resin PID'; \
fi"
${OA_SSH} "${JSTACK_CMD}" > "${JSTACK_FILE}" 2>/dev/null
JSTACK_LINES=$(wc -l < "${JSTACK_FILE}" 2>/dev/null || echo 0)
if [[ ${JSTACK_LINES} -gt 0 ]]; then
log_info "[WATCHDOG] ✅ jstack 现场已保存: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)"
else
log_warn "[WATCHDOG] ❌ jstack 抓取失败"
fi
# --- 触发自愈!---
log_warn "[WATCHDOG] ⚠️ 有新流程(${NEW_REQS})但 message_log=0 → 推送线程确认已死 → 触发 Resin 自愈重启!"
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
@@ -154,12 +173,24 @@ sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
BODY="Resin watchdog 自动重启(v5 流程驱动判定)\n\n"
BODY="Resin watchdog 自动重启(v6 流程驱动判定)\n\n"
BODY="${BODY}最近 ${WINDOW_MIN}min 新流程: ${NEW_REQS} 条\n"
BODY="${BODY}最近 ${WINDOW_MIN}min message_log: ${MSG_COUNT} 条\n"
BODY="${BODY}判定: 有流程但无推送 → 推送线程已死\n\n"
BODY="${BODY}新 Resin PID: ${NEW_PID}\n"
BODY="${BODY}HTTP 80: ${HTTP_CODE}"
BODY="${BODY}HTTP 80: ${HTTP_CODE}\n"
BODY="${BODY}jstack 现场: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)"
# 钉钉附加关键线程片段(BLOCKED / info_read / updateBizState / 推送相关)
if [[ ${JSTACK_LINES} -gt 0 ]]; then
DEADLOCK_LINES=$(grep -A20 "Found one Java-level deadlock\|Found [0-9]* deadlock" "${JSTACK_FILE}" 2>/dev/null | head -25)
BLOCKED_COUNT=$(grep -c "java.lang.Thread.State: BLOCKED" "${JSTACK_FILE}" 2>/dev/null || echo 0)
BODY="${BODY}\n\n🧵 jstack 快览: BLOCKED=${BLOCKED_COUNT}"
if [[ -n "${DEADLOCK_LINES}" ]]; then
BODY="${BODY}\n🔥 DEADLOCK 发现:\n${DEADLOCK_LINES}"
fi
fi
dingtalk_alert "P0" "Resin watchdog 自动重启(有流程但无推送)" "${BODY}"
if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then