Files
oa-monitor/bin/resin-watchdog.sh
T

201 lines
8.5 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin Watchdog (v6 - 自愈前抓 jstack 现场)
# v5: 流程驱动判定(有流程+无推送→自愈)
# v6: 触发自愈前先 jstack 抓 Java 线程现场,保存到 DSH logs/jstack_dump/
# 下次推送线程再死,根因直接可见,不再丢证据
#
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
OA_HOST="39.99.236.138"
OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
WINDOW_MIN=20 # 判定窗口(流程→推送有延迟,给足缓冲)
START_HOUR=8
END_HOUR=22
PID_FILE="${MONITOR_ROOT}/state/resin_pid.last"
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
JSTACK_DIR="${MONITOR_LOGS}/jstack_dump"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
mkdir -p "${JSTACK_DIR}"
# --- 依赖检查 ---
if ! python3 -c 'import paramiko' 2>/dev/null; then
MSG="[WATCHDOG] paramiko 不可用!"
log_error "${MSG}"
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
exit 1
fi
# --- SSH helper ---
cat > "${OA_SSH_PY}" << 'PYEOF'
import paramiko, sys
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=30)
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=30)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
sys.stderr.write(stderr.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
sys.exit(1)
PYEOF
chmod +x "${OA_SSH_PY}"
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
# ============================================================================
# Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果)
# ============================================================================
mkdir -p "$(dirname "${PID_FILE}")"
PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null)
CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}')
CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}')
if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测"
else
LAST_PID=""
LAST_ETIMES=""
if [[ -f "${PID_FILE}" ]]; then
read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true
fi
if [[ -z "${LAST_PID}" ]]; then
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s"
elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then
NOW_HOUR=$(date +%H)
NOW_MIN=$(date +%M)
IS_PREVENTIVE=0
if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then
IS_PREVENTIVE=1
fi
if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then
log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常"
else
RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S')
BODY="Resin PID 异常变化(非预期重启)\n\n"
BODY="${BODY}上次 PID: ${LAST_PID}\n"
BODY="${BODY}新 PID: ${CUR_PID}\n"
BODY="${BODY}推算重启时间: ${RESTART_TIME}\n"
BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒"
dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}"
log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警"
fi
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
else
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
fi
fi
# ============================================================================
# Step 2: 流程-推送联动判定(v5 核心改动)
# ============================================================================
# 2a. 最近 WINDOW_MIN 分钟新流程数(有没有人提流程?)
NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND ID NOT LIKE 'DUMMY_%';" 2>/dev/null)
# 2b. 最近 WINDOW_MIN 分钟 message_log 入库(推送线程活着吗?)
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
# 校验数据合法性
if [[ ! "${NEW_REQS}" =~ ^[0-9]+$ || ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] MySQL 查询失败(reqs=${NEW_REQS}, msg=${MSG_COUNT}), 跳过"
exit 0
fi
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}"
# === 判定分支 ===
# 分支 1: 没人提流程 → 推送线程闲着很正常,不动
if [[ ${NEW_REQS} -eq 0 ]]; then
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min 无新流程(${NEW_REQS}), 推送线程空闲属正常, 不动"
exit 0
fi
# 分支 2: 有推送 → 推送线程活着,不动
if [[ ${MSG_COUNT} -gt 0 ]]; then
log_info "[WATCHDOG] 有新流程(${NEW_REQS}) + 有推送(${MSG_COUNT}), 推送健康 ✅"
exit 0
fi
# 分支 3: 有流程 + 无推送 → 推送线程死了!
HOUR=$(date +%H)
if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), 有流程(${NEW_REQS})但无推送(${MSG_COUNT}), 仅记录"
exit 0
fi
# --- v6 新增: 自愈前抓 jstack 现场(推送线程死锁证据)---
JSTACK_TS=$(date +%Y%m%d_%H%M%S)
JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt"
log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..."
JSTACK_CMD="RESIN_PID=\$(ps -eo pid,cmd | grep java.*resin.xml | grep -v grep | head -1 | awk '{print \$1}'); \
if [ -n \"\$RESIN_PID\" ]; then \
/usr/weaver/jdk1.8.0_151/bin/jstack -l \$RESIN_PID 2>&1; \
else \
echo 'ERROR: cannot find Resin PID'; \
fi"
${OA_SSH} "${JSTACK_CMD}" > "${JSTACK_FILE}" 2>/dev/null
JSTACK_LINES=$(wc -l < "${JSTACK_FILE}" 2>/dev/null || echo 0)
if [[ ${JSTACK_LINES} -gt 0 ]]; then
log_info "[WATCHDOG] ✅ jstack 现场已保存: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)"
else
log_warn "[WATCHDOG] ❌ jstack 抓取失败"
fi
# --- 触发自愈!---
log_warn "[WATCHDOG] ⚠️ 有新流程(${NEW_REQS})但 message_log=0 → 推送线程确认已死 → 触发 Resin 自愈重启!"
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
BODY="Resin watchdog 自动重启(v6 流程驱动判定)\n\n"
BODY="${BODY}最近 ${WINDOW_MIN}min 新流程: ${NEW_REQS} 条\n"
BODY="${BODY}最近 ${WINDOW_MIN}min message_log: ${MSG_COUNT} 条\n"
BODY="${BODY}判定: 有流程但无推送 → 推送线程已死\n\n"
BODY="${BODY}新 Resin PID: ${NEW_PID}\n"
BODY="${BODY}HTTP 80: ${HTTP_CODE}\n"
BODY="${BODY}jstack 现场: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)"
# 钉钉附加关键线程片段(BLOCKED / info_read / updateBizState / 推送相关)
if [[ ${JSTACK_LINES} -gt 0 ]]; then
DEADLOCK_LINES=$(grep -A20 "Found one Java-level deadlock\|Found [0-9]* deadlock" "${JSTACK_FILE}" 2>/dev/null | head -25)
BLOCKED_COUNT=$(grep -c "java.lang.Thread.State: BLOCKED" "${JSTACK_FILE}" 2>/dev/null || echo 0)
BODY="${BODY}\n\n🧵 jstack 快览: BLOCKED=${BLOCKED_COUNT}"
if [[ -n "${DEADLOCK_LINES}" ]]; then
BODY="${BODY}\n🔥 DEADLOCK 发现:\n${DEADLOCK_LINES}"
fi
fi
dingtalk_alert "P0" "Resin watchdog 自动重启(有流程但无推送)" "${BODY}"
if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then
echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}"
fi
log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"