Files
oa-monitor/bin/resin-watchdog.sh
T

240 lines
11 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin Watchdog (v7 - HTTP 心跳 + Threads_running 辅助)
# v5: 流程驱动判定(有流程+无推送→自愈)
# v6: 触发自愈前先 jstack 抓 Java 线程现场
# v7: 加 Resin HTTP 心跳前置 + MySQL Threads_running 辅助,解决 Deadlock 时 info_read/message_log 同时零增长盲区
# 下次推送线程再死,根因直接可见,不再丢证据
#
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
OA_HOST="39.99.236.138"
OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
WINDOW_MIN=20 # 判定窗口(流程→推送有延迟,给足缓冲)
START_HOUR=8
END_HOUR=22
PID_FILE="${MONITOR_ROOT}/state/resin_pid.last"
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
JSTACK_DIR="${MONITOR_LOGS}/jstack_dump"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
mkdir -p "${JSTACK_DIR}"
# --- 依赖检查 ---
if ! python3 -c 'import paramiko' 2>/dev/null; then
MSG="[WATCHDOG] paramiko 不可用!"
log_error "${MSG}"
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
exit 1
fi
# --- SSH helper ---
cat > "${OA_SSH_PY}" << 'PYEOF'
import paramiko, sys
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=30)
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=30)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
sys.stderr.write(stderr.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
sys.exit(1)
PYEOF
chmod +x "${OA_SSH_PY}"
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
# ============================================================================
# Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果)
# ============================================================================
mkdir -p "$(dirname "${PID_FILE}")"
PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null)
CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}')
CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}')
if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测"
else
LAST_PID=""
LAST_ETIMES=""
if [[ -f "${PID_FILE}" ]]; then
read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true
fi
if [[ -z "${LAST_PID}" ]]; then
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s"
elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then
NOW_HOUR=$(date +%H)
NOW_MIN=$(date +%M)
IS_PREVENTIVE=0
if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then
IS_PREVENTIVE=1
fi
if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then
log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常"
else
RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S')
BODY="Resin PID 异常变化(非预期重启)\n\n"
BODY="${BODY}上次 PID: ${LAST_PID}\n"
BODY="${BODY}新 PID: ${CUR_PID}\n"
BODY="${BODY}推算重启时间: ${RESTART_TIME}\n"
BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒"
dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}"
log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警"
fi
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
else
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
fi
fi
# ============================================================================
# Step 2: 三层判定(v7 核心改动)
# Layer 1: Resin HTTP 心跳 → Resin 活着吗?
# Layer 2: MySQL Threads_running → 有没有 Deadlock?
# Layer 3: info_read vs message_log → 推送线程活着吗?
# ============================================================================
# 2a. Resin HTTP 心跳前置检查(任何情况下 Resin 必须能响应)
HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null)
if [[ "${HTTP_CODE}" != "200" ]]; then
log_warn "[WATCHDOG] Resin HTTP=${HTTP_CODE}, 服务不可达 → 触发自愈!"
# === 自愈逻辑(和下方重复,保持独立)===
JSTACK_TS=$(date +%Y%m%d_%H%M%S)
JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt"
log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..."
RESIN_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
if [[ "${RESIN_PID}" =~ ^[0-9]+$ ]]; then
${OA_SSH} "/usr/weaver/jdk1.8.0_151/bin/jstack -l ${RESIN_PID} > ${JSTACK_FILE}" 2>/dev/null || true
JSTACK_LINES=$(${OA_SSH} "wc -l < ${JSTACK_FILE}" 2>/dev/null | awk '{print $1}')
log_info "[WATCHDOG] ✅ jstack 已保存: ${JSTACK_LINES:-0} 行"
fi
${OA_SSH} "/usr/weaver/Resin/bin/stopresin.sh" 2>/dev/null
sleep 5
${OA_SSH} "/usr/weaver/Resin/bin/startresin.sh" 2>/dev/null
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
NEW_HTTP=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null)
log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${NEW_HTTP}"
dingtalk_alert "P0" "Resin HTTP 不可达(${HTTP_CODE}) → 自动自愈重启" "新 PID=${NEW_PID}
HTTP=${NEW_HTTP}"
exit 0
fi
# 2b. 最近 WINDOW_MIN 分钟 info_read 新增(有没有流程进审批节点?)
NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND ID NOT LIKE 'DUMMY_%';" 2>/dev/null)
# 2b. 最近 WINDOW_MIN 分钟 message_log 入库(推送线程活着吗?)
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
# 校验数据合法性
if [[ ! "${NEW_REQS}" =~ ^[0-9]+$ || ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] MySQL 查询失败(reqs=${NEW_REQS}, msg=${MSG_COUNT}), 跳过"
exit 0
fi
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}"
# === 判定分支 ===
# 分支 1: info_read=0 AND message_log=0 → 可能真没人用,也可能 Deadlock 全卡死
if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then
# 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5
TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}')
if [[ "${TR}" =~ ^[0-9]+$ && ${TR} -gt 5 ]]; then
# Threads_running 异常高 → 可能 Deadlock 全卡死 → 触发自愈
log_warn "[WATCHDOG] info_read=0 + message_log=0 + MySQL Threads_running=${TR} > 5 → 疑似 Deadlock 全卡死 → 自愈!"
else
log_info "[WATCHDOG] info_read=0, message_log=0, Threads_running=${TR:-?} 正常 → 推送线程空闲属正常, 不动"
exit 0
fi
fi
# 分支 2: 有推送 → 推送线程活着,不动
if [[ ${MSG_COUNT} -gt 0 ]]; then
log_info "[WATCHDOG] 有新流程(${NEW_REQS}) + 有推送(${MSG_COUNT}), 推送健康 ✅"
exit 0
fi
# 分支 3: 有流程 + 无推送 → 推送线程死了!
HOUR=$(date +%H)
if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), 有流程(${NEW_REQS})但无推送(${MSG_COUNT}), 仅记录"
exit 0
fi
# --- v6 新增: 自愈前抓 jstack 现场(推送线程死锁证据)---
JSTACK_TS=$(date +%Y%m%d_%H%M%S)
JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt"
log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..."
JSTACK_CMD="RESIN_PID=\$(ps -eo pid,cmd | grep java.*resin.xml | grep -v grep | head -1 | awk '{print \$1}'); \
if [ -n \"\$RESIN_PID\" ]; then \
/usr/weaver/jdk1.8.0_151/bin/jstack -l \$RESIN_PID 2>&1; \
else \
echo 'ERROR: cannot find Resin PID'; \
fi"
${OA_SSH} "${JSTACK_CMD}" > "${JSTACK_FILE}" 2>/dev/null
JSTACK_LINES=$(wc -l < "${JSTACK_FILE}" 2>/dev/null || echo 0)
if [[ ${JSTACK_LINES} -gt 0 ]]; then
log_info "[WATCHDOG] ✅ jstack 现场已保存: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)"
else
log_warn "[WATCHDOG] ❌ jstack 抓取失败"
fi
# --- 触发自愈!---
log_warn "[WATCHDOG] ⚠️ 有新流程(${NEW_REQS})但 message_log=0 → 推送线程确认已死 → 触发 Resin 自愈重启!"
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
BODY="Resin watchdog 自动重启(v6 流程驱动判定)\n\n"
BODY="${BODY}最近 ${WINDOW_MIN}min 新流程: ${NEW_REQS} 条\n"
BODY="${BODY}最近 ${WINDOW_MIN}min message_log: ${MSG_COUNT} 条\n"
BODY="${BODY}判定: 有流程但无推送 → 推送线程已死\n\n"
BODY="${BODY}新 Resin PID: ${NEW_PID}\n"
BODY="${BODY}HTTP 80: ${HTTP_CODE}\n"
BODY="${BODY}jstack 现场: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)"
# 钉钉附加关键线程片段(BLOCKED / info_read / updateBizState / 推送相关)
if [[ ${JSTACK_LINES} -gt 0 ]]; then
DEADLOCK_LINES=$(grep -A20 "Found one Java-level deadlock\|Found [0-9]* deadlock" "${JSTACK_FILE}" 2>/dev/null | head -25)
BLOCKED_COUNT=$(grep -c "java.lang.Thread.State: BLOCKED" "${JSTACK_FILE}" 2>/dev/null || echo 0)
BODY="${BODY}\n\n🧵 jstack 快览: BLOCKED=${BLOCKED_COUNT}"
if [[ -n "${DEADLOCK_LINES}" ]]; then
BODY="${BODY}\n🔥 DEADLOCK 发现:\n${DEADLOCK_LINES}"
fi
fi
dingtalk_alert "P0" "Resin watchdog 自动重启(有流程但无推送)" "${BODY}"
if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then
echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}"
fi
log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"