Files
oa-monitor/bin/resin-watchdog.sh
T
oa-monitor 2e1dbb5403 fix: 14:41 updateBizState 卡死根因修复 + watchdog v8 优化
- ALTER ecology_message_info_read ADD INDEX idx_iread_target_biz (TARGETID, USERID, BIZSTATE)
- UPDATE ecology_message_info_read SET BIZSTATE=1 WHERE BIZSTATE=0 (清 164371+3437 条积压)
- watchdog v8: Layer 4 workflow_requestbase 联动检查 + Layer 5 BIZSTATE 积压 P1 告警
- 根因: DUMMY 补 397 万 info_read 后表膨胀 20x → updateBizState 单线程全表扫追不上 → BIZSTATE=0 积压 → message_log 停写
2026-11-13 16:57:20 +08:00

259 lines
12 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin Watchdog (v7 - HTTP 心跳 + Threads_running 辅助)
# v5: 流程驱动判定(有流程+无推送→自愈)
# v6: 触发自愈前先 jstack 抓 Java 线程现场
# v7: 加 Resin HTTP 心跳前置 + MySQL Threads_running 辅助,解决 Deadlock 时 info_read/message_log 同时零增长盲区
# 下次推送线程再死,根因直接可见,不再丢证据
#
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
OA_HOST="39.99.236.138"
OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
WINDOW_MIN=20 # 判定窗口(流程→推送有延迟,给足缓冲)
START_HOUR=8
END_HOUR=22
PID_FILE="${MONITOR_ROOT}/state/resin_pid.last"
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
JSTACK_DIR="${MONITOR_LOGS}/jstack_dump"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
mkdir -p "${JSTACK_DIR}"
# --- 依赖检查 ---
if ! python3 -c 'import paramiko' 2>/dev/null; then
MSG="[WATCHDOG] paramiko 不可用!"
log_error "${MSG}"
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
exit 1
fi
# --- SSH helper ---
cat > "${OA_SSH_PY}" << 'PYEOF'
import paramiko, sys
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=30)
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=30)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
sys.stderr.write(stderr.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
sys.exit(1)
PYEOF
chmod +x "${OA_SSH_PY}"
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
# ============================================================================
# Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果)
# ============================================================================
mkdir -p "$(dirname "${PID_FILE}")"
PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null)
CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}')
CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}')
if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测"
else
LAST_PID=""
LAST_ETIMES=""
if [[ -f "${PID_FILE}" ]]; then
read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true
fi
if [[ -z "${LAST_PID}" ]]; then
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s"
elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then
NOW_HOUR=$(date +%H)
NOW_MIN=$(date +%M)
IS_PREVENTIVE=0
if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then
IS_PREVENTIVE=1
fi
if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then
log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常"
else
RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S')
BODY="Resin PID 异常变化(非预期重启)\n\n"
BODY="${BODY}上次 PID: ${LAST_PID}\n"
BODY="${BODY}新 PID: ${CUR_PID}\n"
BODY="${BODY}推算重启时间: ${RESTART_TIME}\n"
BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒"
dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}"
log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警"
fi
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
else
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
fi
fi
# ============================================================================
# Step 2: 三层判定(v7 核心改动)
# Layer 1: Resin HTTP 心跳 → Resin 活着吗?
# Layer 2: MySQL Threads_running → 有没有 Deadlock?
# Layer 3: info_read vs message_log → 推送线程活着吗?
# ============================================================================
# 2a. Resin HTTP 心跳前置检查(任何情况下 Resin 必须能响应)
HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null)
if [[ "${HTTP_CODE}" != "200" ]]; then
log_warn "[WATCHDOG] Resin HTTP=${HTTP_CODE}, 服务不可达 → 触发自愈!"
# === 自愈逻辑(和下方重复,保持独立)===
JSTACK_TS=$(date +%Y%m%d_%H%M%S)
JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt"
log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..."
RESIN_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
if [[ "${RESIN_PID}" =~ ^[0-9]+$ ]]; then
${OA_SSH} "/usr/weaver/jdk1.8.0_151/bin/jstack -l ${RESIN_PID} > ${JSTACK_FILE}" 2>/dev/null || true
JSTACK_LINES=$(${OA_SSH} "wc -l < ${JSTACK_FILE}" 2>/dev/null | awk '{print $1}')
log_info "[WATCHDOG] ✅ jstack 已保存: ${JSTACK_LINES:-0} 行"
fi
${OA_SSH} "/usr/weaver/Resin/bin/stopresin.sh" 2>/dev/null
sleep 5
${OA_SSH} "/usr/weaver/Resin/bin/startresin.sh" 2>/dev/null
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
NEW_HTTP=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null)
log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${NEW_HTTP}"
dingtalk_alert "P0" "Resin HTTP 不可达(${HTTP_CODE}) → 自动自愈重启" "新 PID=${NEW_PID}
HTTP=${NEW_HTTP}"
exit 0
fi
# 2b. 最近 WINDOW_MIN 分钟 info_read 新增(有没有流程进审批节点?)
NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND ID NOT LIKE 'DUMMY_%';" 2>/dev/null)
# 2b. 最近 WINDOW_MIN 分钟 message_log 入库(推送线程活着吗?)
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
# 校验数据合法性
if [[ ! "${NEW_REQS}" =~ ^[0-9]+$ || ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
log_warn "[WATCHDOG] MySQL 查询失败(reqs=${NEW_REQS}, msg=${MSG_COUNT}), 跳过"
exit 0
fi
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}"
# === v8 新增: BIZSTATE 积压检查(推送线程慢了但没死 → 提前告警)===
BIZ_BACKLOG=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE BIZSTATE=0;" 2>/dev/null)
if [[ "${BIZ_BACKLOG}" =~ ^[0-9]+$ && ${BIZ_BACKLOG} -gt 1000 ]]; then
log_warn "[WATCHDOG] ⚠️ BIZSTATE=0 积压=${BIZ_BACKLOG}(推送线程追不上)→ P1 告警"
dingtalk_alert "P1" "info_read BIZSTATE 积压告警" "当前积压: ${BIZ_BACKLOG} 条
含义: 推送线程追不上业务产生的 info_read 速度
建议: 联系 AI 处理 UPDATE ecology_message_info_read SET BIZSTATE=1 WHERE BIZSTATE=0"
fi
# === 判定分支 ===
# v8 新增: 先查 workflow_requestbase 最近 20min 新流程(判断业务是否真在跑)
WF_NEW=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
# 分支 1: info_read=0 AND message_log=0
if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then
# v8 Layer 4 新增: workflow_requestbase 有新流程 → 业务在跑但两张表都停了
if [[ "${WF_NEW}" =~ ^[0-9]+$ && ${WF_NEW} -gt 0 ]]; then
log_warn "[WATCHDOG] ⚠️ info_read=0 + message_log=0 BUT workflow_requestbase 新流程=${WF_NEW} → 推送线程卡死 → 自愈!"
else
# 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5
TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
-sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}')
if [[ "${TR}" =~ ^[0-9]+$ && ${TR} -gt 5 ]]; then
# Threads_running 异常高 → 可能 Deadlock 全卡死 → 触发自愈
log_warn "[WATCHDOG] info_read=0 + message_log=0 + MySQL Threads_running=${TR} > 5 → 疑似 Deadlock 全卡死 → 自愈!"
else
log_info "[WATCHDOG] info_read=0, message_log=0, Threads_running=${TR:-?} 正常 → 推送线程空闲属正常, 不动"
exit 0
fi
fi
fi
# 分支 2: 有推送 → 推送线程活着,不动
if [[ ${MSG_COUNT} -gt 0 ]]; then
log_info "[WATCHDOG] 有新流程(${NEW_REQS}) + 有推送(${MSG_COUNT}), 推送健康 ✅"
exit 0
fi
# 分支 3: 有流程 + 无推送 → 推送线程死了!
HOUR=$(date +%H)
if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), 有流程(${NEW_REQS})但无推送(${MSG_COUNT}), 仅记录"
exit 0
fi
# --- v6 新增: 自愈前抓 jstack 现场(推送线程死锁证据)---
JSTACK_TS=$(date +%Y%m%d_%H%M%S)
JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt"
log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..."
JSTACK_CMD="RESIN_PID=\$(ps -eo pid,cmd | grep java.*resin.xml | grep -v grep | head -1 | awk '{print \$1}'); \
if [ -n \"\$RESIN_PID\" ]; then \
/usr/weaver/jdk1.8.0_151/bin/jstack -l \$RESIN_PID 2>&1; \
else \
echo 'ERROR: cannot find Resin PID'; \
fi"
${OA_SSH} "${JSTACK_CMD}" > "${JSTACK_FILE}" 2>/dev/null
JSTACK_LINES=$(wc -l < "${JSTACK_FILE}" 2>/dev/null || echo 0)
if [[ ${JSTACK_LINES} -gt 0 ]]; then
log_info "[WATCHDOG] ✅ jstack 现场已保存: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)"
else
log_warn "[WATCHDOG] ❌ jstack 抓取失败"
fi
# --- 触发自愈!---
log_warn "[WATCHDOG] ⚠️ 有新流程(${NEW_REQS})但 message_log=0 → 推送线程确认已死 → 触发 Resin 自愈重启!"
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
sleep 20
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
BODY="Resin watchdog 自动重启(v6 流程驱动判定)\n\n"
BODY="${BODY}最近 ${WINDOW_MIN}min 新流程: ${NEW_REQS} 条\n"
BODY="${BODY}最近 ${WINDOW_MIN}min message_log: ${MSG_COUNT} 条\n"
BODY="${BODY}判定: 有流程但无推送 → 推送线程已死\n\n"
BODY="${BODY}新 Resin PID: ${NEW_PID}\n"
BODY="${BODY}HTTP 80: ${HTTP_CODE}\n"
BODY="${BODY}jstack 现场: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)"
# 钉钉附加关键线程片段(BLOCKED / info_read / updateBizState / 推送相关)
if [[ ${JSTACK_LINES} -gt 0 ]]; then
DEADLOCK_LINES=$(grep -A20 "Found one Java-level deadlock\|Found [0-9]* deadlock" "${JSTACK_FILE}" 2>/dev/null | head -25)
BLOCKED_COUNT=$(grep -c "java.lang.Thread.State: BLOCKED" "${JSTACK_FILE}" 2>/dev/null || echo 0)
BODY="${BODY}\n\n🧵 jstack 快览: BLOCKED=${BLOCKED_COUNT}"
if [[ -n "${DEADLOCK_LINES}" ]]; then
BODY="${BODY}\n🔥 DEADLOCK 发现:\n${DEADLOCK_LINES}"
fi
fi
dingtalk_alert "P0" "Resin watchdog 自动重启(有流程但无推送)" "${BODY}"
if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then
echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}"
fi
log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"