- ALTER ecology_message_info_read ADD INDEX idx_iread_target_biz (TARGETID, USERID, BIZSTATE) - UPDATE ecology_message_info_read SET BIZSTATE=1 WHERE BIZSTATE=0 (清 164371+3437 条积压) - watchdog v8: Layer 4 workflow_requestbase 联动检查 + Layer 5 BIZSTATE 积压 P1 告警 - 根因: DUMMY 补 397 万 info_read 后表膨胀 20x → updateBizState 单线程全表扫追不上 → BIZSTATE=0 积压 → message_log 停写
259 lines
12 KiB
Bash
Executable File
259 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# ============================================================================
|
||
# OA Monitor · Resin Watchdog (v7 - HTTP 心跳 + Threads_running 辅助)
|
||
# v5: 流程驱动判定(有流程+无推送→自愈)
|
||
# v6: 触发自愈前先 jstack 抓 Java 线程现场
|
||
# v7: 加 Resin HTTP 心跳前置 + MySQL Threads_running 辅助,解决 Deadlock 时 info_read/message_log 同时零增长盲区
|
||
# 下次推送线程再死,根因直接可见,不再丢证据
|
||
#
|
||
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
|
||
# ============================================================================
|
||
set -u
|
||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||
|
||
OA_HOST="39.99.236.138"
|
||
OA_USER="root"
|
||
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
|
||
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
|
||
|
||
WINDOW_MIN=20 # 判定窗口(流程→推送有延迟,给足缓冲)
|
||
START_HOUR=8
|
||
END_HOUR=22
|
||
|
||
PID_FILE="${MONITOR_ROOT}/state/resin_pid.last"
|
||
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
|
||
JSTACK_DIR="${MONITOR_LOGS}/jstack_dump"
|
||
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
|
||
|
||
mkdir -p "${JSTACK_DIR}"
|
||
|
||
# --- 依赖检查 ---
|
||
if ! python3 -c 'import paramiko' 2>/dev/null; then
|
||
MSG="[WATCHDOG] paramiko 不可用!"
|
||
log_error "${MSG}"
|
||
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
|
||
exit 1
|
||
fi
|
||
|
||
# --- SSH helper ---
|
||
cat > "${OA_SSH_PY}" << 'PYEOF'
|
||
import paramiko, sys
|
||
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
|
||
ssh = paramiko.SSHClient()
|
||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||
try:
|
||
ssh.connect(host, username=user, password=password, timeout=30)
|
||
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=30)
|
||
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
|
||
sys.stderr.write(stderr.read().decode("utf-8", errors="replace"))
|
||
ssh.close()
|
||
except Exception as e:
|
||
print(f"SSH_ERROR: {e}", file=sys.stderr)
|
||
sys.exit(1)
|
||
PYEOF
|
||
chmod +x "${OA_SSH_PY}"
|
||
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
|
||
|
||
# ============================================================================
|
||
# Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果)
|
||
# ============================================================================
|
||
mkdir -p "$(dirname "${PID_FILE}")"
|
||
|
||
PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null)
|
||
CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}')
|
||
CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}')
|
||
|
||
if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then
|
||
log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测"
|
||
else
|
||
LAST_PID=""
|
||
LAST_ETIMES=""
|
||
if [[ -f "${PID_FILE}" ]]; then
|
||
read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true
|
||
fi
|
||
|
||
if [[ -z "${LAST_PID}" ]]; then
|
||
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
|
||
log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s"
|
||
elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then
|
||
NOW_HOUR=$(date +%H)
|
||
NOW_MIN=$(date +%M)
|
||
IS_PREVENTIVE=0
|
||
if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then
|
||
IS_PREVENTIVE=1
|
||
fi
|
||
|
||
if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then
|
||
log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常"
|
||
else
|
||
RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
BODY="Resin PID 异常变化(非预期重启)\n\n"
|
||
BODY="${BODY}上次 PID: ${LAST_PID}\n"
|
||
BODY="${BODY}新 PID: ${CUR_PID}\n"
|
||
BODY="${BODY}推算重启时间: ${RESTART_TIME}\n"
|
||
BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒"
|
||
dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}"
|
||
|
||
log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警"
|
||
fi
|
||
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
|
||
else
|
||
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
|
||
fi
|
||
fi
|
||
|
||
# ============================================================================
|
||
# Step 2: 三层判定(v7 核心改动)
|
||
# Layer 1: Resin HTTP 心跳 → Resin 活着吗?
|
||
# Layer 2: MySQL Threads_running → 有没有 Deadlock?
|
||
# Layer 3: info_read vs message_log → 推送线程活着吗?
|
||
# ============================================================================
|
||
|
||
# 2a. Resin HTTP 心跳前置检查(任何情况下 Resin 必须能响应)
|
||
HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null)
|
||
if [[ "${HTTP_CODE}" != "200" ]]; then
|
||
log_warn "[WATCHDOG] Resin HTTP=${HTTP_CODE}, 服务不可达 → 触发自愈!"
|
||
# === 自愈逻辑(和下方重复,保持独立)===
|
||
JSTACK_TS=$(date +%Y%m%d_%H%M%S)
|
||
JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt"
|
||
log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..."
|
||
RESIN_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
|
||
if [[ "${RESIN_PID}" =~ ^[0-9]+$ ]]; then
|
||
${OA_SSH} "/usr/weaver/jdk1.8.0_151/bin/jstack -l ${RESIN_PID} > ${JSTACK_FILE}" 2>/dev/null || true
|
||
JSTACK_LINES=$(${OA_SSH} "wc -l < ${JSTACK_FILE}" 2>/dev/null | awk '{print $1}')
|
||
log_info "[WATCHDOG] ✅ jstack 已保存: ${JSTACK_LINES:-0} 行"
|
||
fi
|
||
${OA_SSH} "/usr/weaver/Resin/bin/stopresin.sh" 2>/dev/null
|
||
sleep 5
|
||
${OA_SSH} "/usr/weaver/Resin/bin/startresin.sh" 2>/dev/null
|
||
sleep 20
|
||
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
|
||
NEW_HTTP=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://${OA_HOST}/api/workflow/getRequestList 2>/dev/null)
|
||
log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${NEW_HTTP}"
|
||
dingtalk_alert "P0" "Resin HTTP 不可达(${HTTP_CODE}) → 自动自愈重启" "新 PID=${NEW_PID}
|
||
HTTP=${NEW_HTTP}"
|
||
exit 0
|
||
fi
|
||
|
||
# 2b. 最近 WINDOW_MIN 分钟 info_read 新增(有没有流程进审批节点?)
|
||
NEW_REQS=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s') AND ID NOT LIKE 'DUMMY_%';" 2>/dev/null)
|
||
|
||
# 2b. 最近 WINDOW_MIN 分钟 message_log 入库(推送线程活着吗?)
|
||
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
|
||
|
||
# 校验数据合法性
|
||
if [[ ! "${NEW_REQS}" =~ ^[0-9]+$ || ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
|
||
log_warn "[WATCHDOG] MySQL 查询失败(reqs=${NEW_REQS}, msg=${MSG_COUNT}), 跳过"
|
||
exit 0
|
||
fi
|
||
|
||
log_info "[WATCHDOG] 最近 ${WINDOW_MIN}min: 新流程=${NEW_REQS}, message_log=${MSG_COUNT}"
|
||
|
||
# === v8 新增: BIZSTATE 积压检查(推送线程慢了但没死 → 提前告警)===
|
||
BIZ_BACKLOG=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||
-sN -e "SELECT COUNT(*) FROM ecology_message_info_read WHERE BIZSTATE=0;" 2>/dev/null)
|
||
if [[ "${BIZ_BACKLOG}" =~ ^[0-9]+$ && ${BIZ_BACKLOG} -gt 1000 ]]; then
|
||
log_warn "[WATCHDOG] ⚠️ BIZSTATE=0 积压=${BIZ_BACKLOG}(推送线程追不上)→ P1 告警"
|
||
dingtalk_alert "P1" "info_read BIZSTATE 积压告警" "当前积压: ${BIZ_BACKLOG} 条
|
||
含义: 推送线程追不上业务产生的 info_read 速度
|
||
建议: 联系 AI 处理 UPDATE ecology_message_info_read SET BIZSTATE=1 WHERE BIZSTATE=0"
|
||
fi
|
||
|
||
# === 判定分支 ===
|
||
|
||
# v8 新增: 先查 workflow_requestbase 最近 20min 新流程(判断业务是否真在跑)
|
||
WF_NEW=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||
-sN -e "SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
|
||
|
||
# 分支 1: info_read=0 AND message_log=0
|
||
if [[ ${NEW_REQS} -eq 0 && ${MSG_COUNT} -eq 0 ]]; then
|
||
# v8 Layer 4 新增: workflow_requestbase 有新流程 → 业务在跑但两张表都停了
|
||
if [[ "${WF_NEW}" =~ ^[0-9]+$ && ${WF_NEW} -gt 0 ]]; then
|
||
log_warn "[WATCHDOG] ⚠️ info_read=0 + message_log=0 BUT workflow_requestbase 新流程=${WF_NEW} → 推送线程卡死 → 自愈!"
|
||
else
|
||
# 加一个 MySQL Threads_running 辅助检查:正常 2-4,Deadlock 时会 > 5
|
||
TR=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||
-sN -e "SHOW GLOBAL STATUS LIKE 'Threads_running';" 2>/dev/null | awk '{print $2}')
|
||
|
||
if [[ "${TR}" =~ ^[0-9]+$ && ${TR} -gt 5 ]]; then
|
||
# Threads_running 异常高 → 可能 Deadlock 全卡死 → 触发自愈
|
||
log_warn "[WATCHDOG] info_read=0 + message_log=0 + MySQL Threads_running=${TR} > 5 → 疑似 Deadlock 全卡死 → 自愈!"
|
||
else
|
||
log_info "[WATCHDOG] info_read=0, message_log=0, Threads_running=${TR:-?} 正常 → 推送线程空闲属正常, 不动"
|
||
exit 0
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# 分支 2: 有推送 → 推送线程活着,不动
|
||
if [[ ${MSG_COUNT} -gt 0 ]]; then
|
||
log_info "[WATCHDOG] 有新流程(${NEW_REQS}) + 有推送(${MSG_COUNT}), 推送健康 ✅"
|
||
exit 0
|
||
fi
|
||
|
||
# 分支 3: 有流程 + 无推送 → 推送线程死了!
|
||
HOUR=$(date +%H)
|
||
if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
|
||
log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), 有流程(${NEW_REQS})但无推送(${MSG_COUNT}), 仅记录"
|
||
exit 0
|
||
fi
|
||
|
||
# --- v6 新增: 自愈前抓 jstack 现场(推送线程死锁证据)---
|
||
JSTACK_TS=$(date +%Y%m%d_%H%M%S)
|
||
JSTACK_FILE="${JSTACK_DIR}/resin_jstack_before_selfheal_${JSTACK_TS}.txt"
|
||
|
||
log_warn "[WATCHDOG] 📸 自愈前抓 jstack 现场..."
|
||
JSTACK_CMD="RESIN_PID=\$(ps -eo pid,cmd | grep java.*resin.xml | grep -v grep | head -1 | awk '{print \$1}'); \
|
||
if [ -n \"\$RESIN_PID\" ]; then \
|
||
/usr/weaver/jdk1.8.0_151/bin/jstack -l \$RESIN_PID 2>&1; \
|
||
else \
|
||
echo 'ERROR: cannot find Resin PID'; \
|
||
fi"
|
||
|
||
${OA_SSH} "${JSTACK_CMD}" > "${JSTACK_FILE}" 2>/dev/null
|
||
JSTACK_LINES=$(wc -l < "${JSTACK_FILE}" 2>/dev/null || echo 0)
|
||
|
||
if [[ ${JSTACK_LINES} -gt 0 ]]; then
|
||
log_info "[WATCHDOG] ✅ jstack 现场已保存: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)"
|
||
else
|
||
log_warn "[WATCHDOG] ❌ jstack 抓取失败"
|
||
fi
|
||
|
||
# --- 触发自愈!---
|
||
log_warn "[WATCHDOG] ⚠️ 有新流程(${NEW_REQS})但 message_log=0 → 推送线程确认已死 → 触发 Resin 自愈重启!"
|
||
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
|
||
sleep 20
|
||
|
||
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
|
||
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
|
||
|
||
BODY="Resin watchdog 自动重启(v6 流程驱动判定)\n\n"
|
||
BODY="${BODY}最近 ${WINDOW_MIN}min 新流程: ${NEW_REQS} 条\n"
|
||
BODY="${BODY}最近 ${WINDOW_MIN}min message_log: ${MSG_COUNT} 条\n"
|
||
BODY="${BODY}判定: 有流程但无推送 → 推送线程已死\n\n"
|
||
BODY="${BODY}新 Resin PID: ${NEW_PID}\n"
|
||
BODY="${BODY}HTTP 80: ${HTTP_CODE}\n"
|
||
BODY="${BODY}jstack 现场: jstack_dump/resin_jstack_before_selfheal_${JSTACK_TS}.txt (${JSTACK_LINES} 行)"
|
||
|
||
# 钉钉附加关键线程片段(BLOCKED / info_read / updateBizState / 推送相关)
|
||
if [[ ${JSTACK_LINES} -gt 0 ]]; then
|
||
DEADLOCK_LINES=$(grep -A20 "Found one Java-level deadlock\|Found [0-9]* deadlock" "${JSTACK_FILE}" 2>/dev/null | head -25)
|
||
BLOCKED_COUNT=$(grep -c "java.lang.Thread.State: BLOCKED" "${JSTACK_FILE}" 2>/dev/null || echo 0)
|
||
BODY="${BODY}\n\n🧵 jstack 快览: BLOCKED=${BLOCKED_COUNT}"
|
||
if [[ -n "${DEADLOCK_LINES}" ]]; then
|
||
BODY="${BODY}\n🔥 DEADLOCK 发现:\n${DEADLOCK_LINES}"
|
||
fi
|
||
fi
|
||
|
||
dingtalk_alert "P0" "Resin watchdog 自动重启(有流程但无推送)" "${BODY}"
|
||
|
||
if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then
|
||
echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}"
|
||
fi
|
||
|
||
log_info "[WATCHDOG] Resin 自愈完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"
|