- check-alert.sh 新增 Step0 数据新鲜度前置检查:business_metrics 超 30min 过期 → 先告警采集脚本挂了,跳过业务层检测,防止读旧数据乱报推送断档 - collect-business.sh msg_log_hourly 窗口从 1h 改为 15min(和 crontab 频率对齐) - CURR_HOUR 用 10# 算术强制十进制,防 08/09 被解析成 octal
148 lines
6.2 KiB
Bash
Executable File
148 lines
6.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# ============================================================================
|
||
# OA Monitor · Resin Watchdog (v4 - PID 变化检测 + message_log 入库判定)
|
||
# v3 盲区修复:Resin 被 WatchdogManager kill 后秒拉起,message_log 继续刷看起来"健康"
|
||
# v4 新增:每次巡检记录 Resin PID,PID 变化 -> 钉钉 P1 告警
|
||
# 健康依据(双重):
|
||
# 1. ecology_message_log 最近 15 分钟入库 >= 10 条(推送线程活着)
|
||
# 2. Resin PID 未变化(进程未被 kill 重启)
|
||
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
|
||
# ============================================================================
|
||
set -u
|
||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||
|
||
OA_HOST="39.99.236.138"
|
||
OA_USER="root"
|
||
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
|
||
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
|
||
|
||
MSG_WINDOW_MIN=15
|
||
MSG_THRESHOLD=10
|
||
START_HOUR=8
|
||
END_HOUR=22
|
||
|
||
PID_FILE="${MONITOR_ROOT}/state/resin_pid.last"
|
||
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
|
||
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
|
||
|
||
# --- 依赖检查 ---
|
||
if ! python3 -c 'import paramiko' 2>/dev/null; then
|
||
MSG="[WATCHDOG] paramiko 不可用!"
|
||
log_error "${MSG}"
|
||
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
|
||
exit 1
|
||
fi
|
||
|
||
# --- SSH helper ---
|
||
cat > "${OA_SSH_PY}" << 'PYEOF'
|
||
import paramiko, sys
|
||
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
|
||
ssh = paramiko.SSHClient()
|
||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||
try:
|
||
ssh.connect(host, username=user, password=password, timeout=10)
|
||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
|
||
ssh.close()
|
||
except Exception as e:
|
||
print(f"SSH_ERROR: {e}", file=sys.stderr)
|
||
sys.exit(1)
|
||
PYEOF
|
||
chmod +x "${OA_SSH_PY}"
|
||
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
|
||
|
||
# ============================================================================
|
||
# Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果)
|
||
# ============================================================================
|
||
mkdir -p "$(dirname "${PID_FILE}")"
|
||
|
||
PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null)
|
||
CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}')
|
||
CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}')
|
||
|
||
if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then
|
||
log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测"
|
||
else
|
||
LAST_PID=""
|
||
LAST_ETIMES=""
|
||
if [[ -f "${PID_FILE}" ]]; then
|
||
read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true
|
||
fi
|
||
|
||
if [[ -z "${LAST_PID}" ]]; then
|
||
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
|
||
log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s"
|
||
elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then
|
||
NOW_HOUR=$(date +%H)
|
||
NOW_MIN=$(date +%M)
|
||
IS_PREVENTIVE=0
|
||
if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then
|
||
IS_PREVENTIVE=1
|
||
fi
|
||
|
||
if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then
|
||
log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常"
|
||
else
|
||
RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
BODY="Resin PID 异常变化(非预期重启)\n\n"
|
||
BODY="${BODY}上次 PID: ${LAST_PID}\n"
|
||
BODY="${BODY}新 PID: ${CUR_PID}\n"
|
||
BODY="${BODY}推算重启时间: ${RESTART_TIME}\n"
|
||
BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒\n\n"
|
||
BODY="${BODY}Resin 被 WatchdogManager kill 后拉起。message_log 可能仍在刷(假活跃), watchdog v3 会漏检。"
|
||
dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}"
|
||
|
||
log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警"
|
||
fi
|
||
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
|
||
else
|
||
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
|
||
fi
|
||
fi
|
||
|
||
# ============================================================================
|
||
# Step 2: message_log 入库检测 (原 v3 逻辑)
|
||
# ============================================================================
|
||
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
|
||
|
||
if [[ ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
|
||
log_warn "[WATCHDOG] 查询 message_log 失败, 跳过"
|
||
exit 0
|
||
fi
|
||
|
||
log_info "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 入库 ${MSG_COUNT} 条"
|
||
|
||
HOUR=$(date +%H)
|
||
if [[ ${MSG_COUNT} -ge ${MSG_THRESHOLD} ]]; then
|
||
log_info "[WATCHDOG] message_log 入库正常(${MSG_COUNT} >= ${MSG_THRESHOLD}), 推送健康"
|
||
exit 0
|
||
fi
|
||
|
||
if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
|
||
log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), message_log 仅 ${MSG_COUNT} 条, 仅记录"
|
||
exit 0
|
||
fi
|
||
|
||
# --- 工作时间 + 入库过少 -> 触发重启 ---
|
||
log_warn "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 仅 ${MSG_COUNT} 条 < ${MSG_THRESHOLD}, 触发 Resin 重启!"
|
||
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
|
||
sleep 20
|
||
|
||
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
|
||
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
|
||
|
||
BODY="Resin watchdog 自动重启\n\n"
|
||
BODY="${BODY}message_log 最近 ${MSG_WINDOW_MIN} 分钟入库: ${MSG_COUNT} 条 (阈值 ${MSG_THRESHOLD})\n"
|
||
BODY="${BODY}新 Resin PID: ${NEW_PID}\n"
|
||
BODY="${BODY}HTTP 80: ${HTTP_CODE}\n"
|
||
dingtalk_alert "P0" "Resin watchdog 自动重启(message_log 入库 ${MSG_COUNT} 条 < ${MSG_THRESHOLD})" "${BODY}"
|
||
|
||
if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then
|
||
echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}"
|
||
fi
|
||
|
||
log_info "[WATCHDOG] Resin 重启完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"
|