feat: watchdog v4 + check-alert v2 + collect-business v2 + DUMMY补数据脚本
This commit is contained in:
commit
7ab75364b1
22 files changed
+1795
No files matched your search
Executable
+97
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · Resin Watchdog (v3 - message_log 入库增量判定)
|
||||
# 健康依据:ecology_message_log 最近 15 分钟入库数
|
||||
# - 工作时间(08:00~22:00) 入库 < 10 条 → 判定推送线程死 → 重启 Resin
|
||||
# - 凌晨时段 入库少也不重启(避免误重启)
|
||||
# 部署在 DSH,python paramiko 到 OA 执行重启
|
||||
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
OA_HOST="39.99.236.138"
|
||||
OA_USER="root"
|
||||
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
|
||||
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
|
||||
|
||||
MSG_WINDOW_MIN=15 # 统计最近 15 分钟
|
||||
MSG_THRESHOLD=10 # 入库 < 10 条视为异常
|
||||
START_HOUR=8 # 工作时间起始
|
||||
END_HOUR=22 # 工作时间结束
|
||||
|
||||
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
|
||||
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
|
||||
|
||||
# Step 1: 确保 python paramiko 可用
|
||||
if ! python3 -c 'import paramiko' 2>/dev/null; then
|
||||
MSG="[WATCHDOG] 🔴 python3 + paramiko 不可用!请执行: pip3 install paramiko"
|
||||
log_error "${MSG}"
|
||||
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 2: 写 _oa_ssh.py(仅重启时用)
|
||||
cat > "${OA_SSH_PY}" << 'PYEOF'
|
||||
import paramiko, sys
|
||||
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
ssh.connect(host, username=user, password=password, timeout=10)
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
|
||||
ssh.close()
|
||||
except Exception as e:
|
||||
print(f"SSH_ERROR: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
chmod +x "${OA_SSH_PY}"
|
||||
|
||||
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
|
||||
|
||||
# Step 3: 查 message_log 最近 MSG_WINDOW_MIN 分钟入库数(DSH 直连 MySQL,走 pushlog_time 索引)
|
||||
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||||
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
|
||||
|
||||
if [[ ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
|
||||
log_warn "[WATCHDOG] ⚠️ 查询 message_log 失败(返回: ${MSG_COUNT}),跳过本次"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log_info "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 入库 ${MSG_COUNT} 条"
|
||||
|
||||
# Step 4: 判断
|
||||
HOUR=$(date +%H)
|
||||
if [[ ${MSG_COUNT} -ge ${MSG_THRESHOLD} ]]; then
|
||||
log_info "[WATCHDOG] ✅ message_log 入库正常(${MSG_COUNT} 条 ≥ ${MSG_THRESHOLD} 阈值),推送健康"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 入库 < 阈值
|
||||
if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
|
||||
log_warn "[WATCHDOG] ⚠️ 凌晨时段(${HOUR}时),message_log 仅 ${MSG_COUNT} 条 < ${MSG_THRESHOLD},仅记录不重启"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- 工作时间 + 入库过少 → 触发重启 ---
|
||||
log_warn "[WATCHDOG] 🔴 最近 ${MSG_WINDOW_MIN} 分钟 message_log 仅 ${MSG_COUNT} 条 < ${MSG_THRESHOLD},触发 Resin 重启!"
|
||||
|
||||
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
|
||||
|
||||
sleep 20
|
||||
|
||||
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
|
||||
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
|
||||
|
||||
BODY="**🔴 Resin 被 watchdog 自动重启**\n\n"
|
||||
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
||||
BODY="${BODY}| message_log 最近 ${MSG_WINDOW_MIN} 分钟入库 | **${MSG_COUNT} 条**(阈值 ${MSG_THRESHOLD})|\n"
|
||||
BODY="${BODY}| 新 Resin PID | \`${NEW_PID}\` |\n"
|
||||
BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n\n"
|
||||
BODY="${BODY}> ⚠️ 推送线程静默死掉(message_log 不再入库),watchdog 自动恢复。"
|
||||
BODY="${BODY}> 注:publishMessageList 日志可能仍在刷(假活跃),以 message_log 入库为准。"
|
||||
dingtalk_alert "P0" "Resin watchdog 自动重启(message_log 入库 ${MSG_COUNT} 条 < ${MSG_THRESHOLD})" "${BODY}"
|
||||
|
||||
log_info "[WATCHDOG] ✅ Resin 重启完成,新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"
|
||||
Reference in new issue
Block a user