fix: 修复 check-alert.sh 假阳性 + collect-business.sh 窗口对齐

- check-alert.sh 新增 Step0 数据新鲜度前置检查:business_metrics 超 30min 过期 →
  先告警采集脚本挂了,跳过业务层检测,防止读旧数据乱报推送断档
- collect-business.sh msg_log_hourly 窗口从 1h 改为 15min(和 crontab 频率对齐)
- CURR_HOUR 用 10# 算术强制十进制,防 08/09 被解析成 octal
This commit is contained in:
gsadmin authored and trae-solo-agent committed 2026-10-28 18:38:13 +08:00
1 parent 2e7504baf8
commit 90be249a72
10 files changed
+829

No files matched your search

+98
View File
@@ -0,0 +1,98 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · Resin 凌晨预防性重启
# 每天 02:00 执行,重建 push 线程池,静默死亡前主动清场
# crontab: 0 2 * * * /opt/oa-monitor/bin/resin-preventive-restart.sh >> /opt/oa-monitor/logs/resin-restart.log 2>&1
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
OA_HOST="39.99.236.138"
OA_USER="root"
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
LOG_FILE="${MONITOR_LOGS}/resin-preventive-restart.log"
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
START_TS=$(date '+%Y-%m-%d %H:%M:%S')
echo "=== ${START_TS} Resin 预防性重启开始 ===" | tee -a "${LOG_FILE}"
# Step 1: 写 _oa_ssh.py(paramiko + errors=replace,复用 watchdog 方式)
cat > "${OA_SSH_PY}" << 'PYEOF'
import paramiko, sys
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user, password=password, timeout=15)
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=180)
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
sys.stderr.write(stderr.read().decode("utf-8", errors="replace"))
ssh.close()
except Exception as e:
print(f"SSH_ERROR: {e}", file=sys.stderr)
sys.exit(1)
PYEOF
chmod +x "${OA_SSH_PY}"
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
# Step 2: 钉钉提前通知
dingtalk_alert "P1" "Resin 凌晨预防性重启(开始)" \
"**🛠️ Resin 预防性重启开始**\n\n时间:${START_TS}\n\n> 每天凌晨 02:00 执行,重建推送线程池,防止静默死亡。停机窗口约 30-60 秒。"
# Step 3: 记录旧 PID
OLD_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
echo "旧 PID=${OLD_PID}" | tee -a "${LOG_FILE}"
# Step 4: 停 Resin
echo "--- stopresin.sh ---" | tee -a "${LOG_FILE}"
${OA_SSH} "bash /usr/weaver/Resin/bin/stopresin.sh 2>&1" | tee -a "${LOG_FILE}"
# Step 5: 等端口关闭(最多 30 秒)
for i in $(seq 1 30); do
NEW_CHECK=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null)
if [[ "${NEW_CHECK}" != "200" && "${NEW_CHECK}" != "404" && "${NEW_CHECK}" != "401" ]]; then
echo "[${i}s] 80 端口已关闭 (HTTP=${NEW_CHECK})" | tee -a "${LOG_FILE}"
break
fi
sleep 1
done
sleep 3
# Step 6: 起 Resin
echo "--- startresin.sh ---" | tee -a "${LOG_FILE}"
${OA_SSH} "bash /usr/weaver/Resin/bin/startresin.sh 2>&1" | tee -a "${LOG_FILE}"
# Step 7: 等 Resin 恢复(最多 90 秒)
HTTP_CODE="000"
for i in $(seq 1 90); do
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null)
if [[ "${HTTP_CODE}" == "200" || "${HTTP_CODE}" == "404" || "${HTTP_CODE}" == "401" || "${HTTP_CODE}" == "500" ]]; then
echo "[${i}s] Resin 恢复 (HTTP=${HTTP_CODE})" | tee -a "${LOG_FILE}"
break
fi
sleep 1
done
# Step 8: 记录新 PID
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
END_TS=$(date '+%Y-%m-%d %H:%M:%S')
# Step 9: 钉钉结果通知
BODY="**🛠️ Resin 预防性重启完成**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| 旧 PID | \`${OLD_PID}\` |\n"
BODY="${BODY}| 新 PID | \`${NEW_PID}\` |\n"
BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n"
BODY="${BODY}| 耗时 | $(($(date +%s) - $(date -d "${START_TS}" +%s 2>/dev/null || date +%s)))秒 |\n"
BODY="${BODY}| 结束时间 | ${END_TS} |\n"
if [[ "${NEW_PID}" == "${OLD_PID}" ]]; then
BODY="${BODY}\n> ⚠️ 新旧 PID 相同,重启可能未生效,请检查!"
dingtalk_alert "P1" "Resin 预防性重启(⚠️ PID 未变)" "${BODY}"
else
dingtalk_alert "P1" "Resin 预防性重启(✅ 成功)" "${BODY}"
fi
echo "=== ${END_TS} Resin 预防性重启结束(${OLD_PID}→${NEW_PID}, HTTP=${HTTP_CODE}) ===" | tee -a "${LOG_FILE}"