From 7ab75364b1fe7ff35b293818dd6f533c312e1a16 Mon Sep 17 00:00:00 2001 From: oa-monitor Date: Wed, 4 Nov 2026 17:23:22 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20watchdog=20v4=20+=20check-alert=20v2=20?= =?UTF-8?q?+=20collect-business=20v2=20+=20DUMMY=E8=A1=A5=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 18 ++ bin/_oa_ssh.py | 12 ++ bin/check-alert.sh | 166 +++++++++++++++++++ bin/check-alert.sh.bak.v1.202610281838 | 155 +++++++++++++++++ bin/cleanup.sh | 36 ++++ bin/collect-business.sh | 50 ++++++ bin/collect-business.sh.bak.v1.202610281838 | 55 ++++++ bin/collect-db.sh | 98 +++++++++++ bin/collect-http.sh | 31 ++++ bin/collect-infra.sh | 53 ++++++ bin/init-db.sh | 84 ++++++++++ bin/resin-preventive-restart.sh | 98 +++++++++++ bin/resin-watchdog.sh | 146 ++++++++++++++++ bin/resin-watchdog.sh.bak.202610261116 | 84 ++++++++++ bin/resin-watchdog.sh.bak.gt0.20261030 | 147 ++++++++++++++++ bin/resin-watchdog.sh.bak.threshold.20261030 | 147 ++++++++++++++++ bin/resin-watchdog.sh.bak.v2.202610261128 | 96 +++++++++++ bin/resin-watchdog.sh.bak.v3.202610281415 | 97 +++++++++++ bin/run-long-check.sh | 66 ++++++++ docs/DUMMY-fix-plan.md | 49 ++++++ lib/common.sh | 106 ++++++++++++ state/resin_pid.last | 1 + 22 files changed, 1795 insertions(+) create mode 100644 .gitignore create mode 100755 bin/_oa_ssh.py create mode 100755 bin/check-alert.sh create mode 100755 bin/check-alert.sh.bak.v1.202610281838 create mode 100755 bin/cleanup.sh create mode 100755 bin/collect-business.sh create mode 100755 bin/collect-business.sh.bak.v1.202610281838 create mode 100755 bin/collect-db.sh create mode 100755 bin/collect-http.sh create mode 100755 bin/collect-infra.sh create mode 100755 bin/init-db.sh create mode 100755 bin/resin-preventive-restart.sh create mode 100755 bin/resin-watchdog.sh create mode 100755 bin/resin-watchdog.sh.bak.202610261116 create mode 100755 bin/resin-watchdog.sh.bak.gt0.20261030 create mode 100755 bin/resin-watchdog.sh.bak.threshold.20261030 create mode 100755 bin/resin-watchdog.sh.bak.v2.202610261128 create mode 100755 bin/resin-watchdog.sh.bak.v3.202610281415 create mode 100755 bin/run-long-check.sh create mode 100644 docs/DUMMY-fix-plan.md create mode 100755 lib/common.sh create mode 100644 state/resin_pid.last diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7ecfd60 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +# 敏感/运行时产物 +_state/ +logs/ +*.log +*_bak.* +*.bak.* +_oa_ssh.py + +# 凭证 +.env +*secret* +*credential* + +# 构建产物 +data/ +*.db +*.db-shm +*.db-wal diff --git a/bin/_oa_ssh.py b/bin/_oa_ssh.py new file mode 100755 index 0000000..44e0b21 --- /dev/null +++ b/bin/_oa_ssh.py @@ -0,0 +1,12 @@ +import paramiko, sys +host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +try: + ssh.connect(host, username=user, password=password, timeout=10) + stdin, stdout, stderr = ssh.exec_command(cmd) + sys.stdout.write(stdout.read().decode("utf-8", errors="replace")) + ssh.close() +except Exception as e: + print(f"SSH_ERROR: {e}", file=sys.stderr) + sys.exit(1) diff --git a/bin/check-alert.sh b/bin/check-alert.sh new file mode 100755 index 0000000..2f0e87b --- /dev/null +++ b/bin/check-alert.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 告警检测(每 1 分钟) +# v2 修复(2026-10-28): +# - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警 +# - Step 3 业务层检测被 Step 0 结果门禁 +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) +log_info "[ALERT] ========== 告警检测开始 ==========" + +# ============================================================================ +# Step 0: 数据新鲜度前置检查(防假阳性的第一层) +# ============================================================================ +# business_metrics 由 collect-business.sh 每 15min 写一次 +# db_metrics 由 collect-db.sh 每 30s 写一次 +# 如果最新 business_metrics ts 距现在 > 30min → 采集脚本挂了 → 告警+跳过业务层 +BIZ_TS=$(sqlite_exec "SELECT ts FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1) +DB_TS=$(sqlite_exec "SELECT ts FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1) +BIZ_FRESH=true +BIZ_AGE=0 + +if [[ ! "${BIZ_TS}" =~ ^[0-9]+$ ]]; then + log_error "[ALERT] business_metrics 表无数据或采集从未运行" + BIZ_FRESH=false + BIZ_AGE=999999 +elif [[ ! "${DB_TS}" =~ ^[0-9]+$ ]]; then + log_warn "[ALERT] db_metrics 表无数据(可能 MySQL 断了)" +else + BIZ_AGE=$((NOW - BIZ_TS)) + DB_AGE=$((NOW - DB_TS)) + + if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟 + BIZ_FRESH=false + HUMAN_AGE=$((BIZ_AGE / 60)) + BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec \"SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')\") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec \"SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')\") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警" + dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}" + log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)" + else + log_info "[ALERT] 数据新鲜:biz_age=${BIZ_AGE}s db_age=${DB_AGE}s" + fi +fi + +# ============================================================================ +# 1. DB 层检测(db_metrics 每分钟更新,数据可信) +# ============================================================================ +TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1) +if [[ -n "${TR}" ]] && [[ ${TR} =~ ^[0-9]+$ ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then + BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)" + dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');" +fi + +# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的) +LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;") +if [[ -n "${LONG_QUERIES}" ]]; then + COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ') + DETAIL="" + while IFS='|' read -r pid time user state info; do + DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`" + done <<< "${LONG_QUERIES}" + BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理" + dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');" +fi + +# ============================================================================ +# 2. HTTP 端点检测(http_metrics 每分钟更新,数据可信) +# ============================================================================ +FAIL_COUNT=0 +FAIL_DETAIL="" +for endpoint in "${OA_ENDPOINTS[@]}"; do + FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;" | head -1) + FAIL_HERE=${FAIL_HERE:-0} + if [[ "${FAIL_HERE}" =~ ^[0-9]+$ ]] && [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then + FAIL_COUNT=$((FAIL_COUNT + 1)) + LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;" | head -1) + FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}" + fi +done + +if [[ ${FAIL_COUNT} -gt 0 ]]; then + BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态" + dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');" +fi + +# ============================================================================ +# 3. 业务层检测(被 BIZ_FRESH 门禁) +# ============================================================================ +if [[ "${BIZ_FRESH}" != "true" ]]; then + log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)" +else + # 数据新鲜 → 正常业务层判断 + HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1) + ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1) + HOUR_MSG=${HOUR_MSG:-0} + ONGOING=${ONGOING:-0} + + CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal + WORK_HOURS_ENABLED=false + if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then + WORK_HOURS_ENABLED=true + fi + + if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then + if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then + BODY="**🔴 P0 钉钉推送完全断档!**\n\n" + BODY="${BODY}| 指标 | 值 |\n|---|---|\n" + BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n" + BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n" + BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n" + BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n" + BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n" + BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n" + BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**" + dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');" + fi + else + log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警" + fi +fi + +# ============================================================================ +# 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信) +# ============================================================================ +DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1) +PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;" | head -1) +if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then + BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> 请立即检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`" + dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');" +fi + +CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1) +AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;" | head -1) +if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]] && [[ "${CURR_COUNT}" =~ ^[0-9]+$ ]]; then + AVG_INT=${AVG_COUNT%%.*} + DELTA=$(( CURR_COUNT - AVG_INT )) + ABS_DELTA=${DELTA#-} + if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then + BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a" + dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');" + fi +fi + +DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1) +if [[ -n "${DISK_PCT}" ]] && [[ "${DISK_PCT}" =~ ^[0-9.]+$ ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then + BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)" + dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');" +fi + +INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1) +if [[ -n "${INODE_PCT}" ]] && [[ "${INODE_PCT}" =~ ^[0-9.]+$ ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then + BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)" + dingtalk_alert "P1" "DSH inode 告警" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');" +fi + +log_info "[ALERT] ========== 告警检测完成 ==========" diff --git a/bin/check-alert.sh.bak.v1.202610281838 b/bin/check-alert.sh.bak.v1.202610281838 new file mode 100755 index 0000000..3219166 --- /dev/null +++ b/bin/check-alert.sh.bak.v1.202610281838 @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 告警检测(每 1 分钟) +# 遍历所有指标,超过阈值触发钉钉告警 +# 告警冷却由 common.sh 的 alert_cooldown 表处理(10 分钟同类型不重复) +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) +log_info "[ALERT] ========== 告警检测开始 ==========" + +# ============================================================================ +# 1. DB 层检测 +# ============================================================================ +# Threads_running 过高 +TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;") +if [[ -n "${TR}" ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then + BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;')\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;')" + dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');" +fi + +# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的) +LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;") +if [[ -n "${LONG_QUERIES}" ]]; then + COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ') + DETAIL="" + while IFS='|' read -r pid time user state info; do + DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`" + done <<< "${LONG_QUERIES}" + BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理" + dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');" +fi + +# ============================================================================ +# 2. HTTP 端点检测 +# ============================================================================ +# 最近 3 次的失败次数 +FAIL_COUNT=0 +FAIL_DETAIL="" +for endpoint in "${OA_ENDPOINTS[@]}"; do + FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;") + FAIL_HERE=${FAIL_HERE:-0} + if [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then + FAIL_COUNT=$((FAIL_COUNT + 1)) + LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;") + FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}" + fi +done + +if [[ ${FAIL_COUNT} -gt 0 ]]; then + BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态" + dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');" +fi + +# ============================================================================ +# 3. 业务层检测 +# ============================================================================ +# 凌晨没人操作 OA 时 msg=0 本来正常——加时间窗口和业务活跃双重判断 +HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;") +ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;") +HOUR_MSG=${HOUR_MSG:-0} +ONGOING=${ONGOING:-0} + +# 当前小时数(用于判断是否工作时间) +CURR_HOUR=$(date +%H | sed 's/^0//') # 去掉前导零 +WORK_HOURS_ENABLED=false +if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then + WORK_HOURS_ENABLED=true # 08:00~22:00 才算工作时间 +fi + +if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then + # 只保留 P0:工作时间内,有业务在跑,但消息一条都没推出来 → 推送链路完全断裂 + # P1 已砍掉(噪音太大:上午刚开始上班 msg=99 也会触发,实际推送是正常的) + if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then + BODY="**🔴 P0 钉钉推送完全断档!**\n\n" + BODY="${BODY}| 指标 | 值 |\n|---|---|\n" + BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n" + BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n" + BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n\n" + BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n" + BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n" + BODY="${BODY}> **应急:SSH 到 OA 服务器执行 `ps aux | grep Resin` → `bash /usr/weaver/Resin/bin/stopresin.sh && startresin.sh`**" + if dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"; then + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');" + fi + fi +else + log_info "[BIZ] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警检查" +fi + +# ============================================================================ +# 4. 基础设施层检测 +# ============================================================================ +# Docker daemon 挂了 +DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;") +if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then + BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> ⚠️ 今天的 rm -rf /ipaas 事故中 Docker daemon 重启过。请立即登录 DSH 检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`" + dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');" +fi + +# 容器数突变(对比最近 30 分钟的平均数) +CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;") +if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]]; then + DELTA=$(( CURR_COUNT - ${AVG_COUNT%%.*} )) + ABS_DELTA=${DELTA#-} + if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then + BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a" + dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');" + fi +fi + +# 磁盘 +DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +if [[ -n "${DISK_PCT}" ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then + BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)\n\n> 请检查:df -h → du -sh /var/lib/docker/* → pm2 logs 清理 → 旧 SQLite 数据归档" + dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');" +fi + +# inode +INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +if [[ -n "${INODE_PCT}" ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then + BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)\n\n> 可能有大量小文件,检查 find / -type f | wc -l" + dingtalk_alert "P1" "DSH inode 告警" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');" +fi + +# Docker root 写入突增(读最新一条和上一条的差值) +ROOT_NOW=$(sqlite_exec "SELECT docker_root_write_mb FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +ROOT_PREV=$(sqlite_exec "SELECT docker_root_write_mb FROM infra_metrics ORDER BY ts DESC LIMIT 1 OFFSET 1;") +ROOT_DELTA=0 +if [[ -n "${ROOT_NOW}" ]] && [[ -n "${ROOT_PREV}" ]]; then + # 截断小数,bash $(()) 只认整数 + ROOT_NOW_INT=${ROOT_NOW%%.*} + ROOT_PREV_INT=${ROOT_PREV%%.*} + if [[ "${ROOT_NOW_INT}" =~ ^[0-9]+$ ]] && [[ "${ROOT_PREV_INT}" =~ ^[0-9]+$ ]]; then + ROOT_DELTA=$(( ROOT_NOW_INT - ROOT_PREV_INT )) + fi +fi +if [[ ${ROOT_DELTA} -ge ${ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN} ]]; then + BODY="**Docker overlay2 增长 ${ROOT_DELTA}MB**(阈值 ${ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN}MB)\n\n> 可能有容器在疯狂写数据,检查:docker stats && docker logs" + dingtalk_alert "P2" "Docker data-root 写入突增" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P2', 'infra', 'Docker root 写入突增', 'delta_mb=${ROOT_DELTA}');" +fi + +log_info "[ALERT] ========== 告警检测完成 ==========" diff --git a/bin/cleanup.sh b/bin/cleanup.sh new file mode 100755 index 0000000..3332145 --- /dev/null +++ b/bin/cleanup.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 数据清理(每天 03:00) +# 保留最近 7 天指标,alerts 和 alert_cooldown 保留 30 天 +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) +RETENTION_7D=$(( NOW - 7 * 86400 )) +RETENTION_30D=$(( NOW - 30 * 86400 )) + +log_info "[CLEANUP] 开始清理,阈值 ts < ${RETENTION_7D}(7天),alerts < ${RETENTION_30D}(30天)" + +# 各层指标表 +for table in db_metrics http_metrics business_metrics infra_metrics db_processlist; do + BEFORE=$(sqlite_exec "SELECT COUNT(*) FROM ${table};") + sqlite_exec "DELETE FROM ${table} WHERE ts < ${RETENTION_7D};" + AFTER=$(sqlite_exec "SELECT COUNT(*) FROM ${table};") + log_info "[CLEANUP] ${table}: ${BEFORE} → ${AFTER}(删了 $((BEFORE - AFTER)) 条)" +done + +# alerts 保留 30 天 +BEFORE=$(sqlite_exec "SELECT COUNT(*) FROM alerts;") +sqlite_exec "DELETE FROM alerts WHERE ts < ${RETENTION_30D};" +AFTER=$(sqlite_exec "SELECT COUNT(*) FROM alerts;") +log_info "[CLEANUP] alerts: ${BEFORE} → ${AFTER}(删了 $((BEFORE - AFTER)) 条)" + +# VACUUM 压缩(可选,不频繁) +if (( NOW % (86400 * 7) < 3600 )); then + log_info "[CLEANUP] 每周 VACUUM SQLite..." + sqlite_exec "VACUUM;" +fi + +log_info "[CLEANUP] 完成" diff --git a/bin/collect-business.sh b/bin/collect-business.sh new file mode 100755 index 0000000..98edeeb --- /dev/null +++ b/bin/collect-business.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 业务指标采集(每 15 分钟,crontab: */15 * * * *) +# v2 修复(2026-10-28): +# - msg_log_hourly 实际用 15min 窗口(和 crontab 频率对齐),跨天逻辑简化 +# - 时间字段正确使用 CREATEDATE varchar(10) + CREATETIME varchar(8) +# ecology_message_log 没有 DATETIME 类型的 pushlog_time 字段! +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) +NOW_DATE=$(date '+%Y-%m-%d') + +# --- 1. ecology_message_log 15 分钟增量 --- +# v2 简化:15min 窗口几乎不跨天,直接用 CURDATE() +MSG_WINDOW=15 +MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e " + SELECT COUNT(*) FROM ecology_message_log + WHERE CREATEDATE = CURDATE() + AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW} MINUTE, '%H:%i:%s'); +" 2>/dev/null | tail -1) +MSG_COUNT=${MSG_COUNT:-0} + +# --- 2. 正在流转的审批数 --- +ONGOING=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e " + SELECT COUNT(*) FROM workflow_requestbase + WHERE CURRENTNODEID > 0 AND (STATUS IS NULL OR STATUS NOT LIKE '%归档%'); +" 2>/dev/null | tail -1) +ONGOING=${ONGOING:-0} + +# --- 3. 今日新建审批数 --- +NEW_TODAY=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e " + SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE = '${NOW_DATE}'; +" 2>/dev/null | tail -1) +NEW_TODAY=${NEW_TODAY:-0} + +# --- 4. 超时任务数 --- +OVERDUE=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e " + SELECT COUNT(*) FROM workflow_requestlog + WHERE timeout = 1 AND operatedate >= DATE_SUB(CURDATE(), INTERVAL 1 DAY); +" 2>/dev/null | tail -1) +OVERDUE=${OVERDUE:-0} + +sqlite_exec "INSERT INTO business_metrics VALUES ( + ${NOW}, ${MSG_COUNT}, 0.0, ${ONGOING}, ${OVERDUE} +);" + +log_info "[BIZ] ✅ msg_15min=${MSG_COUNT} ongoing=${ONGOING} new_today=${NEW_TODAY} overdue=${OVERDUE}" diff --git a/bin/collect-business.sh.bak.v1.202610281838 b/bin/collect-business.sh.bak.v1.202610281838 new file mode 100755 index 0000000..6dc8ad1 --- /dev/null +++ b/bin/collect-business.sh.bak.v1.202610281838 @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 业务指标采集(每 15 分钟) +# ⚠️ ecology_message_log 没有 created datetime 列! +# 时间用 CREATEDATE varchar(10) + CREATETIME varchar(8) 拼接 +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) +NOW_DATE=$(date '+%Y-%m-%d') +NOW_HOUR=$(date '+%H') +NOW_MIN=$(date '+%M') +# 1 小时前的时间(用于计算小时增量) +HOUR_AGO_DATE=$(date -v-1H '+%Y-%m-%d' 2>/dev/null || date --date='1 hour ago' '+%Y-%m-%d' 2>/dev/null) +HOUR_AGO_TIME=$(date -v-1H '+%H:%M:%S' 2>/dev/null || date --date='1 hour ago' '+%H:%M:%S' 2>/dev/null) + +# --- 1. ecology_message_log 小时增量(正确用 CREATEDATE + CREATETIME 拼接)--- +MSG_HOURLY=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e " + SELECT COUNT(*) FROM ecology_message_log + WHERE (CREATEDATE = '${NOW_DATE}' AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL 1 HOUR, '%H:%i:%s')) + OR (CREATEDATE = '${HOUR_AGO_DATE}' AND CREATETIME >= '${HOUR_AGO_TIME}'); +" 2>/dev/null | tail -1) +MSG_HOURLY=${MSG_HOURLY:-0} + +# --- 2. 正在流转的审批数 --- +# 泛微 CURRENTSTATUS 全 NULL,STATUS 存中文节点名(如"行政审批层级1 /审批人1 至 归档") +# 判断:CURRENTNODEID > 0 且不包含"归档" +ONGOING=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e " + SELECT COUNT(*) FROM workflow_requestbase + WHERE CURRENTNODEID > 0 AND (STATUS IS NULL OR STATUS NOT LIKE '%归档%'); +" 2>/dev/null | tail -1) +ONGOING=${ONGOING:-0} + +# --- 3. 今日新建审批数 --- +NEW_TODAY=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e " + SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE = '${NOW_DATE}'; +" 2>/dev/null | tail -1) +NEW_TODAY=${NEW_TODAY:-0} + +# --- 4. 超时任务数 --- +OVERDUE=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e " + SELECT COUNT(*) FROM workflow_requestlog + WHERE timeout = 1 AND operatedate >= DATE_SUB(CURDATE(), INTERVAL 1 DAY); +" 2>/dev/null | tail -1) +OVERDUE=${OVERDUE:-0} + +FAIL_RATE="0.0" # ecology_message_log 没有失败率字段,暂置 0 + +sqlite_exec "INSERT INTO business_metrics VALUES ( + ${NOW}, ${MSG_HOURLY}, ${FAIL_RATE}, ${ONGOING}, ${OVERDUE} +);" + +log_info "[BIZ] ✅ msg_hourly=${MSG_HOURLY} ongoing=${ONGOING} new_today=${NEW_TODAY} overdue=${OVERDUE}" diff --git a/bin/collect-db.sh b/bin/collect-db.sh new file mode 100755 index 0000000..9574b18 --- /dev/null +++ b/bin/collect-db.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · DB 层采集(每 30 秒) +# 在 DSH 宿主机上直接 mysql 连 OA DB(DSH 本身就是跳板,不需要再 ssh) +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) + +# ---- MySQL 直连(DSH 上直接连 OA DB 公网映射)---- +DB_MYSQL() { + MYSQL_PWD="${DB_PASS}" timeout 12 mysql -h39.100.72.102 -P3306 -uecology ecology "$@" 2>/dev/null +} + +db_exec() { + DB_MYSQL -e "$1" +} + +# ---- 1. SHOW STATUS ---- +log_info "[DB] SHOW STATUS" +STATUS=$(DB_MYSQL -e "SHOW GLOBAL STATUS" | awk -F'\t' ' + $1=="Threads_running"||$1=="Threads_connected"||$1=="Threads_idle"|| + $1=="Uptime"||$1=="Innodb_rows_read"||$1=="Innodb_rows_inserted"|| + $1=="Innodb_rows_updated"||$1=="Innodb_rows_deleted"||$1=="Slow_queries" { print } +') + +THREADS_RUNNING=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_running"{print $2}') +THREADS_CONNECTED=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_connected"{print $2}') +THREADS_IDLE=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_idle"{print $2}') +UPTIME=$(echo "${STATUS}" | awk -F'\t' '$1=="Uptime"{print $2}') +INNODB_READ=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_read"{print $2}') +INNODB_INS=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_inserted"{print $2}') +INNODB_UPD=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_updated"{print $2}') +INNODB_DEL=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_deleted"{print $2}') +SLOW_QUERIES=$(echo "${STATUS}" | awk -F'\t' '$1=="Slow_queries"{print $2}') + +INNODB_READ=${INNODB_READ:-0} +INNODB_INS=${INNODB_INS:-0} +INNODB_UPD=${INNODB_UPD:-0} +INNODB_DEL=${INNODB_DEL:-0} +INNODB_WRITE=$(( INNODB_INS + INNODB_UPD + INNODB_DEL )) +THREADS_RUNNING=${THREADS_RUNNING:-0} +THREADS_CONNECTED=${THREADS_CONNECTED:-0} +THREADS_IDLE=${THREADS_IDLE:-0} +UPTIME=${UPTIME:-0} +SLOW_QUERIES=${SLOW_QUERIES:-0} + +# ---- 2. 上一次快照 → 差值 ---- +PREV=$(sqlite_exec "SELECT * FROM db_metrics ORDER BY ts DESC LIMIT 1;") +QPS_READ=0; QPS_WRITE=0; SLOW_DELTA=0 + +if [[ -n "${PREV}" ]]; then + PREV_UPTIME=$(echo "${PREV}" | cut -d'|' -f5) + PREV_READ=$(echo "${PREV}" | cut -d'|' -f6) + PREV_WRITE=$(echo "${PREV}" | cut -d'|' -f7) + PREV_SLOW=$(echo "${PREV}" | cut -d'|' -f9) + PREV_UPTIME=${PREV_UPTIME:-0}; PREV_READ=${PREV_READ:-0} + PREV_WRITE=${PREV_WRITE:-0}; PREV_SLOW=${PREV_SLOW:-0} + DIFF_SEC=$(( NOW - PREV_UPTIME )) + if [[ ${DIFF_SEC} -gt 0 ]]; then + QPS_READ=$(( (INNODB_READ - PREV_READ) / DIFF_SEC )) + QPS_WRITE=$(( (INNODB_WRITE - PREV_WRITE) / DIFF_SEC )) + fi + SLOW_DELTA=$(( SLOW_QUERIES - PREV_SLOW )) +fi + +# ---- 3. 写 SQLite ---- +sqlite_exec "INSERT INTO db_metrics VALUES ( + ${NOW}, ${THREADS_RUNNING}, ${THREADS_CONNECTED}, ${THREADS_IDLE}, + ${UPTIME}, ${QPS_READ}, ${QPS_WRITE}, ${SLOW_DELTA} +);" + +log_info "[DB] ✅ 写入:Threads_running=${THREADS_RUNNING} QPS_read=${QPS_READ} QPS_write=${QPS_WRITE}" + +# ---- 4. 长查询 processlist ---- +log_info "[DB] processlist" +PLIST=$(DB_MYSQL -e 'SHOW PROCESSLIST;') + +echo "${PLIST}" | awk -F'\t' ' + NR==1 { next } + NF < 8 { next } + { + pid=$1; user=$2; state=$6; time=$7; info=$8 + if (time+0 >= 5 && pid+0 > 0) { + gsub(/\t/, " ", info) + printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300) + } + } +' | while IFS=$'\t' read -r pid time user state info; do + [[ -z "${pid}" ]] && continue + info_safe=$(echo "${info}" | sed "s/'/\"/g") + sqlite_exec "INSERT INTO db_processlist (ts, pid, user, state, time_sec, info) + VALUES (${NOW}, ${pid}, '${user}', '${state}', ${time}, '${info_safe}');" +done + +log_info "[DB] ✅ 完成" diff --git a/bin/collect-http.sh b/bin/collect-http.sh new file mode 100755 index 0000000..bd94e56 --- /dev/null +++ b/bin/collect-http.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · HTTP 端点健康检查(每 30 秒) +# 在 DSH 上直接 curl +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) + +for endpoint in "${OA_ENDPOINTS[@]}"; do + result=$(curl -sk -o /dev/null -w '%{http_code}|%{time_total}' --max-time 10 "${endpoint}" 2>/dev/null) + status=$(echo "${result}" | cut -d'|' -f1) + time_ms=$(echo "${result}" | cut -d'|' -f2 | awk '{printf "%d", $1*1000}') + status=${status:-0} + time_ms=${time_ms:-0} + + case "${status}" in + 000) healthy=0 ;; # 连不上 = 挂了 + 5*) healthy=0 ;; # 5xx = 服务异常 + *) healthy=1 ;; # 2xx/3xx/4xx 都算活着(4xx 是业务路径不对,但服务本身通了) + esac + + sqlite_exec "INSERT INTO http_metrics (ts, endpoint, status_code, resp_time_ms, healthy) + VALUES (${NOW}, '${endpoint}', ${status}, ${time_ms}, ${healthy});" + + log_info "[HTTP] ${endpoint} status=${status} time=${time_ms}ms healthy=${healthy}" +done + +log_info "[HTTP] 全部完成" diff --git a/bin/collect-infra.sh b/bin/collect-infra.sh new file mode 100755 index 0000000..f89ada7 --- /dev/null +++ b/bin/collect-infra.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 基础设施层采集(每 30 秒) +# 在 DSH 上直接执行 docker / df / du +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) +DOCKER_ROOT="/var/lib/docker" + +# ---- Docker daemon + 容器数 ---- +CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ') +if [[ "${CONTAINER_COUNT}" =~ ^[0-9]+$ ]] && [[ ${CONTAINER_COUNT} -ge 0 ]]; then + DOCKER_OK=1 +else + DOCKER_OK=0 + CONTAINER_COUNT=0 +fi + +# ---- Docker root 写入量 ---- +# docker_root_write_mb 字段存储 "上一次 du overlay2 的总大小 MB" +# 告警脚本里直接看 "本次 du - 上次存的值" 的差值 +PREV_TOTAL_MB=$(sqlite_exec "SELECT COALESCE(docker_root_write_mb, 0) FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +CURRENT_TOTAL_MB=$(du -sm "${DOCKER_ROOT}/overlay2" 2>/dev/null | cut -f1 | tr -d ' ') +PREV_TOTAL_MB=${PREV_TOTAL_MB:-0} +CURRENT_TOTAL_MB=${CURRENT_TOTAL_MB:-0} + +# 计算增量(日志用) +DELTA=0 +if [[ "${PREV_TOTAL_MB}" =~ ^[0-9]+$ ]] && [[ ${CURRENT_TOTAL_MB} -gt ${PREV_TOTAL_MB} ]]; then + DELTA=$(( CURRENT_TOTAL_MB - PREV_TOTAL_MB )) +elif [[ ! "${PREV_TOTAL_MB}" =~ ^[0-9]+$ ]]; then + DELTA=-1 # 首次采,设 -1 表示"无对比" +fi + +# ---- 磁盘 / inode / load ---- +DISK_PCT=$(df / | tail -1 | awk '{print $5}' | tr -d '%') +INODE_PCT=$(df -i / | tail -1 | awk '{print $5}' | tr -d '%') +LOADAVG=$(cat /proc/loadavg | awk '{print $1}') +DISK_PCT=${DISK_PCT:-0} +INODE_PCT=${INODE_PCT:-0} +LOADAVG=${LOADAVG:-0} + +# 写 SQLite(把"当前总大小 MB"存起来,下次算差值) +# 注意 docker_root_write_mb 存的是整数 MB(du -sm 输出本身就是整数,.0 是 sqlite REAL 类型加的) +sqlite_exec "INSERT INTO infra_metrics VALUES ( + ${NOW}, ${DOCKER_OK}, ${CONTAINER_COUNT}, CAST(${CURRENT_TOTAL_MB} AS INTEGER), + ${DISK_PCT}, ${INODE_PCT}, ${LOADAVG} +);" + +log_info "[INFRA] ✅ docker_ok=${DOCKER_OK} containers=${CONTAINER_COUNT} root_overlay2=${CURRENT_TOTAL_MB}MB delta=${DELTA}MB disk=${DISK_PCT}% inode=${INODE_PCT}% loadavg=${LOADAVG}" diff --git a/bin/init-db.sh b/bin/init-db.sh new file mode 100755 index 0000000..0ea0c1b --- /dev/null +++ b/bin/init-db.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · SQLite 数据库初始化 +# 只建表,不删数据;幂等可重复执行 +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +log_info "初始化 SQLite 数据库:${MONITOR_DB}" + +sqlite_exec "CREATE TABLE IF NOT EXISTS db_metrics ( + ts INTEGER PRIMARY KEY, + threads_running INTEGER, + threads_connected INTEGER, + threads_idle INTEGER, + uptime_sec INTEGER, + qps_read INTEGER, + qps_write INTEGER, + slow_queries INTEGER -- 近 30s 慢查询数(>30s) +);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS db_processlist ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER, + pid INTEGER, + user TEXT, + state TEXT, + time_sec INTEGER, + info TEXT +);" + +sqlite_exec "CREATE INDEX IF NOT EXISTS idx_db_proc_ts ON db_processlist(ts);" +sqlite_exec "CREATE INDEX IF NOT EXISTS idx_db_proc_time ON db_processlist(time_sec);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS http_metrics ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER, + endpoint TEXT, + status_code INTEGER, + resp_time_ms INTEGER, + healthy INTEGER -- 0/1 +);" + +sqlite_exec "CREATE INDEX IF NOT EXISTS idx_http_ts_ep ON http_metrics(ts, endpoint);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS business_metrics ( + ts INTEGER PRIMARY KEY, + msg_log_hourly INTEGER, -- 本小时 ecology_message_log 增量 + msg_log_fail_rate REAL, -- 失败率 + ongoing_requests INTEGER, -- 进行中流程数 + overdue_tasks INTEGER -- 超时待办数 +);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS infra_metrics ( + ts INTEGER PRIMARY KEY, + docker_daemon_ok INTEGER, -- 0/1 + docker_container_count INTEGER, + docker_root_write_mb REAL, -- 1 分钟 Docker data-root 写入量 MB + disk_usage_pct REAL, + inode_usage_pct REAL, + load_avg REAL +);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS alerts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER, + level TEXT, -- P0/P1/P2 + category TEXT, -- db/http/business/infra + title TEXT, + detail TEXT, + ack INTEGER DEFAULT 0, -- 0=未确认 1=已确认 + resolved INTEGER DEFAULT 0 -- 0=未恢复 1=已恢复 +);" + +sqlite_exec "CREATE INDEX IF NOT EXISTS idx_alerts_ts ON alerts(ts);" +sqlite_exec "CREATE INDEX IF NOT EXISTS idx_alerts_ack ON alerts(ack, resolved);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS alert_cooldown ( + alert_key TEXT PRIMARY KEY, + last_alert_ts INTEGER +);" + +log_info "SQLite schema 初始化完成" diff --git a/bin/resin-preventive-restart.sh b/bin/resin-preventive-restart.sh new file mode 100755 index 0000000..ae6cef2 --- /dev/null +++ b/bin/resin-preventive-restart.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · Resin 凌晨预防性重启 +# 每天 02:00 执行,重建 push 线程池,静默死亡前主动清场 +# crontab: 0 2 * * * /opt/oa-monitor/bin/resin-preventive-restart.sh >> /opt/oa-monitor/logs/resin-restart.log 2>&1 +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +OA_HOST="39.99.236.138" +OA_USER="root" +OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM" +LOG_FILE="${MONITOR_LOGS}/resin-preventive-restart.log" +OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py" + +START_TS=$(date '+%Y-%m-%d %H:%M:%S') +echo "=== ${START_TS} Resin 预防性重启开始 ===" | tee -a "${LOG_FILE}" + +# Step 1: 写 _oa_ssh.py(paramiko + errors=replace,复用 watchdog 方式) +cat > "${OA_SSH_PY}" << 'PYEOF' +import paramiko, sys +host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +try: + ssh.connect(host, username=user, password=password, timeout=15) + stdin, stdout, stderr = ssh.exec_command(cmd, timeout=180) + sys.stdout.write(stdout.read().decode("utf-8", errors="replace")) + sys.stderr.write(stderr.read().decode("utf-8", errors="replace")) + ssh.close() +except Exception as e: + print(f"SSH_ERROR: {e}", file=sys.stderr) + sys.exit(1) +PYEOF +chmod +x "${OA_SSH_PY}" +OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}" + +# Step 2: 钉钉提前通知 +dingtalk_alert "P1" "Resin 凌晨预防性重启(开始)" \ + "**🛠️ Resin 预防性重启开始**\n\n时间:${START_TS}\n\n> 每天凌晨 02:00 执行,重建推送线程池,防止静默死亡。停机窗口约 30-60 秒。" + +# Step 3: 记录旧 PID +OLD_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null) +echo "旧 PID=${OLD_PID}" | tee -a "${LOG_FILE}" + +# Step 4: 停 Resin +echo "--- stopresin.sh ---" | tee -a "${LOG_FILE}" +${OA_SSH} "bash /usr/weaver/Resin/bin/stopresin.sh 2>&1" | tee -a "${LOG_FILE}" + +# Step 5: 等端口关闭(最多 30 秒) +for i in $(seq 1 30); do + NEW_CHECK=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null) + if [[ "${NEW_CHECK}" != "200" && "${NEW_CHECK}" != "404" && "${NEW_CHECK}" != "401" ]]; then + echo "[${i}s] 80 端口已关闭 (HTTP=${NEW_CHECK})" | tee -a "${LOG_FILE}" + break + fi + sleep 1 +done + +sleep 3 + +# Step 6: 起 Resin +echo "--- startresin.sh ---" | tee -a "${LOG_FILE}" +${OA_SSH} "bash /usr/weaver/Resin/bin/startresin.sh 2>&1" | tee -a "${LOG_FILE}" + +# Step 7: 等 Resin 恢复(最多 90 秒) +HTTP_CODE="000" +for i in $(seq 1 90); do + HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null) + if [[ "${HTTP_CODE}" == "200" || "${HTTP_CODE}" == "404" || "${HTTP_CODE}" == "401" || "${HTTP_CODE}" == "500" ]]; then + echo "[${i}s] Resin 恢复 (HTTP=${HTTP_CODE})" | tee -a "${LOG_FILE}" + break + fi + sleep 1 +done + +# Step 8: 记录新 PID +NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null) +END_TS=$(date '+%Y-%m-%d %H:%M:%S') + +# Step 9: 钉钉结果通知 +BODY="**🛠️ Resin 预防性重启完成**\n\n" +BODY="${BODY}| 指标 | 值 |\n|---|---|\n" +BODY="${BODY}| 旧 PID | \`${OLD_PID}\` |\n" +BODY="${BODY}| 新 PID | \`${NEW_PID}\` |\n" +BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n" +BODY="${BODY}| 耗时 | $(($(date +%s) - $(date -d "${START_TS}" +%s 2>/dev/null || date +%s)))秒 |\n" +BODY="${BODY}| 结束时间 | ${END_TS} |\n" + +if [[ "${NEW_PID}" == "${OLD_PID}" ]]; then + BODY="${BODY}\n> ⚠️ 新旧 PID 相同,重启可能未生效,请检查!" + dingtalk_alert "P1" "Resin 预防性重启(⚠️ PID 未变)" "${BODY}" +else + dingtalk_alert "P1" "Resin 预防性重启(✅ 成功)" "${BODY}" +fi + +echo "=== ${END_TS} Resin 预防性重启结束(${OLD_PID}→${NEW_PID}, HTTP=${HTTP_CODE}) ===" | tee -a "${LOG_FILE}" diff --git a/bin/resin-watchdog.sh b/bin/resin-watchdog.sh new file mode 100755 index 0000000..870307b --- /dev/null +++ b/bin/resin-watchdog.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · Resin Watchdog (v4 - PID 变化检测 + message_log 入库判定) +# v3 盲区修复:Resin 被 WatchdogManager kill 后秒拉起,message_log 继续刷看起来"健康" +# v4 新增:每次巡检记录 Resin PID,PID 变化 -> 钉钉 P1 告警 +# 健康依据(双重): +# 1. ecology_message_log 最近 15 分钟入库 count > 0(推送线程活着) +# 2. Resin PID 未变化(进程未被 kill 重启) +# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +OA_HOST="39.99.236.138" +OA_USER="root" +OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM" +OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1" + +MSG_WINDOW_MIN=15 +START_HOUR=8 +END_HOUR=22 + +PID_FILE="${MONITOR_ROOT}/state/resin_pid.last" +LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log" +OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py" + +# --- 依赖检查 --- +if ! python3 -c 'import paramiko' 2>/dev/null; then + MSG="[WATCHDOG] paramiko 不可用!" + log_error "${MSG}" + dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}" + exit 1 +fi + +# --- SSH helper --- +cat > "${OA_SSH_PY}" << 'PYEOF' +import paramiko, sys +host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +try: + ssh.connect(host, username=user, password=password, timeout=10) + stdin, stdout, stderr = ssh.exec_command(cmd) + sys.stdout.write(stdout.read().decode("utf-8", errors="replace")) + ssh.close() +except Exception as e: + print(f"SSH_ERROR: {e}", file=sys.stderr) + sys.exit(1) +PYEOF +chmod +x "${OA_SSH_PY}" +OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}" + +# ============================================================================ +# Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果) +# ============================================================================ +mkdir -p "$(dirname "${PID_FILE}")" + +PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null) +CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}') +CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}') + +if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then + log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测" +else + LAST_PID="" + LAST_ETIMES="" + if [[ -f "${PID_FILE}" ]]; then + read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true + fi + + if [[ -z "${LAST_PID}" ]]; then + echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" + log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s" + elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then + NOW_HOUR=$(date +%H) + NOW_MIN=$(date +%M) + IS_PREVENTIVE=0 + if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then + IS_PREVENTIVE=1 + fi + + if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then + log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常" + else + RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S') + + BODY="Resin PID 异常变化(非预期重启)\n\n" + BODY="${BODY}上次 PID: ${LAST_PID}\n" + BODY="${BODY}新 PID: ${CUR_PID}\n" + BODY="${BODY}推算重启时间: ${RESTART_TIME}\n" + BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒\n\n" + BODY="${BODY}Resin 被 WatchdogManager kill 后拉起。message_log 可能仍在刷(假活跃), watchdog v3 会漏检。" + dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}" + + log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警" + fi + echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" + else + echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" + fi +fi + +# ============================================================================ +# Step 2: message_log 入库检测 (原 v3 逻辑) +# ============================================================================ +MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null) + +if [[ ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then + log_warn "[WATCHDOG] 查询 message_log 失败, 跳过" + exit 0 +fi + +log_info "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 入库 ${MSG_COUNT} 条" + +HOUR=$(date +%H) +if [[ ${MSG_COUNT} -gt 0 ]]; then + log_info "[WATCHDOG] message_log 入库正常(${MSG_COUNT} > 0), 推送健康" + exit 0 +fi + +if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then + log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), message_log 仅 ${MSG_COUNT} 条, 仅记录" + exit 0 +fi + +# --- 工作时间 + 入库过少 -> 触发重启 --- +log_warn "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log count=0, 推送线程可能已死, 触发 Resin 重启!" +${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}" +sleep 20 + +NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1) +HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null) + +BODY="Resin watchdog 自动重启\n\n" +BODY="${BODY}message_log 最近 ${MSG_WINDOW_MIN} 分钟入库: ${MSG_COUNT} 条 (count > 0)\n" +BODY="${BODY}新 Resin PID: ${NEW_PID}\n" +BODY="${BODY}HTTP 80: ${HTTP_CODE}\n" +dingtalk_alert "P0" "Resin watchdog 自动重启(message_log count=0)" "${BODY}" + +if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then + echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}" +fi + +log_info "[WATCHDOG] Resin 重启完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}" diff --git a/bin/resin-watchdog.sh.bak.202610261116 b/bin/resin-watchdog.sh.bak.202610261116 new file mode 100755 index 0000000..f48192b --- /dev/null +++ b/bin/resin-watchdog.sh.bak.202610261116 @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · Resin Watchdog +# 监控 Util_Message 线程活跃度(publishMessageList),超过 30 分钟无活动则自动重启 Resin +# 部署在 DSH 上,通过 sshpass 到 OA 服务器(39.99.236.138)执行 +# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +OA_HOST="39.99.236.138" +OA_USER="root" +OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM" +OA_LOG="/usr/weaver/Resin/log/stdout.log" +OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1" +MAX_IDLE_MIN=30 # 30 分钟没 publishMessageList 活动就重启 +RESIN_PATTERN="com.cloudstore.dev.api.util.Util_Message.*publishMessageList is start" + +NOW=$(date +%s) + +# --- 1. sshpass 可用性检查 --- +if ! command -v sshpass &>/dev/null; then + log_warn "[WATCHDOG] sshpass 未安装,尝试安装..." + ssh root@${OA_HOST} "which sshpass 2>/dev/null" && SSHPASS_OK=true || SSHPASS_OK=false +fi + +# --- 2. ssh 到 OA 服务器查 publishMessageList 最后活动时间 --- +LAST_TS=$(sshpass -p "${OA_PASS}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${OA_USER}@${OA_HOST} " + # 从 stdout.log 最后 500 行找 publishMessageList + tail -5000 ${OA_LOG} 2>/dev/null | grep -E \"${RESIN_PATTERN}\" | tail -1 | grep -oP '\\d{4}\\.\\d{2}\\.\\d{2} \\d{2}:\\d{2}:\\d{2}' | head -1 +" 2>/dev/null) + +# 如果 stdout.log 没有,查 yesterday.log +if [[ -z "${LAST_TS}" ]]; then + LAST_TS=$(sshpass -p "${OA_PASS}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${OA_USER}@${OA_HOST} " + find /usr/weaver/Resin/log -name '*.log*' -newer ${OA_LOG}.yesterday 2>/dev/null | head -3 | xargs grep -lE '${RESIN_PATTERN}' 2>/dev/null | head -1 | xargs grep -E '${RESIN_PATTERN}' 2>/dev/null | tail -1 | grep -oP '\\d{4}\\.\\d{2}\\.\\d{2} \\d{2}:\\d{2}:\\d{2}' | head -1 + " 2>/dev/null) +fi + +# --- 3. 判断是否需要重启 --- +if [[ -z "${LAST_TS}" ]]; then + # 完全没找到 publishMessageList 记录——可能是新部署,不要贸然重启 + log_warn "[WATCHDOG] ⚠️ stdout.log 里没找到 publishMessageList 记录,可能是新部署。跳过本次。" + exit 0 +fi + +# 把日志时间转成 epoch +LAST_EPOCH=$(date -j -f "%Y.%m.%d %H:%M:%S" "${LAST_TS}" +%s 2>/dev/null || date -d "${LAST_TS}" +%s 2>/dev/null) +if [[ -z "${LAST_EPOCH}" ]]; then + log_warn "[WATCHDOG] ⚠️ 无法解析日志时间: ${LAST_TS},跳过本次。" + exit 0 +fi + +IDLE_MIN=$(( (NOW - LAST_EPOCH) / 60 )) + +log_info "[WATCHDOG] 最后 publishMessageList: ${LAST_TS},已空闲 ${IDLE_MIN} 分钟(阈值 ${MAX_IDLE_MIN} 分钟)" + +if [[ ${IDLE_MIN} -ge ${MAX_IDLE_MIN} ]]; then + log_warn "[WATCHDOG] 🔴 空闲 ${IDLE_MIN} 分钟 ≥ ${MAX_IDLE_MIN} 分钟!触发 Resin 重启!" + + # --- 4. 执行重启 --- + sshpass -p "${OA_PASS}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=15 ${OA_USER}@${OA_HOST} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}" + + sleep 20 + + # --- 5. 验证新 Resin PID --- + NEW_PID=$(sshpass -p "${OA_PASS}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${OA_USER}@${OA_HOST} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null) + HTTP_CODE=$(sshpass -p "${OA_PASS}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${OA_USER}@${OA_HOST} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp 2>/dev/null" 2>/dev/null) + + BODY="**🔴 Resin 被 watchdog 自动重启**\n\n" + BODY="${BODY}| 指标 | 值 |\n|---|---|\n" + BODY="${BODY}| publishMessageList 最后活动 | \`${LAST_TS}\` |\n" + BODY="${BODY}| 空闲时长 | **${IDLE_MIN} 分钟** |\n" + BODY="${BODY}| 新 Resin PID | \`${NEW_PID}\` |\n" + BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n\n" + BODY="${BODY}> ⚠️ 推送线程静默死掉,watchdog 自动恢复。\n" + BODY="${BODY}> 建议:泛微需要检查 Util_Message 线程为什么会静默挂掉。" + dingtalk_alert "P0" "Resin watchdog 自动重启(推送线程空闲 ${IDLE_MIN} 分钟)" "${BODY}" + + log_info "[WATCHDOG] ✅ Resin 重启完成,新 PID=${NEW_PID}, HTTP=${HTTP_CODE}" +else + log_info "[WATCHDOG] ✅ publishMessageList 活跃(最近 ${IDLE_MIN} 分钟前),Resin 健康" +fi diff --git a/bin/resin-watchdog.sh.bak.gt0.20261030 b/bin/resin-watchdog.sh.bak.gt0.20261030 new file mode 100755 index 0000000..673204e --- /dev/null +++ b/bin/resin-watchdog.sh.bak.gt0.20261030 @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · Resin Watchdog (v4 - PID 变化检测 + message_log 入库判定) +# v3 盲区修复:Resin 被 WatchdogManager kill 后秒拉起,message_log 继续刷看起来"健康" +# v4 新增:每次巡检记录 Resin PID,PID 变化 -> 钉钉 P1 告警 +# 健康依据(双重): +# 1. ecology_message_log 最近 15 分钟入库 >= 10 条(推送线程活着) +# 2. Resin PID 未变化(进程未被 kill 重启) +# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +OA_HOST="39.99.236.138" +OA_USER="root" +OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM" +OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1" + +MSG_WINDOW_MIN=15 +MSG_THRESHOLD=1 +START_HOUR=8 +END_HOUR=22 + +PID_FILE="${MONITOR_ROOT}/state/resin_pid.last" +LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log" +OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py" + +# --- 依赖检查 --- +if ! python3 -c 'import paramiko' 2>/dev/null; then + MSG="[WATCHDOG] paramiko 不可用!" + log_error "${MSG}" + dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}" + exit 1 +fi + +# --- SSH helper --- +cat > "${OA_SSH_PY}" << 'PYEOF' +import paramiko, sys +host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +try: + ssh.connect(host, username=user, password=password, timeout=10) + stdin, stdout, stderr = ssh.exec_command(cmd) + sys.stdout.write(stdout.read().decode("utf-8", errors="replace")) + ssh.close() +except Exception as e: + print(f"SSH_ERROR: {e}", file=sys.stderr) + sys.exit(1) +PYEOF +chmod +x "${OA_SSH_PY}" +OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}" + +# ============================================================================ +# Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果) +# ============================================================================ +mkdir -p "$(dirname "${PID_FILE}")" + +PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null) +CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}') +CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}') + +if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then + log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测" +else + LAST_PID="" + LAST_ETIMES="" + if [[ -f "${PID_FILE}" ]]; then + read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true + fi + + if [[ -z "${LAST_PID}" ]]; then + echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" + log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s" + elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then + NOW_HOUR=$(date +%H) + NOW_MIN=$(date +%M) + IS_PREVENTIVE=0 + if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then + IS_PREVENTIVE=1 + fi + + if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then + log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常" + else + RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S') + + BODY="Resin PID 异常变化(非预期重启)\n\n" + BODY="${BODY}上次 PID: ${LAST_PID}\n" + BODY="${BODY}新 PID: ${CUR_PID}\n" + BODY="${BODY}推算重启时间: ${RESTART_TIME}\n" + BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒\n\n" + BODY="${BODY}Resin 被 WatchdogManager kill 后拉起。message_log 可能仍在刷(假活跃), watchdog v3 会漏检。" + dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}" + + log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警" + fi + echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" + else + echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" + fi +fi + +# ============================================================================ +# Step 2: message_log 入库检测 (原 v3 逻辑) +# ============================================================================ +MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null) + +if [[ ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then + log_warn "[WATCHDOG] 查询 message_log 失败, 跳过" + exit 0 +fi + +log_info "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 入库 ${MSG_COUNT} 条" + +HOUR=$(date +%H) +if [[ ${MSG_COUNT} -ge ${MSG_THRESHOLD} ]]; then + log_info "[WATCHDOG] message_log 入库正常(${MSG_COUNT} >= ${MSG_THRESHOLD}), 推送健康" + exit 0 +fi + +if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then + log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), message_log 仅 ${MSG_COUNT} 条, 仅记录" + exit 0 +fi + +# --- 工作时间 + 入库过少 -> 触发重启 --- +log_warn "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 仅 ${MSG_COUNT} 条 < ${MSG_THRESHOLD}, 触发 Resin 重启!" +${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}" +sleep 20 + +NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1) +HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null) + +BODY="Resin watchdog 自动重启\n\n" +BODY="${BODY}message_log 最近 ${MSG_WINDOW_MIN} 分钟入库: ${MSG_COUNT} 条 (阈值 ${MSG_THRESHOLD})\n" +BODY="${BODY}新 Resin PID: ${NEW_PID}\n" +BODY="${BODY}HTTP 80: ${HTTP_CODE}\n" +dingtalk_alert "P0" "Resin watchdog 自动重启(message_log 入库 ${MSG_COUNT} 条 < ${MSG_THRESHOLD})" "${BODY}" + +if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then + echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}" +fi + +log_info "[WATCHDOG] Resin 重启完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}" diff --git a/bin/resin-watchdog.sh.bak.threshold.20261030 b/bin/resin-watchdog.sh.bak.threshold.20261030 new file mode 100755 index 0000000..6d53bc9 --- /dev/null +++ b/bin/resin-watchdog.sh.bak.threshold.20261030 @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · Resin Watchdog (v4 - PID 变化检测 + message_log 入库判定) +# v3 盲区修复:Resin 被 WatchdogManager kill 后秒拉起,message_log 继续刷看起来"健康" +# v4 新增:每次巡检记录 Resin PID,PID 变化 -> 钉钉 P1 告警 +# 健康依据(双重): +# 1. ecology_message_log 最近 15 分钟入库 >= 10 条(推送线程活着) +# 2. Resin PID 未变化(进程未被 kill 重启) +# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +OA_HOST="39.99.236.138" +OA_USER="root" +OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM" +OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1" + +MSG_WINDOW_MIN=15 +MSG_THRESHOLD=10 +START_HOUR=8 +END_HOUR=22 + +PID_FILE="${MONITOR_ROOT}/state/resin_pid.last" +LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log" +OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py" + +# --- 依赖检查 --- +if ! python3 -c 'import paramiko' 2>/dev/null; then + MSG="[WATCHDOG] paramiko 不可用!" + log_error "${MSG}" + dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}" + exit 1 +fi + +# --- SSH helper --- +cat > "${OA_SSH_PY}" << 'PYEOF' +import paramiko, sys +host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +try: + ssh.connect(host, username=user, password=password, timeout=10) + stdin, stdout, stderr = ssh.exec_command(cmd) + sys.stdout.write(stdout.read().decode("utf-8", errors="replace")) + ssh.close() +except Exception as e: + print(f"SSH_ERROR: {e}", file=sys.stderr) + sys.exit(1) +PYEOF +chmod +x "${OA_SSH_PY}" +OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}" + +# ============================================================================ +# Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果) +# ============================================================================ +mkdir -p "$(dirname "${PID_FILE}")" + +PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null) +CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}') +CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}') + +if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then + log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测" +else + LAST_PID="" + LAST_ETIMES="" + if [[ -f "${PID_FILE}" ]]; then + read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true + fi + + if [[ -z "${LAST_PID}" ]]; then + echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" + log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s" + elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then + NOW_HOUR=$(date +%H) + NOW_MIN=$(date +%M) + IS_PREVENTIVE=0 + if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then + IS_PREVENTIVE=1 + fi + + if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then + log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常" + else + RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S') + + BODY="Resin PID 异常变化(非预期重启)\n\n" + BODY="${BODY}上次 PID: ${LAST_PID}\n" + BODY="${BODY}新 PID: ${CUR_PID}\n" + BODY="${BODY}推算重启时间: ${RESTART_TIME}\n" + BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒\n\n" + BODY="${BODY}Resin 被 WatchdogManager kill 后拉起。message_log 可能仍在刷(假活跃), watchdog v3 会漏检。" + dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}" + + log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警" + fi + echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" + else + echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}" + fi +fi + +# ============================================================================ +# Step 2: message_log 入库检测 (原 v3 逻辑) +# ============================================================================ +MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null) + +if [[ ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then + log_warn "[WATCHDOG] 查询 message_log 失败, 跳过" + exit 0 +fi + +log_info "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 入库 ${MSG_COUNT} 条" + +HOUR=$(date +%H) +if [[ ${MSG_COUNT} -ge ${MSG_THRESHOLD} ]]; then + log_info "[WATCHDOG] message_log 入库正常(${MSG_COUNT} >= ${MSG_THRESHOLD}), 推送健康" + exit 0 +fi + +if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then + log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), message_log 仅 ${MSG_COUNT} 条, 仅记录" + exit 0 +fi + +# --- 工作时间 + 入库过少 -> 触发重启 --- +log_warn "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 仅 ${MSG_COUNT} 条 < ${MSG_THRESHOLD}, 触发 Resin 重启!" +${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}" +sleep 20 + +NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1) +HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null) + +BODY="Resin watchdog 自动重启\n\n" +BODY="${BODY}message_log 最近 ${MSG_WINDOW_MIN} 分钟入库: ${MSG_COUNT} 条 (阈值 ${MSG_THRESHOLD})\n" +BODY="${BODY}新 Resin PID: ${NEW_PID}\n" +BODY="${BODY}HTTP 80: ${HTTP_CODE}\n" +dingtalk_alert "P0" "Resin watchdog 自动重启(message_log 入库 ${MSG_COUNT} 条 < ${MSG_THRESHOLD})" "${BODY}" + +if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then + echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}" +fi + +log_info "[WATCHDOG] Resin 重启完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}" diff --git a/bin/resin-watchdog.sh.bak.v2.202610261128 b/bin/resin-watchdog.sh.bak.v2.202610261128 new file mode 100755 index 0000000..33b09c6 --- /dev/null +++ b/bin/resin-watchdog.sh.bak.v2.202610261128 @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · Resin Watchdog (v2 - python paramiko 版) +# 监控 Util_Message 线程活跃度(publishMessageList),超过 30 分钟无活动则自动重启 Resin +# 部署在 DSH 上,通过 python paramiko 到 OA 服务器(39.99.236.138)执行 +# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +OA_HOST="39.99.236.138" +OA_USER="root" +OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM" +OA_LOG="/usr/weaver/Resin/log/stdout.log" +OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1" +MAX_IDLE_MIN=30 + +LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log" +OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py" + +NOW=$(date +%s) + +# Step 1: 确保 python paramiko 可用 +if ! python3 -c 'import paramiko' 2>/dev/null; then + MSG="[WATCHDOG] 🔴 python3 + paramiko 不可用!请执行: pip3 install paramiko" + log_error "${MSG}" + dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}" + exit 1 +fi + +# Step 2: 写 _oa_ssh.py(paramiko + errors=replace 处理 GBK 乱码) +cat > "${OA_SSH_PY}" << 'PYEOF' +import paramiko, sys +host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +try: + ssh.connect(host, username=user, password=password, timeout=10) + stdin, stdout, stderr = ssh.exec_command(cmd) + sys.stdout.write(stdout.read().decode("utf-8", errors="replace")) + ssh.close() +except Exception as e: + print(f"SSH_ERROR: {e}", file=sys.stderr) + sys.exit(1) +PYEOF +chmod +x "${OA_SSH_PY}" + +OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}" + +# Step 3: 提取最后一个 publishMessageList 时间(tail -5000 ≈ 10 分钟覆盖) +LAST_RAW=$(${OA_SSH} "tail -5000 ${OA_LOG} 2>/dev/null | grep publishMessageList | tail -1" 2>/dev/null) +LAST_TS=$(echo "${LAST_RAW}" | grep -oE '[0-9]{4}\.[0-9]{2}\.[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}' | head -1) + +if [[ -z "${LAST_TS}" ]]; then + log_warn "[WATCHDOG] ⚠️ stdout.log 里没找到 publishMessageList 记录,跳过本次" + exit 0 +fi + +# Step 4: 解析时间戳 → epoch +LAST_EPOCH=$(date -d "$(echo ${LAST_TS} | sed "s|\.|-|g")" +%s 2>/dev/null) +if [[ -z "${LAST_EPOCH}" ]]; then + log_warn "[WATCHDOG] ⚠️ 无法解析日志时间: ${LAST_TS},跳过本次" + exit 0 +fi + +IDLE_MIN=$(( (NOW - LAST_EPOCH) / 60 )) +log_info "[WATCHDOG] 最后 publishMessageList: ${LAST_TS},已空闲 ${IDLE_MIN} 分钟(阈值 ${MAX_IDLE_MIN})" + +# Step 5: 判断是否需要重启 +if [[ ${IDLE_MIN} -lt ${MAX_IDLE_MIN} ]]; then + log_info "[WATCHDOG] ✅ publishMessageList 活跃(最近 ${IDLE_MIN} 分钟前),Resin 健康" + exit 0 +fi + +# --- 触发重启 --- +log_warn "[WATCHDOG] 🔴 空闲 ${IDLE_MIN} 分钟 ≥ ${MAX_IDLE_MIN} 分钟!触发 Resin 重启!" + +${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}" + +sleep 20 + +NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1) +HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null) + +BODY="**🔴 Resin 被 watchdog 自动重启**\n\n" +BODY="${BODY}| 指标 | 值 |\n|---|---|\n" +BODY="${BODY}| publishMessageList 最后活动 | \`${LAST_TS}\` |\n" +BODY="${BODY}| 空闲时长 | **${IDLE_MIN} 分钟** |\n" +BODY="${BODY}| 新 Resin PID | \`${NEW_PID}\` |\n" +BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n\n" +BODY="${BODY}> ⚠️ 推送线程静默死掉,watchdog 自动恢复。" +BODY="${BODY}> 建议:泛微需要检查 Util_Message 线程为什么会静默挂掉。" +dingtalk_alert "P0" "Resin watchdog 自动重启(推送线程空闲 ${IDLE_MIN} 分钟)" "${BODY}" + +log_info "[WATCHDOG] ✅ Resin 重启完成,新 PID=${NEW_PID}, HTTP=${HTTP_CODE}" diff --git a/bin/resin-watchdog.sh.bak.v3.202610281415 b/bin/resin-watchdog.sh.bak.v3.202610281415 new file mode 100755 index 0000000..021ebb7 --- /dev/null +++ b/bin/resin-watchdog.sh.bak.v3.202610281415 @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · Resin Watchdog (v3 - message_log 入库增量判定) +# 健康依据:ecology_message_log 最近 15 分钟入库数 +# - 工作时间(08:00~22:00) 入库 < 10 条 → 判定推送线程死 → 重启 Resin +# - 凌晨时段 入库少也不重启(避免误重启) +# 部署在 DSH,python paramiko 到 OA 执行重启 +# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +OA_HOST="39.99.236.138" +OA_USER="root" +OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM" +OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1" + +MSG_WINDOW_MIN=15 # 统计最近 15 分钟 +MSG_THRESHOLD=10 # 入库 < 10 条视为异常 +START_HOUR=8 # 工作时间起始 +END_HOUR=22 # 工作时间结束 + +LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log" +OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py" + +# Step 1: 确保 python paramiko 可用 +if ! python3 -c 'import paramiko' 2>/dev/null; then + MSG="[WATCHDOG] 🔴 python3 + paramiko 不可用!请执行: pip3 install paramiko" + log_error "${MSG}" + dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}" + exit 1 +fi + +# Step 2: 写 _oa_ssh.py(仅重启时用) +cat > "${OA_SSH_PY}" << 'PYEOF' +import paramiko, sys +host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] +ssh = paramiko.SSHClient() +ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +try: + ssh.connect(host, username=user, password=password, timeout=10) + stdin, stdout, stderr = ssh.exec_command(cmd) + sys.stdout.write(stdout.read().decode("utf-8", errors="replace")) + ssh.close() +except Exception as e: + print(f"SSH_ERROR: {e}", file=sys.stderr) + sys.exit(1) +PYEOF +chmod +x "${OA_SSH_PY}" + +OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}" + +# Step 3: 查 message_log 最近 MSG_WINDOW_MIN 分钟入库数(DSH 直连 MySQL,走 pushlog_time 索引) +MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \ + -sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null) + +if [[ ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then + log_warn "[WATCHDOG] ⚠️ 查询 message_log 失败(返回: ${MSG_COUNT}),跳过本次" + exit 0 +fi + +log_info "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 入库 ${MSG_COUNT} 条" + +# Step 4: 判断 +HOUR=$(date +%H) +if [[ ${MSG_COUNT} -ge ${MSG_THRESHOLD} ]]; then + log_info "[WATCHDOG] ✅ message_log 入库正常(${MSG_COUNT} 条 ≥ ${MSG_THRESHOLD} 阈值),推送健康" + exit 0 +fi + +# 入库 < 阈值 +if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then + log_warn "[WATCHDOG] ⚠️ 凌晨时段(${HOUR}时),message_log 仅 ${MSG_COUNT} 条 < ${MSG_THRESHOLD},仅记录不重启" + exit 0 +fi + +# --- 工作时间 + 入库过少 → 触发重启 --- +log_warn "[WATCHDOG] 🔴 最近 ${MSG_WINDOW_MIN} 分钟 message_log 仅 ${MSG_COUNT} 条 < ${MSG_THRESHOLD},触发 Resin 重启!" + +${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}" + +sleep 20 + +NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1) +HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null) + +BODY="**🔴 Resin 被 watchdog 自动重启**\n\n" +BODY="${BODY}| 指标 | 值 |\n|---|---|\n" +BODY="${BODY}| message_log 最近 ${MSG_WINDOW_MIN} 分钟入库 | **${MSG_COUNT} 条**(阈值 ${MSG_THRESHOLD})|\n" +BODY="${BODY}| 新 Resin PID | \`${NEW_PID}\` |\n" +BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n\n" +BODY="${BODY}> ⚠️ 推送线程静默死掉(message_log 不再入库),watchdog 自动恢复。" +BODY="${BODY}> 注:publishMessageList 日志可能仍在刷(假活跃),以 message_log 入库为准。" +dingtalk_alert "P0" "Resin watchdog 自动重启(message_log 入库 ${MSG_COUNT} 条 < ${MSG_THRESHOLD})" "${BODY}" + +log_info "[WATCHDOG] ✅ Resin 重启完成,新 PID=${NEW_PID}, HTTP=${HTTP_CODE}" diff --git a/bin/run-long-check.sh b/bin/run-long-check.sh new file mode 100755 index 0000000..99f4af6 --- /dev/null +++ b/bin/run-long-check.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 长查询检测 dry-run KILL(每 10 分钟) +# 默认 DRY_RUN=1(只报告,不真 KILL)。--execute 才真的 KILL +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +DRY_RUN=1 +for arg in "$@"; do + [[ "${arg}" == "--execute" ]] && DRY_RUN=0 + [[ "${arg}" == "--dry-run" ]] && DRY_RUN=1 +done + +NOW=$(date +%s) + +MYSQL_RUN() { + MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology "$@" 2>/dev/null +} + +log_info "[LONG-CHECK] 开始(DRY_RUN=${DRY_RUN},阈值 ${ALERT_LONG_QUERY_SEC}s)" + +RESULT=$(MYSQL_RUN "SHOW FULL PROCESSLIST;" | awk -F'\t' -v t="${ALERT_LONG_QUERY_SEC}" ' + NR==1 { next } + NF < 10 { next } + { + pid=$1; user=$2; state=$6; time=$7; info=$9 + if (time+0 >= t+0 && time+0 > 0) { + gsub(/\t/, " ", info) + printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300) + } + } +') + +if [[ -z "${RESULT}" ]]; then + log_info "[LONG-CHECK] ✅ 无长查询" + exit 0 +fi + +COUNT=$(echo "${RESULT}" | wc -l | tr -d ' ') +log_warn "[LONG-CHECK] ⚠️ ${COUNT} 条长查询" + +DETAIL="" +while IFS=$'\t' read -r pid time user state info; do + [[ -z "${pid}" ]] && continue + DETAIL="${DETAIL}\n- PID \`${pid}\` **${time}s** \`${user}\` ${state}" + DETAIL="${DETAIL}\n \`\`\`\n ${info}\n \`\`\`" +done <<< "${RESULT}" + +MODE_LABEL=$([[ ${DRY_RUN} -eq 1 ]] && echo "🔒 DRY-RUN" || echo "🚨 执行 KILL") +BODY="**${COUNT} 条长查询**(${ALERT_LONG_QUERY_SEC}s 阈值)· ${MODE_LABEL}\n\n${DETAIL}" +dingtalk_alert "P0" "长查询阻塞需处理" "${BODY}" + +sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询', '${COUNT} 条, dry_run=${DRY_RUN}');" + +if [[ ${DRY_RUN} -eq 0 ]]; then + log_warn "[LONG-CHECK] 执行 KILL!" + while IFS=$'\t' read -r pid time user state info; do + [[ -z "${pid}" ]] && continue + MYSQL_RUN "KILL ${pid};" + log_warn "[LONG-CHECK] KILL ${pid} (${time}s, ${user})" + done <<< "${RESULT}" +else + log_info "[LONG-CHECK] DRY-RUN,不执行 KILL" +fi diff --git a/docs/DUMMY-fix-plan.md b/docs/DUMMY-fix-plan.md new file mode 100644 index 0000000..173c169 --- /dev/null +++ b/docs/DUMMY-fix-plan.md @@ -0,0 +1,49 @@ +DUMMY 补数据 · 执行与回滚方案 +============================== + +目标:根治 updateBizState 死循环 → 推送线程饿死 → Resin 反复重启 +表:ecology_message_info_read → 只 INSERT,不删不改 +脚本:/opt/oa-monitor/bin/fix-info-read-dummy.py +执行时间:2026-11-04 下午 + +问题一句话 +---------- +泛微归档流程的 info_read 记录被清空了(归档 389 万条 vs info_read 归档=0 条)。 +updateBizState 线程池扫归档流程 → 找不到 info_read → 10 线程全部死循环占满 → +推送线程饿死 → message_log 停写 → Resin 被 watchdog 重启。5 天 7 次重启。 + +根因证据 +-------- +info_read 归档记录: 0 条 +归档流程总量: 389 万条 +10-26 jstack 抓到 updateBizState 10 线程全 WAITING(实锤) +10-26 补 60/138 两人 DUMMY → 立即恢复正常 2 天(治疗验证) +11-04 上午 updateBizState 扫到未补的用户 → 3 小时死 3 次(复现) +watchdog 日志 20 次 count=0 → 重启 + +执行步骤 +-------- +Phase 0: 前置检查(Resin HTTP 200 / MySQL Threads_running<15 / 现有 DUMMY 存在) +Phase 1: 备份(CREATE TABLE ... AS SELECT) +Phase 2: 分批 INSERT + Phase 2a: CREATER TOP 100 业务用户(userid != 1) + Phase 2b: LASTOPERATOR TOP 100 业务用户(userid != 1) + 每批 5000 条 + sleep 1s + 每批前检查 Resin/MySQL 健康 +Phase 3: 后置验证 + 钉钉通知 + +锁库风险控制 +---------- +每批 INSERT 5000 条,单次持有表锁 < 1s +批次间隔 sleep(1) +总批数 ≈ 48 批,总时长 60-90 min + +回滚方案(3 种粒度) +------------------- +粒度1 全部回滚: DELETE FROM ecology_message_info_read WHERE ID LIKE "DUMMY_B_%"; +粒度2 按用户: DELETE FROM ecology_message_info_read WHERE ID LIKE "DUMMY_B_%_1723_%"; +粒度3 按时间: DELETE FROM ecology_message_info_read WHERE ID LIKE "DUMMY_B_%" AND CREATEDATE >= "2026-11-04"; + +紧急刹车 +-------- +kill $(ps -eo pid,cmd | grep fix-info-read | grep -v grep | awk "{print \\$1}") +断点续插: 重跑脚本即可 diff --git a/lib/common.sh b/lib/common.sh new file mode 100755 index 0000000..a26fbdc --- /dev/null +++ b/lib/common.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 全局配置与公共函数 +# 所有采集/告警脚本 source 此文件,只修改这里的变量即可调整行为 +# ============================================================================ +set -u # 未定义变量报错 +# 注意:不写 set -e,采集脚本单条命令失败不应整体中断 + +# ---- 路径 ---- +export MONITOR_ROOT="/opt/oa-monitor" +export MONITOR_DATA="${MONITOR_ROOT}/data" +export MONITOR_LOGS="${MONITOR_ROOT}/logs" +export MONITOR_DB="${MONITOR_DATA}/oa-monitor.db" + +mkdir -p "${MONITOR_DATA}" "${MONITOR_LOGS}" + +# ---- 工具 ---- +export MYSQL_CMD="mysql -h39.100.72.102 -P3306 -uecology" +export DB_NAME="ecology" +export DB_PASS='Z!BYT+jSDDyYHMdX+qFSLzJCYXW' + +export SSH_DSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i ~/.ssh/id_ed25519 root@47.108.21.167" + +# ---- OA 端点(DSH 上访问 OA 应用服务器内网 IP,更稳)---- +OA_HTTP="http://39.99.236.138" +OA_ENDPOINTS=( + "${OA_HTTP}:80/wui/main/Login.jsp" + "${OA_HTTP}:8444/api/portal/portalinfo" + "${OA_HTTP}:8444/api/portal/sso/getLoginInfo.jsp" + "${OA_HTTP}/api/workflow/getRequestList" +) + +# ---- 钉钉 Webhook(群机器人,2026-10-22)---- +export DINGTALK_WEBHOOK="https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249" +export DINGTALK_SECRET="" + +# ---- 告警阈值 ---- +export ALERT_THREADS_RUNNING_HIGH=50 # 当前活跃线程 > 50 告警 +export ALERT_SLOW_QUERY_SEC=30 # 慢查询 > 30s 告警 +export ALERT_SLOW_QUERY_COUNT=5 # 同进程号出现 5 次以上告警 +export ALERT_HTTP_FAIL_COUNT=3 # HTTP 端点连续失败 3 次告警 +export ALERT_MSG_LOG_HOUR_MIN=100 # 1 小时消息增量 < 100 条告警(可能钉钉推送断了) +export ALERT_DOCKER_CONTAINER_DELTA=3 # 容器数突变 > ±3 告警 +export ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN=1024 # Docker data-root 1 分钟写 > 1GB 告警 +export ALERT_DISK_USAGE_PCT=85 # 磁盘使用率 > 85% 告警 +export ALERT_INODE_USAGE_PCT=80 # inode 使用率 > 80% 告警 +export ALERT_DOCKER_DAEMON_FAIL_COUNT=2 # Docker daemon 连续失败 2 次告警 +export ALERT_LONG_QUERY_SEC=300 # 长查询 > 300s 触发 KILL(dry-run 默认) + +# ---- 告警抑制 ---- +export ALERT_COOLDOWN_SEC=600 # 同一告警类型 10 分钟内不重复发 + +# ---- SQLite 工具函数 ---- +sqlite_exec() { + sqlite3 "${MONITOR_DB}" "$@" 2>/dev/null +} + +# ---- 日志函数 ---- +log() { + local level="$1"; shift + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MONITOR_LOGS}/oa-monitor.log" +} + +log_info() { log "INFO" "$@"; } +log_warn() { log "WARN" "$@"; } +log_error() { log "ERROR" "$@"; } + +# ---- 钉钉告警(markdown 格式)---- +# 用法:dingtalk_alert <markdown_body> +# level: P0 / P1 / P2 +dingtalk_alert() { + local level="$1" title="$2" body="$3" + local timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + + # 告警冷却检查 + local key="${level}:${title}" + local last_ts + last_ts=$(sqlite_exec "SELECT COALESCE(last_alert_ts, 0) FROM alert_cooldown WHERE alert_key='${key}';") + local now_epoch + now_epoch=$(date +%s) + if [[ "${last_ts}" =~ ^[0-9]+$ ]] && (( now_epoch - last_ts < ALERT_COOLDOWN_SEC )); then + log_info "告警抑制:${key}(距上次 ${now_epoch}s < ${ALERT_COOLDOWN_SEC}s)" + return 0 + fi + + # 构造 JSON + local json + json=$(cat <<JSON +{"msgtype":"markdown","markdown":{"title":"[OA监控] ${level} ${title}","text":"### [OA监控] **${level}** ${title}\n\n**时间**:${timestamp}\n\n**主机**:DSH (47.108.21.167)\n\n---\n\n${body}\n\n---\n\n> OA 运维监控平台 · $(basename "$0")"}} +JSON +) + + # 发送 + local http_code + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Content-Type: application/json" \ + -d "${json}" "${DINGTALK_WEBHOOK}" 2>/dev/null) + + if [[ "${http_code}" == "200" ]]; then + log_info "钉钉告警发送成功:[${level}] ${title}" + sqlite_exec "INSERT OR REPLACE INTO alert_cooldown VALUES ('${key}', ${now_epoch});" + else + log_error "钉钉告警发送失败(HTTP ${http_code}):[${level}] ${title}" + fi +} diff --git a/state/resin_pid.last b/state/resin_pid.last new file mode 100644 index 0000000..757643f --- /dev/null +++ b/state/resin_pid.last @@ -0,0 +1 @@ +13286 9890 1793773202