fix: 修复 check-alert.sh 假阳性 + collect-business.sh 窗口对齐
- check-alert.sh 新增 Step0 数据新鲜度前置检查:business_metrics 超 30min 过期 → 先告警采集脚本挂了,跳过业务层检测,防止读旧数据乱报推送断档 - collect-business.sh msg_log_hourly 窗口从 1h 改为 15min(和 crontab 频率对齐) - CURR_HOUR 用 10# 算术强制十进制,防 08/09 被解析成 octal
This commit is contained in:
1 parent
2e7504baf8
commit
90be249a72
10 files changed
+829
No files matched your search
Executable
+166
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 告警检测(每 1 分钟)
|
||||
# v2 修复(2026-10-28):
|
||||
# - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警
|
||||
# - Step 3 业务层检测被 Step 0 结果门禁
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
log_info "[ALERT] ========== 告警检测开始 =========="
|
||||
|
||||
# ============================================================================
|
||||
# Step 0: 数据新鲜度前置检查(防假阳性的第一层)
|
||||
# ============================================================================
|
||||
# business_metrics 由 collect-business.sh 每 15min 写一次
|
||||
# db_metrics 由 collect-db.sh 每 30s 写一次
|
||||
# 如果最新 business_metrics ts 距现在 > 30min → 采集脚本挂了 → 告警+跳过业务层
|
||||
BIZ_TS=$(sqlite_exec "SELECT ts FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
DB_TS=$(sqlite_exec "SELECT ts FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
BIZ_FRESH=true
|
||||
BIZ_AGE=0
|
||||
|
||||
if [[ ! "${BIZ_TS}" =~ ^[0-9]+$ ]]; then
|
||||
log_error "[ALERT] business_metrics 表无数据或采集从未运行"
|
||||
BIZ_FRESH=false
|
||||
BIZ_AGE=999999
|
||||
elif [[ ! "${DB_TS}" =~ ^[0-9]+$ ]]; then
|
||||
log_warn "[ALERT] db_metrics 表无数据(可能 MySQL 断了)"
|
||||
else
|
||||
BIZ_AGE=$((NOW - BIZ_TS))
|
||||
DB_AGE=$((NOW - DB_TS))
|
||||
|
||||
if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟
|
||||
BIZ_FRESH=false
|
||||
HUMAN_AGE=$((BIZ_AGE / 60))
|
||||
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec \"SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')\") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec \"SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')\") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
|
||||
dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}"
|
||||
log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)"
|
||||
else
|
||||
log_info "[ALERT] 数据新鲜:biz_age=${BIZ_AGE}s db_age=${DB_AGE}s"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 1. DB 层检测(db_metrics 每分钟更新,数据可信)
|
||||
# ============================================================================
|
||||
TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
if [[ -n "${TR}" ]] && [[ ${TR} =~ ^[0-9]+$ ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then
|
||||
BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;' | head -1)"
|
||||
dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');"
|
||||
fi
|
||||
|
||||
# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的)
|
||||
LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;")
|
||||
if [[ -n "${LONG_QUERIES}" ]]; then
|
||||
COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ')
|
||||
DETAIL=""
|
||||
while IFS='|' read -r pid time user state info; do
|
||||
DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`"
|
||||
done <<< "${LONG_QUERIES}"
|
||||
BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理"
|
||||
dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 2. HTTP 端点检测(http_metrics 每分钟更新,数据可信)
|
||||
# ============================================================================
|
||||
FAIL_COUNT=0
|
||||
FAIL_DETAIL=""
|
||||
for endpoint in "${OA_ENDPOINTS[@]}"; do
|
||||
FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;" | head -1)
|
||||
FAIL_HERE=${FAIL_HERE:-0}
|
||||
if [[ "${FAIL_HERE}" =~ ^[0-9]+$ ]] && [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then
|
||||
FAIL_COUNT=$((FAIL_COUNT + 1))
|
||||
LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${FAIL_COUNT} -gt 0 ]]; then
|
||||
BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态"
|
||||
dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 3. 业务层检测(被 BIZ_FRESH 门禁)
|
||||
# ============================================================================
|
||||
if [[ "${BIZ_FRESH}" != "true" ]]; then
|
||||
log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)"
|
||||
else
|
||||
# 数据新鲜 → 正常业务层判断
|
||||
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
HOUR_MSG=${HOUR_MSG:-0}
|
||||
ONGOING=${ONGOING:-0}
|
||||
|
||||
CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal
|
||||
WORK_HOURS_ENABLED=false
|
||||
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
|
||||
WORK_HOURS_ENABLED=true
|
||||
fi
|
||||
|
||||
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
|
||||
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then
|
||||
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
|
||||
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
||||
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n"
|
||||
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n"
|
||||
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n"
|
||||
BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n"
|
||||
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n"
|
||||
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n"
|
||||
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**"
|
||||
dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
|
||||
fi
|
||||
else
|
||||
log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信)
|
||||
# ============================================================================
|
||||
DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;" | head -1)
|
||||
if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then
|
||||
BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> 请立即检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`"
|
||||
dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');"
|
||||
fi
|
||||
|
||||
CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;" | head -1)
|
||||
if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]] && [[ "${CURR_COUNT}" =~ ^[0-9]+$ ]]; then
|
||||
AVG_INT=${AVG_COUNT%%.*}
|
||||
DELTA=$(( CURR_COUNT - AVG_INT ))
|
||||
ABS_DELTA=${DELTA#-}
|
||||
if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then
|
||||
BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a"
|
||||
dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');"
|
||||
fi
|
||||
fi
|
||||
|
||||
DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
if [[ -n "${DISK_PCT}" ]] && [[ "${DISK_PCT}" =~ ^[0-9.]+$ ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then
|
||||
BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)"
|
||||
dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');"
|
||||
fi
|
||||
|
||||
INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
if [[ -n "${INODE_PCT}" ]] && [[ "${INODE_PCT}" =~ ^[0-9.]+$ ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then
|
||||
BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)"
|
||||
dingtalk_alert "P1" "DSH inode 告警" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');"
|
||||
fi
|
||||
|
||||
log_info "[ALERT] ========== 告警检测完成 =========="
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 数据清理(每天 03:00)
|
||||
# 保留最近 7 天指标,alerts 和 alert_cooldown 保留 30 天
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
RETENTION_7D=$(( NOW - 7 * 86400 ))
|
||||
RETENTION_30D=$(( NOW - 30 * 86400 ))
|
||||
|
||||
log_info "[CLEANUP] 开始清理,阈值 ts < ${RETENTION_7D}(7天),alerts < ${RETENTION_30D}(30天)"
|
||||
|
||||
# 各层指标表
|
||||
for table in db_metrics http_metrics business_metrics infra_metrics db_processlist; do
|
||||
BEFORE=$(sqlite_exec "SELECT COUNT(*) FROM ${table};")
|
||||
sqlite_exec "DELETE FROM ${table} WHERE ts < ${RETENTION_7D};"
|
||||
AFTER=$(sqlite_exec "SELECT COUNT(*) FROM ${table};")
|
||||
log_info "[CLEANUP] ${table}: ${BEFORE} → ${AFTER}(删了 $((BEFORE - AFTER)) 条)"
|
||||
done
|
||||
|
||||
# alerts 保留 30 天
|
||||
BEFORE=$(sqlite_exec "SELECT COUNT(*) FROM alerts;")
|
||||
sqlite_exec "DELETE FROM alerts WHERE ts < ${RETENTION_30D};"
|
||||
AFTER=$(sqlite_exec "SELECT COUNT(*) FROM alerts;")
|
||||
log_info "[CLEANUP] alerts: ${BEFORE} → ${AFTER}(删了 $((BEFORE - AFTER)) 条)"
|
||||
|
||||
# VACUUM 压缩(可选,不频繁)
|
||||
if (( NOW % (86400 * 7) < 3600 )); then
|
||||
log_info "[CLEANUP] 每周 VACUUM SQLite..."
|
||||
sqlite_exec "VACUUM;"
|
||||
fi
|
||||
|
||||
log_info "[CLEANUP] 完成"
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 业务指标采集(每 15 分钟,crontab: */15 * * * *)
|
||||
# v2 修复(2026-10-28):
|
||||
# - msg_log_hourly 实际用 15min 窗口(和 crontab 频率对齐),跨天逻辑简化
|
||||
# - 时间字段正确使用 CREATEDATE varchar(10) + CREATETIME varchar(8)
|
||||
# ecology_message_log 没有 DATETIME 类型的 pushlog_time 字段!
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
NOW_DATE=$(date '+%Y-%m-%d')
|
||||
|
||||
# --- 1. ecology_message_log 15 分钟增量 ---
|
||||
# v2 简化:15min 窗口几乎不跨天,直接用 CURDATE()
|
||||
MSG_WINDOW=15
|
||||
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e "
|
||||
SELECT COUNT(*) FROM ecology_message_log
|
||||
WHERE CREATEDATE = CURDATE()
|
||||
AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW} MINUTE, '%H:%i:%s');
|
||||
" 2>/dev/null | tail -1)
|
||||
MSG_COUNT=${MSG_COUNT:-0}
|
||||
|
||||
# --- 2. 正在流转的审批数 ---
|
||||
ONGOING=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e "
|
||||
SELECT COUNT(*) FROM workflow_requestbase
|
||||
WHERE CURRENTNODEID > 0 AND (STATUS IS NULL OR STATUS NOT LIKE '%归档%');
|
||||
" 2>/dev/null | tail -1)
|
||||
ONGOING=${ONGOING:-0}
|
||||
|
||||
# --- 3. 今日新建审批数 ---
|
||||
NEW_TODAY=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e "
|
||||
SELECT COUNT(*) FROM workflow_requestbase WHERE CREATEDATE = '${NOW_DATE}';
|
||||
" 2>/dev/null | tail -1)
|
||||
NEW_TODAY=${NEW_TODAY:-0}
|
||||
|
||||
# --- 4. 超时任务数 ---
|
||||
OVERDUE=$(MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -N -e "
|
||||
SELECT COUNT(*) FROM workflow_requestlog
|
||||
WHERE timeout = 1 AND operatedate >= DATE_SUB(CURDATE(), INTERVAL 1 DAY);
|
||||
" 2>/dev/null | tail -1)
|
||||
OVERDUE=${OVERDUE:-0}
|
||||
|
||||
sqlite_exec "INSERT INTO business_metrics VALUES (
|
||||
${NOW}, ${MSG_COUNT}, 0.0, ${ONGOING}, ${OVERDUE}
|
||||
);"
|
||||
|
||||
log_info "[BIZ] ✅ msg_15min=${MSG_COUNT} ongoing=${ONGOING} new_today=${NEW_TODAY} overdue=${OVERDUE}"
|
||||
Executable
+98
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · DB 层采集(每 30 秒)
|
||||
# 在 DSH 宿主机上直接 mysql 连 OA DB(DSH 本身就是跳板,不需要再 ssh)
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
|
||||
# ---- MySQL 直连(DSH 上直接连 OA DB 公网映射)----
|
||||
DB_MYSQL() {
|
||||
MYSQL_PWD="${DB_PASS}" timeout 12 mysql -h39.100.72.102 -P3306 -uecology ecology "$@" 2>/dev/null
|
||||
}
|
||||
|
||||
db_exec() {
|
||||
DB_MYSQL -e "$1"
|
||||
}
|
||||
|
||||
# ---- 1. SHOW STATUS ----
|
||||
log_info "[DB] SHOW STATUS"
|
||||
STATUS=$(DB_MYSQL -e "SHOW GLOBAL STATUS" | awk -F'\t' '
|
||||
$1=="Threads_running"||$1=="Threads_connected"||$1=="Threads_idle"||
|
||||
$1=="Uptime"||$1=="Innodb_rows_read"||$1=="Innodb_rows_inserted"||
|
||||
$1=="Innodb_rows_updated"||$1=="Innodb_rows_deleted"||$1=="Slow_queries" { print }
|
||||
')
|
||||
|
||||
THREADS_RUNNING=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_running"{print $2}')
|
||||
THREADS_CONNECTED=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_connected"{print $2}')
|
||||
THREADS_IDLE=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_idle"{print $2}')
|
||||
UPTIME=$(echo "${STATUS}" | awk -F'\t' '$1=="Uptime"{print $2}')
|
||||
INNODB_READ=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_read"{print $2}')
|
||||
INNODB_INS=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_inserted"{print $2}')
|
||||
INNODB_UPD=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_updated"{print $2}')
|
||||
INNODB_DEL=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_deleted"{print $2}')
|
||||
SLOW_QUERIES=$(echo "${STATUS}" | awk -F'\t' '$1=="Slow_queries"{print $2}')
|
||||
|
||||
INNODB_READ=${INNODB_READ:-0}
|
||||
INNODB_INS=${INNODB_INS:-0}
|
||||
INNODB_UPD=${INNODB_UPD:-0}
|
||||
INNODB_DEL=${INNODB_DEL:-0}
|
||||
INNODB_WRITE=$(( INNODB_INS + INNODB_UPD + INNODB_DEL ))
|
||||
THREADS_RUNNING=${THREADS_RUNNING:-0}
|
||||
THREADS_CONNECTED=${THREADS_CONNECTED:-0}
|
||||
THREADS_IDLE=${THREADS_IDLE:-0}
|
||||
UPTIME=${UPTIME:-0}
|
||||
SLOW_QUERIES=${SLOW_QUERIES:-0}
|
||||
|
||||
# ---- 2. 上一次快照 → 差值 ----
|
||||
PREV=$(sqlite_exec "SELECT * FROM db_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
QPS_READ=0; QPS_WRITE=0; SLOW_DELTA=0
|
||||
|
||||
if [[ -n "${PREV}" ]]; then
|
||||
PREV_UPTIME=$(echo "${PREV}" | cut -d'|' -f5)
|
||||
PREV_READ=$(echo "${PREV}" | cut -d'|' -f6)
|
||||
PREV_WRITE=$(echo "${PREV}" | cut -d'|' -f7)
|
||||
PREV_SLOW=$(echo "${PREV}" | cut -d'|' -f9)
|
||||
PREV_UPTIME=${PREV_UPTIME:-0}; PREV_READ=${PREV_READ:-0}
|
||||
PREV_WRITE=${PREV_WRITE:-0}; PREV_SLOW=${PREV_SLOW:-0}
|
||||
DIFF_SEC=$(( NOW - PREV_UPTIME ))
|
||||
if [[ ${DIFF_SEC} -gt 0 ]]; then
|
||||
QPS_READ=$(( (INNODB_READ - PREV_READ) / DIFF_SEC ))
|
||||
QPS_WRITE=$(( (INNODB_WRITE - PREV_WRITE) / DIFF_SEC ))
|
||||
fi
|
||||
SLOW_DELTA=$(( SLOW_QUERIES - PREV_SLOW ))
|
||||
fi
|
||||
|
||||
# ---- 3. 写 SQLite ----
|
||||
sqlite_exec "INSERT INTO db_metrics VALUES (
|
||||
${NOW}, ${THREADS_RUNNING}, ${THREADS_CONNECTED}, ${THREADS_IDLE},
|
||||
${UPTIME}, ${QPS_READ}, ${QPS_WRITE}, ${SLOW_DELTA}
|
||||
);"
|
||||
|
||||
log_info "[DB] ✅ 写入:Threads_running=${THREADS_RUNNING} QPS_read=${QPS_READ} QPS_write=${QPS_WRITE}"
|
||||
|
||||
# ---- 4. 长查询 processlist ----
|
||||
log_info "[DB] processlist"
|
||||
PLIST=$(DB_MYSQL -e 'SHOW PROCESSLIST;')
|
||||
|
||||
echo "${PLIST}" | awk -F'\t' '
|
||||
NR==1 { next }
|
||||
NF < 8 { next }
|
||||
{
|
||||
pid=$1; user=$2; state=$6; time=$7; info=$8
|
||||
if (time+0 >= 5 && pid+0 > 0) {
|
||||
gsub(/\t/, " ", info)
|
||||
printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300)
|
||||
}
|
||||
}
|
||||
' | while IFS=$'\t' read -r pid time user state info; do
|
||||
[[ -z "${pid}" ]] && continue
|
||||
info_safe=$(echo "${info}" | sed "s/'/\"/g")
|
||||
sqlite_exec "INSERT INTO db_processlist (ts, pid, user, state, time_sec, info)
|
||||
VALUES (${NOW}, ${pid}, '${user}', '${state}', ${time}, '${info_safe}');"
|
||||
done
|
||||
|
||||
log_info "[DB] ✅ 完成"
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · HTTP 端点健康检查(每 30 秒)
|
||||
# 在 DSH 上直接 curl
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
|
||||
for endpoint in "${OA_ENDPOINTS[@]}"; do
|
||||
result=$(curl -sk -o /dev/null -w '%{http_code}|%{time_total}' --max-time 10 "${endpoint}" 2>/dev/null)
|
||||
status=$(echo "${result}" | cut -d'|' -f1)
|
||||
time_ms=$(echo "${result}" | cut -d'|' -f2 | awk '{printf "%d", $1*1000}')
|
||||
status=${status:-0}
|
||||
time_ms=${time_ms:-0}
|
||||
|
||||
case "${status}" in
|
||||
000) healthy=0 ;; # 连不上 = 挂了
|
||||
5*) healthy=0 ;; # 5xx = 服务异常
|
||||
*) healthy=1 ;; # 2xx/3xx/4xx 都算活着(4xx 是业务路径不对,但服务本身通了)
|
||||
esac
|
||||
|
||||
sqlite_exec "INSERT INTO http_metrics (ts, endpoint, status_code, resp_time_ms, healthy)
|
||||
VALUES (${NOW}, '${endpoint}', ${status}, ${time_ms}, ${healthy});"
|
||||
|
||||
log_info "[HTTP] ${endpoint} status=${status} time=${time_ms}ms healthy=${healthy}"
|
||||
done
|
||||
|
||||
log_info "[HTTP] 全部完成"
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 基础设施层采集(每 30 秒)
|
||||
# 在 DSH 上直接执行 docker / df / du
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
DOCKER_ROOT="/var/lib/docker"
|
||||
|
||||
# ---- Docker daemon + 容器数 ----
|
||||
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ')
|
||||
if [[ "${CONTAINER_COUNT}" =~ ^[0-9]+$ ]] && [[ ${CONTAINER_COUNT} -ge 0 ]]; then
|
||||
DOCKER_OK=1
|
||||
else
|
||||
DOCKER_OK=0
|
||||
CONTAINER_COUNT=0
|
||||
fi
|
||||
|
||||
# ---- Docker root 写入量 ----
|
||||
# docker_root_write_mb 字段存储 "上一次 du overlay2 的总大小 MB"
|
||||
# 告警脚本里直接看 "本次 du - 上次存的值" 的差值
|
||||
PREV_TOTAL_MB=$(sqlite_exec "SELECT COALESCE(docker_root_write_mb, 0) FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
CURRENT_TOTAL_MB=$(du -sm "${DOCKER_ROOT}/overlay2" 2>/dev/null | cut -f1 | tr -d ' ')
|
||||
PREV_TOTAL_MB=${PREV_TOTAL_MB:-0}
|
||||
CURRENT_TOTAL_MB=${CURRENT_TOTAL_MB:-0}
|
||||
|
||||
# 计算增量(日志用)
|
||||
DELTA=0
|
||||
if [[ "${PREV_TOTAL_MB}" =~ ^[0-9]+$ ]] && [[ ${CURRENT_TOTAL_MB} -gt ${PREV_TOTAL_MB} ]]; then
|
||||
DELTA=$(( CURRENT_TOTAL_MB - PREV_TOTAL_MB ))
|
||||
elif [[ ! "${PREV_TOTAL_MB}" =~ ^[0-9]+$ ]]; then
|
||||
DELTA=-1 # 首次采,设 -1 表示"无对比"
|
||||
fi
|
||||
|
||||
# ---- 磁盘 / inode / load ----
|
||||
DISK_PCT=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||||
INODE_PCT=$(df -i / | tail -1 | awk '{print $5}' | tr -d '%')
|
||||
LOADAVG=$(cat /proc/loadavg | awk '{print $1}')
|
||||
DISK_PCT=${DISK_PCT:-0}
|
||||
INODE_PCT=${INODE_PCT:-0}
|
||||
LOADAVG=${LOADAVG:-0}
|
||||
|
||||
# 写 SQLite(把"当前总大小 MB"存起来,下次算差值)
|
||||
# 注意 docker_root_write_mb 存的是整数 MB(du -sm 输出本身就是整数,.0 是 sqlite REAL 类型加的)
|
||||
sqlite_exec "INSERT INTO infra_metrics VALUES (
|
||||
${NOW}, ${DOCKER_OK}, ${CONTAINER_COUNT}, CAST(${CURRENT_TOTAL_MB} AS INTEGER),
|
||||
${DISK_PCT}, ${INODE_PCT}, ${LOADAVG}
|
||||
);"
|
||||
|
||||
log_info "[INFRA] ✅ docker_ok=${DOCKER_OK} containers=${CONTAINER_COUNT} root_overlay2=${CURRENT_TOTAL_MB}MB delta=${DELTA}MB disk=${DISK_PCT}% inode=${INODE_PCT}% loadavg=${LOADAVG}"
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · SQLite 数据库初始化
|
||||
# 只建表,不删数据;幂等可重复执行
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
log_info "初始化 SQLite 数据库:${MONITOR_DB}"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS db_metrics (
|
||||
ts INTEGER PRIMARY KEY,
|
||||
threads_running INTEGER,
|
||||
threads_connected INTEGER,
|
||||
threads_idle INTEGER,
|
||||
uptime_sec INTEGER,
|
||||
qps_read INTEGER,
|
||||
qps_write INTEGER,
|
||||
slow_queries INTEGER -- 近 30s 慢查询数(>30s)
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS db_processlist (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER,
|
||||
pid INTEGER,
|
||||
user TEXT,
|
||||
state TEXT,
|
||||
time_sec INTEGER,
|
||||
info TEXT
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_db_proc_ts ON db_processlist(ts);"
|
||||
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_db_proc_time ON db_processlist(time_sec);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS http_metrics (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER,
|
||||
endpoint TEXT,
|
||||
status_code INTEGER,
|
||||
resp_time_ms INTEGER,
|
||||
healthy INTEGER -- 0/1
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_http_ts_ep ON http_metrics(ts, endpoint);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS business_metrics (
|
||||
ts INTEGER PRIMARY KEY,
|
||||
msg_log_hourly INTEGER, -- 本小时 ecology_message_log 增量
|
||||
msg_log_fail_rate REAL, -- 失败率
|
||||
ongoing_requests INTEGER, -- 进行中流程数
|
||||
overdue_tasks INTEGER -- 超时待办数
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS infra_metrics (
|
||||
ts INTEGER PRIMARY KEY,
|
||||
docker_daemon_ok INTEGER, -- 0/1
|
||||
docker_container_count INTEGER,
|
||||
docker_root_write_mb REAL, -- 1 分钟 Docker data-root 写入量 MB
|
||||
disk_usage_pct REAL,
|
||||
inode_usage_pct REAL,
|
||||
load_avg REAL
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS alerts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER,
|
||||
level TEXT, -- P0/P1/P2
|
||||
category TEXT, -- db/http/business/infra
|
||||
title TEXT,
|
||||
detail TEXT,
|
||||
ack INTEGER DEFAULT 0, -- 0=未确认 1=已确认
|
||||
resolved INTEGER DEFAULT 0 -- 0=未恢复 1=已恢复
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_alerts_ts ON alerts(ts);"
|
||||
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_alerts_ack ON alerts(ack, resolved);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS alert_cooldown (
|
||||
alert_key TEXT PRIMARY KEY,
|
||||
last_alert_ts INTEGER
|
||||
);"
|
||||
|
||||
log_info "SQLite schema 初始化完成"
|
||||
Executable
+98
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · Resin 凌晨预防性重启
|
||||
# 每天 02:00 执行,重建 push 线程池,静默死亡前主动清场
|
||||
# crontab: 0 2 * * * /opt/oa-monitor/bin/resin-preventive-restart.sh >> /opt/oa-monitor/logs/resin-restart.log 2>&1
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
OA_HOST="39.99.236.138"
|
||||
OA_USER="root"
|
||||
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
|
||||
LOG_FILE="${MONITOR_LOGS}/resin-preventive-restart.log"
|
||||
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
|
||||
|
||||
START_TS=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo "=== ${START_TS} Resin 预防性重启开始 ===" | tee -a "${LOG_FILE}"
|
||||
|
||||
# Step 1: 写 _oa_ssh.py(paramiko + errors=replace,复用 watchdog 方式)
|
||||
cat > "${OA_SSH_PY}" << 'PYEOF'
|
||||
import paramiko, sys
|
||||
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
ssh.connect(host, username=user, password=password, timeout=15)
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=180)
|
||||
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
|
||||
sys.stderr.write(stderr.read().decode("utf-8", errors="replace"))
|
||||
ssh.close()
|
||||
except Exception as e:
|
||||
print(f"SSH_ERROR: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
chmod +x "${OA_SSH_PY}"
|
||||
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
|
||||
|
||||
# Step 2: 钉钉提前通知
|
||||
dingtalk_alert "P1" "Resin 凌晨预防性重启(开始)" \
|
||||
"**🛠️ Resin 预防性重启开始**\n\n时间:${START_TS}\n\n> 每天凌晨 02:00 执行,重建推送线程池,防止静默死亡。停机窗口约 30-60 秒。"
|
||||
|
||||
# Step 3: 记录旧 PID
|
||||
OLD_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
|
||||
echo "旧 PID=${OLD_PID}" | tee -a "${LOG_FILE}"
|
||||
|
||||
# Step 4: 停 Resin
|
||||
echo "--- stopresin.sh ---" | tee -a "${LOG_FILE}"
|
||||
${OA_SSH} "bash /usr/weaver/Resin/bin/stopresin.sh 2>&1" | tee -a "${LOG_FILE}"
|
||||
|
||||
# Step 5: 等端口关闭(最多 30 秒)
|
||||
for i in $(seq 1 30); do
|
||||
NEW_CHECK=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null)
|
||||
if [[ "${NEW_CHECK}" != "200" && "${NEW_CHECK}" != "404" && "${NEW_CHECK}" != "401" ]]; then
|
||||
echo "[${i}s] 80 端口已关闭 (HTTP=${NEW_CHECK})" | tee -a "${LOG_FILE}"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
sleep 3
|
||||
|
||||
# Step 6: 起 Resin
|
||||
echo "--- startresin.sh ---" | tee -a "${LOG_FILE}"
|
||||
${OA_SSH} "bash /usr/weaver/Resin/bin/startresin.sh 2>&1" | tee -a "${LOG_FILE}"
|
||||
|
||||
# Step 7: 等 Resin 恢复(最多 90 秒)
|
||||
HTTP_CODE="000"
|
||||
for i in $(seq 1 90); do
|
||||
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/ 2>/dev/null" 2>/dev/null)
|
||||
if [[ "${HTTP_CODE}" == "200" || "${HTTP_CODE}" == "404" || "${HTTP_CODE}" == "401" || "${HTTP_CODE}" == "500" ]]; then
|
||||
echo "[${i}s] Resin 恢复 (HTTP=${HTTP_CODE})" | tee -a "${LOG_FILE}"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Step 8: 记录新 PID
|
||||
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}' | head -1" 2>/dev/null)
|
||||
END_TS=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# Step 9: 钉钉结果通知
|
||||
BODY="**🛠️ Resin 预防性重启完成**\n\n"
|
||||
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
||||
BODY="${BODY}| 旧 PID | \`${OLD_PID}\` |\n"
|
||||
BODY="${BODY}| 新 PID | \`${NEW_PID}\` |\n"
|
||||
BODY="${BODY}| HTTP 80 端口 | \`${HTTP_CODE}\` |\n"
|
||||
BODY="${BODY}| 耗时 | $(($(date +%s) - $(date -d "${START_TS}" +%s 2>/dev/null || date +%s)))秒 |\n"
|
||||
BODY="${BODY}| 结束时间 | ${END_TS} |\n"
|
||||
|
||||
if [[ "${NEW_PID}" == "${OLD_PID}" ]]; then
|
||||
BODY="${BODY}\n> ⚠️ 新旧 PID 相同,重启可能未生效,请检查!"
|
||||
dingtalk_alert "P1" "Resin 预防性重启(⚠️ PID 未变)" "${BODY}"
|
||||
else
|
||||
dingtalk_alert "P1" "Resin 预防性重启(✅ 成功)" "${BODY}"
|
||||
fi
|
||||
|
||||
echo "=== ${END_TS} Resin 预防性重启结束(${OLD_PID}→${NEW_PID}, HTTP=${HTTP_CODE}) ===" | tee -a "${LOG_FILE}"
|
||||
Executable
+147
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · Resin Watchdog (v4 - PID 变化检测 + message_log 入库判定)
|
||||
# v3 盲区修复:Resin 被 WatchdogManager kill 后秒拉起,message_log 继续刷看起来"健康"
|
||||
# v4 新增:每次巡检记录 Resin PID,PID 变化 -> 钉钉 P1 告警
|
||||
# 健康依据(双重):
|
||||
# 1. ecology_message_log 最近 15 分钟入库 >= 10 条(推送线程活着)
|
||||
# 2. Resin PID 未变化(进程未被 kill 重启)
|
||||
# crontab: */5 * * * * /opt/oa-monitor/bin/resin-watchdog.sh
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
OA_HOST="39.99.236.138"
|
||||
OA_USER="root"
|
||||
OA_PASS="KTOFKjVkVOGlNgvs8yUP1+PM1VM"
|
||||
OA_RESTART_CMD="bash /usr/weaver/Resin/bin/stopresin.sh 2>&1; sleep 5; bash /usr/weaver/Resin/bin/startresin.sh 2>&1"
|
||||
|
||||
MSG_WINDOW_MIN=15
|
||||
MSG_THRESHOLD=10
|
||||
START_HOUR=8
|
||||
END_HOUR=22
|
||||
|
||||
PID_FILE="${MONITOR_ROOT}/state/resin_pid.last"
|
||||
LOG_FILE="${MONITOR_LOGS}/resin-watchdog.log"
|
||||
OA_SSH_PY="${MONITOR_ROOT}/bin/_oa_ssh.py"
|
||||
|
||||
# --- 依赖检查 ---
|
||||
if ! python3 -c 'import paramiko' 2>/dev/null; then
|
||||
MSG="[WATCHDOG] paramiko 不可用!"
|
||||
log_error "${MSG}"
|
||||
dingtalk_alert "P0" "Resin Watchdog 无法运行" "${MSG}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- SSH helper ---
|
||||
cat > "${OA_SSH_PY}" << 'PYEOF'
|
||||
import paramiko, sys
|
||||
host, user, password, cmd = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
ssh.connect(host, username=user, password=password, timeout=10)
|
||||
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||
sys.stdout.write(stdout.read().decode("utf-8", errors="replace"))
|
||||
ssh.close()
|
||||
except Exception as e:
|
||||
print(f"SSH_ERROR: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
chmod +x "${OA_SSH_PY}"
|
||||
OA_SSH="python3 ${OA_SSH_PY} ${OA_HOST} ${OA_USER} ${OA_PASS}"
|
||||
|
||||
# ============================================================================
|
||||
# Step 1: PID 变化检测(每次都跑,不依赖 message_log 结果)
|
||||
# ============================================================================
|
||||
mkdir -p "$(dirname "${PID_FILE}")"
|
||||
|
||||
PID_INFO=$(${OA_SSH} "ps -eo pid,etimes,cmd | grep 'java.*-Dresin.home.*/Resin/' | grep -v watchdog | grep -v grep | awk '{print \$1, \$2}' | head -1" 2>/dev/null)
|
||||
CUR_PID=$(echo "${PID_INFO}" | awk '{print $1}')
|
||||
CUR_ETIMES=$(echo "${PID_INFO}" | awk '{print $2}')
|
||||
|
||||
if [[ ! "${CUR_PID}" =~ ^[0-9]+$ ]]; then
|
||||
log_warn "[WATCHDOG] 无法获取 Resin PID, 跳过 PID 检测"
|
||||
else
|
||||
LAST_PID=""
|
||||
LAST_ETIMES=""
|
||||
if [[ -f "${PID_FILE}" ]]; then
|
||||
read -r LAST_PID LAST_ETIMES _ < "${PID_FILE}" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [[ -z "${LAST_PID}" ]]; then
|
||||
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
|
||||
log_info "[WATCHDOG] PID 基线初始化: Resin PID=${CUR_PID}, 已运行 ${CUR_ETIMES}s"
|
||||
elif [[ "${LAST_PID}" != "${CUR_PID}" ]]; then
|
||||
NOW_HOUR=$(date +%H)
|
||||
NOW_MIN=$(date +%M)
|
||||
IS_PREVENTIVE=0
|
||||
if [[ "${NOW_HOUR}" == "02" && "${NOW_MIN}" -ge 0 && "${NOW_MIN}" -le 10 ]]; then
|
||||
IS_PREVENTIVE=1
|
||||
fi
|
||||
|
||||
if [[ "${IS_PREVENTIVE}" -eq 1 ]]; then
|
||||
log_info "[WATCHDOG] PID 变化在预防性重启窗口(02:00-02:10): ${LAST_PID} -> ${CUR_PID}, 正常"
|
||||
else
|
||||
RESTART_TIME=$(date -d "@$(($(date +%s) - CUR_ETIMES))" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
BODY="Resin PID 异常变化(非预期重启)\n\n"
|
||||
BODY="${BODY}上次 PID: ${LAST_PID}\n"
|
||||
BODY="${BODY}新 PID: ${CUR_PID}\n"
|
||||
BODY="${BODY}推算重启时间: ${RESTART_TIME}\n"
|
||||
BODY="${BODY}新进程已运行: ${CUR_ETIMES} 秒\n\n"
|
||||
BODY="${BODY}Resin 被 WatchdogManager kill 后拉起。message_log 可能仍在刷(假活跃), watchdog v3 会漏检。"
|
||||
dingtalk_alert "P1" "Resin PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算 ${RESTART_TIME})" "${BODY}"
|
||||
|
||||
log_warn "[WATCHDOG] PID 异常变化: ${LAST_PID} -> ${CUR_PID} (推算重启 ${RESTART_TIME}), 已发 P1 告警"
|
||||
fi
|
||||
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
|
||||
else
|
||||
echo "${CUR_PID} ${CUR_ETIMES} $(date +%s)" > "${PID_FILE}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# Step 2: message_log 入库检测 (原 v3 逻辑)
|
||||
# ============================================================================
|
||||
MSG_COUNT=$(MYSQL_PWD="${DB_PASS}" mysql -h39.100.72.102 -P3306 -uecology "${DB_NAME}" \
|
||||
-sN -e "SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= DATE_FORMAT(NOW() - INTERVAL ${MSG_WINDOW_MIN} MINUTE, '%H:%i:%s');" 2>/dev/null)
|
||||
|
||||
if [[ ! "${MSG_COUNT}" =~ ^[0-9]+$ ]]; then
|
||||
log_warn "[WATCHDOG] 查询 message_log 失败, 跳过"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log_info "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 入库 ${MSG_COUNT} 条"
|
||||
|
||||
HOUR=$(date +%H)
|
||||
if [[ ${MSG_COUNT} -ge ${MSG_THRESHOLD} ]]; then
|
||||
log_info "[WATCHDOG] message_log 入库正常(${MSG_COUNT} >= ${MSG_THRESHOLD}), 推送健康"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${HOUR} -lt ${START_HOUR} || ${HOUR} -ge ${END_HOUR} ]]; then
|
||||
log_warn "[WATCHDOG] 凌晨时段(${HOUR}时), message_log 仅 ${MSG_COUNT} 条, 仅记录"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- 工作时间 + 入库过少 -> 触发重启 ---
|
||||
log_warn "[WATCHDOG] 最近 ${MSG_WINDOW_MIN} 分钟 message_log 仅 ${MSG_COUNT} 条 < ${MSG_THRESHOLD}, 触发 Resin 重启!"
|
||||
${OA_SSH} "${OA_RESTART_CMD}" 2>&1 | tee -a "${LOG_FILE}"
|
||||
sleep 20
|
||||
|
||||
NEW_PID=$(${OA_SSH} "ps -eo pid,cmd | grep 'java.*Resin.*resin.xml' | grep -v grep | awk '{print \$1}'" 2>/dev/null | head -1)
|
||||
HTTP_CODE=$(${OA_SSH} "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:80/wui/main/Login.jsp" 2>/dev/null)
|
||||
|
||||
BODY="Resin watchdog 自动重启\n\n"
|
||||
BODY="${BODY}message_log 最近 ${MSG_WINDOW_MIN} 分钟入库: ${MSG_COUNT} 条 (阈值 ${MSG_THRESHOLD})\n"
|
||||
BODY="${BODY}新 Resin PID: ${NEW_PID}\n"
|
||||
BODY="${BODY}HTTP 80: ${HTTP_CODE}\n"
|
||||
dingtalk_alert "P0" "Resin watchdog 自动重启(message_log 入库 ${MSG_COUNT} 条 < ${MSG_THRESHOLD})" "${BODY}"
|
||||
|
||||
if [[ "${NEW_PID}" =~ ^[0-9]+$ ]]; then
|
||||
echo "${NEW_PID} 0 $(date +%s)" > "${PID_FILE}"
|
||||
fi
|
||||
|
||||
log_info "[WATCHDOG] Resin 重启完成, 新 PID=${NEW_PID}, HTTP=${HTTP_CODE}"
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 长查询检测 dry-run KILL(每 10 分钟)
|
||||
# 默认 DRY_RUN=1(只报告,不真 KILL)。--execute 才真的 KILL
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
DRY_RUN=1
|
||||
for arg in "$@"; do
|
||||
[[ "${arg}" == "--execute" ]] && DRY_RUN=0
|
||||
[[ "${arg}" == "--dry-run" ]] && DRY_RUN=1
|
||||
done
|
||||
|
||||
NOW=$(date +%s)
|
||||
|
||||
MYSQL_RUN() {
|
||||
MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology "$@" 2>/dev/null
|
||||
}
|
||||
|
||||
log_info "[LONG-CHECK] 开始(DRY_RUN=${DRY_RUN},阈值 ${ALERT_LONG_QUERY_SEC}s)"
|
||||
|
||||
RESULT=$(MYSQL_RUN "SHOW FULL PROCESSLIST;" | awk -F'\t' -v t="${ALERT_LONG_QUERY_SEC}" '
|
||||
NR==1 { next }
|
||||
NF < 10 { next }
|
||||
{
|
||||
pid=$1; user=$2; state=$6; time=$7; info=$9
|
||||
if (time+0 >= t+0 && time+0 > 0) {
|
||||
gsub(/\t/, " ", info)
|
||||
printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300)
|
||||
}
|
||||
}
|
||||
')
|
||||
|
||||
if [[ -z "${RESULT}" ]]; then
|
||||
log_info "[LONG-CHECK] ✅ 无长查询"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
COUNT=$(echo "${RESULT}" | wc -l | tr -d ' ')
|
||||
log_warn "[LONG-CHECK] ⚠️ ${COUNT} 条长查询"
|
||||
|
||||
DETAIL=""
|
||||
while IFS=$'\t' read -r pid time user state info; do
|
||||
[[ -z "${pid}" ]] && continue
|
||||
DETAIL="${DETAIL}\n- PID \`${pid}\` **${time}s** \`${user}\` ${state}"
|
||||
DETAIL="${DETAIL}\n \`\`\`\n ${info}\n \`\`\`"
|
||||
done <<< "${RESULT}"
|
||||
|
||||
MODE_LABEL=$([[ ${DRY_RUN} -eq 1 ]] && echo "🔒 DRY-RUN" || echo "🚨 执行 KILL")
|
||||
BODY="**${COUNT} 条长查询**(${ALERT_LONG_QUERY_SEC}s 阈值)· ${MODE_LABEL}\n\n${DETAIL}"
|
||||
dingtalk_alert "P0" "长查询阻塞需处理" "${BODY}"
|
||||
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询', '${COUNT} 条, dry_run=${DRY_RUN}');"
|
||||
|
||||
if [[ ${DRY_RUN} -eq 0 ]]; then
|
||||
log_warn "[LONG-CHECK] 执行 KILL!"
|
||||
while IFS=$'\t' read -r pid time user state info; do
|
||||
[[ -z "${pid}" ]] && continue
|
||||
MYSQL_RUN "KILL ${pid};"
|
||||
log_warn "[LONG-CHECK] KILL ${pid} (${time}s, ${user})"
|
||||
done <<< "${RESULT}"
|
||||
else
|
||||
log_info "[LONG-CHECK] DRY-RUN,不执行 KILL"
|
||||
fi
|
||||
Reference in new issue
Block a user