fix: check-alert v3 修复钉钉推送断档 P0 假阳性

根因:sqlite_exec 吞掉 stderr(2>/dev/null),查询失败返回空串,
0 默认为 0 导致误报

修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,
查询失败跳过检测并 warn 日志
This commit is contained in:
oa-monitor committed 2026-11-04 19:41:18 +08:00
1 parent 7ab75364b1
commit 20fa48fb34
17 files changed
+1747 -26

No files matched your search

+46 -25
View File
@@ -4,6 +4,9 @@
# v2 修复(2026-10-28):
# - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警
# - Step 3 业务层检测被 Step 0 结果门禁
# v3 修复(2026-11-04):
# - 业务层 P0 推送断档假阳性:sqlite_exec 吞错误(2>/dev/null)→ 查询失败返回空串 → :-0 默认为 0 → 误报
# - 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,查询失败跳过检测并 warn
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -36,7 +39,7 @@ else
if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟
BIZ_FRESH=false
HUMAN_AGE=$((BIZ_AGE / 60))
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec \"SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')\") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec \"SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')\") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec "SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec "SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}"
log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)"
else
@@ -94,34 +97,52 @@ fi
if [[ "${BIZ_FRESH}" != "true" ]]; then
log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)"
else
# 数据新鲜 → 正常业务层判断
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
HOUR_MSG=${HOUR_MSG:-0}
ONGOING=${ONGOING:-0}
# v3 修复:显式调用 sqlite3 + 检查退出码 + 正则校验,区分"查询失败"和"结果为 0"
MSG_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null)
MSG_RC=$?
ONGOING_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null)
ONGOING_RC=$?
CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal
WORK_HOURS_ENABLED=false
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
WORK_HOURS_ENABLED=true
# 双重校验:退出码必须为 0,且返回值非空且为纯数字
BIZ_QUERY_OK=true
if [[ ${MSG_RC} -ne 0 ]] || [[ -z "${MSG_RAW}" ]] || [[ ! "${MSG_RAW}" =~ ^[0-9]+$ ]]; then
log_warn "[ALERT] SQLite msg_log_hourly 查询异常 rc=${MSG_RC} raw='${MSG_RAW}',跳过业务层(防假阳性)"
BIZ_QUERY_OK=false
fi
if [[ ${ONGOING_RC} -ne 0 ]] || [[ -z "${ONGOING_RAW}" ]] || [[ ! "${ONGOING_RAW}" =~ ^[0-9]+$ ]]; then
log_warn "[ALERT] SQLite ongoing_requests 查询异常 rc=${ONGOING_RC} raw='${ONGOING_RAW}',跳过业务层(防假阳性)"
BIZ_QUERY_OK=false
fi
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n"
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n"
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n"
BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n"
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n"
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n"
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**"
dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
fi
if [[ "${BIZ_QUERY_OK}" != "true" ]]; then
log_warn "[ALERT] ⏸️ 业务层数据查询异常,跳过钉钉推送断档 P0 检测"
else
log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警"
HOUR_MSG=${MSG_RAW}
ONGOING=${ONGOING_RAW}
CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal
WORK_HOURS_ENABLED=false
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
WORK_HOURS_ENABLED=true
fi
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n"
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n"
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n"
BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n"
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n"
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n"
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**"
dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
fi
else
log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警"
fi
fi
fi
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""fix-info-read-dummy.py — 补 DUMMY 根治 updateBizState 死循环 (Python 3.6 兼容)"""
import sys, os, time, subprocess, json, urllib.request
DB_HOST, DB_PORT, DB_USER = "39.100.72.102", 3306, "ecology"
DB_PASS, DB_NAME = "Z!BYT+jSDDyYHMdX+qFSLzJCYXW", "ecology"
OA_HOST, OA_USER, OA_PASS = "39.99.236.138", "root", "KTOFKjVkVOGlNgvs8yUP1+PM1VM"
DINGTALK = "https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249"
BATCH = 5000
LOG = "/opt/oa-monitor/logs/fix-info-read.log"
# === 正确值(从之前 DUMMY_FIX_ 样例提取)===
MESSAGETYPE = 25 # int!不是 'archive'
BIZTYPE = 86 # int!上次成功用的
STATE = 2 # 已读
BIZSTATE = 1 # 已更新(关键!让 updateBizState 不再重试)
GROUPTYPE = "archive-fix"
BIZTITLE = "dummy fix"
def log(msg, level="INFO"):
line = "[%s] [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S"), level, msg)
print(line, flush=True)
with open(LOG, "a") as f:
f.write(line + "\n")
def run_mysql(sql, fetch=True):
env = os.environ.copy(); env["MYSQL_PWD"] = DB_PASS
r = subprocess.run(["mysql","-h"+DB_HOST,"-P"+str(DB_PORT),"-u"+DB_USER,DB_NAME,"-N","-e",sql],
env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=600)
if r.returncode != 0:
log("MySQL ERROR: " + r.stderr.decode().strip(), "ERROR")
return None
if not fetch: return True
return r.stdout.decode().strip()
def dingtalk(level, title, body=""):
text = "**[%s] %s**\n\n%s\n\n> OA DUMMY @ %s" % (level, title, body, time.strftime("%Y-%m-%d %H:%M:%S"))
try:
urllib.request.urlopen(urllib.request.Request(DINGTALK, json.dumps({"msgtype":"markdown","markdown":{"title":title,"text":text}}).encode(), {"Content-Type":"application/json"}), timeout=5)
except Exception as e: log("钉钉: "+str(e), "WARN")
def check_http():
env = os.environ.copy()
cmd = ("python3 -c 'import paramiko; ssh=paramiko.SSHClient(); ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()); "
"ssh.connect(\"%s\",username=\"%s\",password=\"%s\",timeout=15); "
"_,o,_=ssh.exec_command(\"curl -s -o /dev/null -w %%{http_code} http://127.0.0.1:80/\"); "
"print(o.read().decode(\"utf-8\",\"replace\")); ssh.close()'" % (OA_HOST, OA_USER, OA_PASS))
r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=25, env=env)
return "200" in r.stdout.decode().strip()
def check_tr():
r = run_mysql("SHOW GLOBAL STATUS WHERE Variable_name='Threads_running';")
if not r: return 999
try: return int(r.split("\t")[-1].strip())
except: return 999
def wait_safe(max_wait=120):
for i in range(max_wait // 5):
if check_http() and check_tr() < 30: return True
log("等待恢复 (%ds) HTTP=%s TR=%d" % (i*5, check_http(), check_tr()), "WARN")
time.sleep(5)
return False
# === Phase 0 ===
log("===== Phase 0: 前置检查 =====")
DRY = "--dry-run" in sys.argv
if DRY: log("*** DRY RUN ***")
ok = check_http(); log("Resin HTTP 80 = %s" % ok)
if not ok: log("Resin 不通!中止!", "CRITICAL"); sys.exit(1)
tr = check_tr(); log("Threads_running = %d" % tr)
if tr >= 15:
wait_safe(180); tr = check_tr()
existing = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_FIX_%';")
log("现有 DUMMY_FIX = %s" % existing)
log("Phase 1: 备份表...")
run_mysql("CREATE TABLE IF NOT EXISTS ecology_message_info_read_bak_20261104 AS SELECT * FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';", fetch=False)
dingtalk("INFO", "DUMMY 补数据启动", "HTTP=200 ✅\nTR=%d\nDUMMY_FIX=%s" % (tr, existing))
# === Phase 2 ===
total = [0]
def insert_user(uid, prefix, col):
last_id = 0; batch = 0; user_total = 0
while True:
sql = ("INSERT INTO ecology_message_info_read "
"(ID, MESSAGEID, MESSAGETYPE, USERID, TARGETID, STATE, BIZSTATE, "
"CREATEDATE, CREATETIME, MESSAGEGROUPTYPE, BIZTITLE, BIZTYPE) "
"SELECT CONCAT('%s%s_', REQUESTID), 0, %d, %d, REQUESTID, %d, %d, "
"CURDATE(), CURTIME(), '%s', '%s', %d "
"FROM workflow_requestbase "
"WHERE %s = %d AND STATUS LIKE '%%归档%%' AND DELETED = 0 AND REQUESTID > %d "
"ORDER BY REQUESTID LIMIT %d;" %
(prefix, "%d", MESSAGETYPE, uid, STATE, BIZSTATE, GROUPTYPE, BIZTITLE, BIZTYPE,
col, uid, last_id, BATCH))
if DRY:
batch += 1; user_total += BATCH; last_id += BATCH
if batch >= 5: break
time.sleep(0.2); continue
run_mysql(sql, fetch=False)
rc = run_mysql("SELECT ROW_COUNT();")
inserted = int(rc) if rc and rc.isdigit() else 0
if inserted == 0: break
last_sql = "SELECT MAX(REQUESTID) FROM workflow_requestbase WHERE %s = %d AND REQUESTID > %d LIMIT %d;" % (col, uid, last_id, BATCH)
lv = run_mysql(last_sql)
if lv and lv.isdigit(): last_id = int(lv)
batch += 1; user_total += inserted; total[0] += inserted
log(" User=%d batch=%d +%d rows (total=%d)" % (uid, batch, inserted, total[0]))
if not check_http() or check_tr() >= 30:
log("异常!暂停...", "WARN"); wait_safe(10)
time.sleep(1)
if user_total > 0:
log("User=%d 完成: %d 条 %d 批" % (uid, user_total, batch))
# Phase 2a — 全量业务用户(不限 TOP 100)
log("===== Phase 2a: CREATER (全量) =====")
creaters_raw = run_mysql("SELECT CREATER, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND CREATER!=1 GROUP BY CREATER ORDER BY COUNT(*) DESC;")
creaters = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in creaters_raw.strip().split("\n") if l.strip()]
log("CREATER 用户=%d 归档总量=%d" % (len(creaters), sum(c for _,c in creaters)))
for i, (uid, tot) in enumerate(creaters, 1):
log("--- CREATER #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
insert_user(uid, "DUMMY_B_CREATER_", "CREATER")
if i % 10 == 0:
dingtalk("INFO", "CREATER %d/100" % i, "已插入=%d" % total[0])
# Phase 2b — 全量业务用户(不限 TOP 100)
log("===== Phase 2b: LASTOPERATOR (全量) =====")
ops_raw = run_mysql("SELECT LASTOPERATOR, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND LASTOPERATOR!=1 GROUP BY LASTOPERATOR ORDER BY COUNT(*) DESC;")
ops = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in ops_raw.strip().split("\n") if l.strip()]
log("OPERATOR 用户=%d 归档总量=%d" % (len(ops), sum(c for _,c in ops)))
for i, (uid, tot) in enumerate(ops, 1):
log("--- OPERATOR #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
insert_user(uid, "DUMMY_B_OPERATOR_", "LASTOPERATOR")
if i % 10 == 0:
dingtalk("INFO", "OPERATOR %d/100" % i, "已插入=%d" % total[0])
# Phase 3
log("===== Phase 3: 验证 =====")
fc = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';")
log("DUMMY_B 总数 = %s" % fc)
http_ok = check_http(); log("Resin HTTP = %s" % http_ok)
tr2 = check_tr(); log("Threads_running = %d" % tr2)
lm = run_mysql("SELECT MAX(CREATETIME) FROM ecology_message_log WHERE CREATEDATE=CURDATE();")
log("message_log 最新 = %s" % lm)
dingtalk("P0", "DUMMY 补数据完成 ✅",
"**DUMMY_B**: %s 条\n**HTTP**: %s\n**TR**: %d\n**最新消息**: %s\n\n回滚: DELETE FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%%';" %
(fc, http_ok, tr2, lm))
log("===== 全部完成 =====")