fix: check-alert v3 修复钉钉推送断档 P0 假阳性
根因:sqlite_exec 吞掉 stderr(2>/dev/null),查询失败返回空串, 0 默认为 0 导致误报 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验, 查询失败跳过检测并 warn 日志
This commit is contained in:
1 parent
7ab75364b1
commit
20fa48fb34
17 files changed
+1747
-26
No files matched your search
+46
-25
@@ -4,6 +4,9 @@
|
||||
# v2 修复(2026-10-28):
|
||||
# - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警
|
||||
# - Step 3 业务层检测被 Step 0 结果门禁
|
||||
# v3 修复(2026-11-04):
|
||||
# - 业务层 P0 推送断档假阳性:sqlite_exec 吞错误(2>/dev/null)→ 查询失败返回空串 → :-0 默认为 0 → 误报
|
||||
# - 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,查询失败跳过检测并 warn
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
@@ -36,7 +39,7 @@ else
|
||||
if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟
|
||||
BIZ_FRESH=false
|
||||
HUMAN_AGE=$((BIZ_AGE / 60))
|
||||
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec \"SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')\") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec \"SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')\") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
|
||||
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec "SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec "SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
|
||||
dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}"
|
||||
log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)"
|
||||
else
|
||||
@@ -94,34 +97,52 @@ fi
|
||||
if [[ "${BIZ_FRESH}" != "true" ]]; then
|
||||
log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)"
|
||||
else
|
||||
# 数据新鲜 → 正常业务层判断
|
||||
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
HOUR_MSG=${HOUR_MSG:-0}
|
||||
ONGOING=${ONGOING:-0}
|
||||
# v3 修复:显式调用 sqlite3 + 检查退出码 + 正则校验,区分"查询失败"和"结果为 0"
|
||||
MSG_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null)
|
||||
MSG_RC=$?
|
||||
ONGOING_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null)
|
||||
ONGOING_RC=$?
|
||||
|
||||
CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal
|
||||
WORK_HOURS_ENABLED=false
|
||||
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
|
||||
WORK_HOURS_ENABLED=true
|
||||
# 双重校验:退出码必须为 0,且返回值非空且为纯数字
|
||||
BIZ_QUERY_OK=true
|
||||
if [[ ${MSG_RC} -ne 0 ]] || [[ -z "${MSG_RAW}" ]] || [[ ! "${MSG_RAW}" =~ ^[0-9]+$ ]]; then
|
||||
log_warn "[ALERT] SQLite msg_log_hourly 查询异常 rc=${MSG_RC} raw='${MSG_RAW}',跳过业务层(防假阳性)"
|
||||
BIZ_QUERY_OK=false
|
||||
fi
|
||||
if [[ ${ONGOING_RC} -ne 0 ]] || [[ -z "${ONGOING_RAW}" ]] || [[ ! "${ONGOING_RAW}" =~ ^[0-9]+$ ]]; then
|
||||
log_warn "[ALERT] SQLite ongoing_requests 查询异常 rc=${ONGOING_RC} raw='${ONGOING_RAW}',跳过业务层(防假阳性)"
|
||||
BIZ_QUERY_OK=false
|
||||
fi
|
||||
|
||||
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
|
||||
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then
|
||||
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
|
||||
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
||||
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n"
|
||||
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n"
|
||||
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n"
|
||||
BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n"
|
||||
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n"
|
||||
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n"
|
||||
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**"
|
||||
dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
|
||||
fi
|
||||
if [[ "${BIZ_QUERY_OK}" != "true" ]]; then
|
||||
log_warn "[ALERT] ⏸️ 业务层数据查询异常,跳过钉钉推送断档 P0 检测"
|
||||
else
|
||||
log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警"
|
||||
HOUR_MSG=${MSG_RAW}
|
||||
ONGOING=${ONGOING_RAW}
|
||||
|
||||
CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal
|
||||
WORK_HOURS_ENABLED=false
|
||||
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
|
||||
WORK_HOURS_ENABLED=true
|
||||
fi
|
||||
|
||||
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
|
||||
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then
|
||||
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
|
||||
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
||||
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n"
|
||||
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n"
|
||||
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n"
|
||||
BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n"
|
||||
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n"
|
||||
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n"
|
||||
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**"
|
||||
dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
|
||||
fi
|
||||
else
|
||||
log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
Executable
+161
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""fix-info-read-dummy.py — 补 DUMMY 根治 updateBizState 死循环 (Python 3.6 兼容)"""
|
||||
import sys, os, time, subprocess, json, urllib.request
|
||||
|
||||
DB_HOST, DB_PORT, DB_USER = "39.100.72.102", 3306, "ecology"
|
||||
DB_PASS, DB_NAME = "Z!BYT+jSDDyYHMdX+qFSLzJCYXW", "ecology"
|
||||
OA_HOST, OA_USER, OA_PASS = "39.99.236.138", "root", "KTOFKjVkVOGlNgvs8yUP1+PM1VM"
|
||||
DINGTALK = "https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249"
|
||||
BATCH = 5000
|
||||
LOG = "/opt/oa-monitor/logs/fix-info-read.log"
|
||||
|
||||
# === 正确值(从之前 DUMMY_FIX_ 样例提取)===
|
||||
MESSAGETYPE = 25 # int!不是 'archive'
|
||||
BIZTYPE = 86 # int!上次成功用的
|
||||
STATE = 2 # 已读
|
||||
BIZSTATE = 1 # 已更新(关键!让 updateBizState 不再重试)
|
||||
GROUPTYPE = "archive-fix"
|
||||
BIZTITLE = "dummy fix"
|
||||
|
||||
def log(msg, level="INFO"):
|
||||
line = "[%s] [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S"), level, msg)
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def run_mysql(sql, fetch=True):
|
||||
env = os.environ.copy(); env["MYSQL_PWD"] = DB_PASS
|
||||
r = subprocess.run(["mysql","-h"+DB_HOST,"-P"+str(DB_PORT),"-u"+DB_USER,DB_NAME,"-N","-e",sql],
|
||||
env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=600)
|
||||
if r.returncode != 0:
|
||||
log("MySQL ERROR: " + r.stderr.decode().strip(), "ERROR")
|
||||
return None
|
||||
if not fetch: return True
|
||||
return r.stdout.decode().strip()
|
||||
|
||||
def dingtalk(level, title, body=""):
|
||||
text = "**[%s] %s**\n\n%s\n\n> OA DUMMY @ %s" % (level, title, body, time.strftime("%Y-%m-%d %H:%M:%S"))
|
||||
try:
|
||||
urllib.request.urlopen(urllib.request.Request(DINGTALK, json.dumps({"msgtype":"markdown","markdown":{"title":title,"text":text}}).encode(), {"Content-Type":"application/json"}), timeout=5)
|
||||
except Exception as e: log("钉钉: "+str(e), "WARN")
|
||||
|
||||
def check_http():
|
||||
env = os.environ.copy()
|
||||
cmd = ("python3 -c 'import paramiko; ssh=paramiko.SSHClient(); ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()); "
|
||||
"ssh.connect(\"%s\",username=\"%s\",password=\"%s\",timeout=15); "
|
||||
"_,o,_=ssh.exec_command(\"curl -s -o /dev/null -w %%{http_code} http://127.0.0.1:80/\"); "
|
||||
"print(o.read().decode(\"utf-8\",\"replace\")); ssh.close()'" % (OA_HOST, OA_USER, OA_PASS))
|
||||
r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=25, env=env)
|
||||
return "200" in r.stdout.decode().strip()
|
||||
|
||||
def check_tr():
|
||||
r = run_mysql("SHOW GLOBAL STATUS WHERE Variable_name='Threads_running';")
|
||||
if not r: return 999
|
||||
try: return int(r.split("\t")[-1].strip())
|
||||
except: return 999
|
||||
|
||||
def wait_safe(max_wait=120):
|
||||
for i in range(max_wait // 5):
|
||||
if check_http() and check_tr() < 30: return True
|
||||
log("等待恢复 (%ds) HTTP=%s TR=%d" % (i*5, check_http(), check_tr()), "WARN")
|
||||
time.sleep(5)
|
||||
return False
|
||||
|
||||
# === Phase 0 ===
|
||||
log("===== Phase 0: 前置检查 =====")
|
||||
DRY = "--dry-run" in sys.argv
|
||||
if DRY: log("*** DRY RUN ***")
|
||||
|
||||
ok = check_http(); log("Resin HTTP 80 = %s" % ok)
|
||||
if not ok: log("Resin 不通!中止!", "CRITICAL"); sys.exit(1)
|
||||
|
||||
tr = check_tr(); log("Threads_running = %d" % tr)
|
||||
if tr >= 15:
|
||||
wait_safe(180); tr = check_tr()
|
||||
|
||||
existing = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_FIX_%';")
|
||||
log("现有 DUMMY_FIX = %s" % existing)
|
||||
|
||||
log("Phase 1: 备份表...")
|
||||
run_mysql("CREATE TABLE IF NOT EXISTS ecology_message_info_read_bak_20261104 AS SELECT * FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';", fetch=False)
|
||||
|
||||
dingtalk("INFO", "DUMMY 补数据启动", "HTTP=200 ✅\nTR=%d\nDUMMY_FIX=%s" % (tr, existing))
|
||||
|
||||
# === Phase 2 ===
|
||||
total = [0]
|
||||
|
||||
def insert_user(uid, prefix, col):
|
||||
last_id = 0; batch = 0; user_total = 0
|
||||
while True:
|
||||
sql = ("INSERT INTO ecology_message_info_read "
|
||||
"(ID, MESSAGEID, MESSAGETYPE, USERID, TARGETID, STATE, BIZSTATE, "
|
||||
"CREATEDATE, CREATETIME, MESSAGEGROUPTYPE, BIZTITLE, BIZTYPE) "
|
||||
"SELECT CONCAT('%s%s_', REQUESTID), 0, %d, %d, REQUESTID, %d, %d, "
|
||||
"CURDATE(), CURTIME(), '%s', '%s', %d "
|
||||
"FROM workflow_requestbase "
|
||||
"WHERE %s = %d AND STATUS LIKE '%%归档%%' AND DELETED = 0 AND REQUESTID > %d "
|
||||
"ORDER BY REQUESTID LIMIT %d;" %
|
||||
(prefix, "%d", MESSAGETYPE, uid, STATE, BIZSTATE, GROUPTYPE, BIZTITLE, BIZTYPE,
|
||||
col, uid, last_id, BATCH))
|
||||
if DRY:
|
||||
batch += 1; user_total += BATCH; last_id += BATCH
|
||||
if batch >= 5: break
|
||||
time.sleep(0.2); continue
|
||||
|
||||
run_mysql(sql, fetch=False)
|
||||
rc = run_mysql("SELECT ROW_COUNT();")
|
||||
inserted = int(rc) if rc and rc.isdigit() else 0
|
||||
if inserted == 0: break
|
||||
|
||||
last_sql = "SELECT MAX(REQUESTID) FROM workflow_requestbase WHERE %s = %d AND REQUESTID > %d LIMIT %d;" % (col, uid, last_id, BATCH)
|
||||
lv = run_mysql(last_sql)
|
||||
if lv and lv.isdigit(): last_id = int(lv)
|
||||
|
||||
batch += 1; user_total += inserted; total[0] += inserted
|
||||
log(" User=%d batch=%d +%d rows (total=%d)" % (uid, batch, inserted, total[0]))
|
||||
|
||||
if not check_http() or check_tr() >= 30:
|
||||
log("异常!暂停...", "WARN"); wait_safe(10)
|
||||
time.sleep(1)
|
||||
|
||||
if user_total > 0:
|
||||
log("User=%d 完成: %d 条 %d 批" % (uid, user_total, batch))
|
||||
|
||||
# Phase 2a — 全量业务用户(不限 TOP 100)
|
||||
log("===== Phase 2a: CREATER (全量) =====")
|
||||
creaters_raw = run_mysql("SELECT CREATER, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND CREATER!=1 GROUP BY CREATER ORDER BY COUNT(*) DESC;")
|
||||
creaters = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in creaters_raw.strip().split("\n") if l.strip()]
|
||||
log("CREATER 用户=%d 归档总量=%d" % (len(creaters), sum(c for _,c in creaters)))
|
||||
|
||||
for i, (uid, tot) in enumerate(creaters, 1):
|
||||
log("--- CREATER #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
|
||||
insert_user(uid, "DUMMY_B_CREATER_", "CREATER")
|
||||
if i % 10 == 0:
|
||||
dingtalk("INFO", "CREATER %d/100" % i, "已插入=%d" % total[0])
|
||||
|
||||
# Phase 2b — 全量业务用户(不限 TOP 100)
|
||||
log("===== Phase 2b: LASTOPERATOR (全量) =====")
|
||||
ops_raw = run_mysql("SELECT LASTOPERATOR, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND LASTOPERATOR!=1 GROUP BY LASTOPERATOR ORDER BY COUNT(*) DESC;")
|
||||
ops = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in ops_raw.strip().split("\n") if l.strip()]
|
||||
log("OPERATOR 用户=%d 归档总量=%d" % (len(ops), sum(c for _,c in ops)))
|
||||
|
||||
for i, (uid, tot) in enumerate(ops, 1):
|
||||
log("--- OPERATOR #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
|
||||
insert_user(uid, "DUMMY_B_OPERATOR_", "LASTOPERATOR")
|
||||
if i % 10 == 0:
|
||||
dingtalk("INFO", "OPERATOR %d/100" % i, "已插入=%d" % total[0])
|
||||
|
||||
# Phase 3
|
||||
log("===== Phase 3: 验证 =====")
|
||||
fc = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';")
|
||||
log("DUMMY_B 总数 = %s" % fc)
|
||||
http_ok = check_http(); log("Resin HTTP = %s" % http_ok)
|
||||
tr2 = check_tr(); log("Threads_running = %d" % tr2)
|
||||
lm = run_mysql("SELECT MAX(CREATETIME) FROM ecology_message_log WHERE CREATEDATE=CURDATE();")
|
||||
log("message_log 最新 = %s" % lm)
|
||||
|
||||
dingtalk("P0", "DUMMY 补数据完成 ✅",
|
||||
"**DUMMY_B**: %s 条\n**HTTP**: %s\n**TR**: %d\n**最新消息**: %s\n\n回滚: DELETE FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%%';" %
|
||||
(fc, http_ok, tr2, lm))
|
||||
log("===== 全部完成 =====")
|
||||
Reference in new issue
Block a user