fix: check-alert v3 修复钉钉推送断档 P0 假阳性

根因:sqlite_exec 吞掉 stderr(2>/dev/null),查询失败返回空串,
0 默认为 0 导致误报

修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,
查询失败跳过检测并 warn 日志
This commit is contained in:
oa-monitor committed 2026-11-04 19:41:18 +08:00
1 parent 7ab75364b1
commit 20fa48fb34
17 files changed
+1747 -26

No files matched your search

+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""fix-info-read-dummy.py — 补 DUMMY 根治 updateBizState 死循环 (Python 3.6 兼容)"""
import sys, os, time, subprocess, json, urllib.request
DB_HOST, DB_PORT, DB_USER = "39.100.72.102", 3306, "ecology"
DB_PASS, DB_NAME = "Z!BYT+jSDDyYHMdX+qFSLzJCYXW", "ecology"
OA_HOST, OA_USER, OA_PASS = "39.99.236.138", "root", "KTOFKjVkVOGlNgvs8yUP1+PM1VM"
DINGTALK = "https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249"
BATCH = 5000
LOG = "/opt/oa-monitor/logs/fix-info-read.log"
# === 正确值(从之前 DUMMY_FIX_ 样例提取)===
MESSAGETYPE = 25 # int!不是 'archive'
BIZTYPE = 86 # int!上次成功用的
STATE = 2 # 已读
BIZSTATE = 1 # 已更新(关键!让 updateBizState 不再重试)
GROUPTYPE = "archive-fix"
BIZTITLE = "dummy fix"
def log(msg, level="INFO"):
line = "[%s] [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S"), level, msg)
print(line, flush=True)
with open(LOG, "a") as f:
f.write(line + "\n")
def run_mysql(sql, fetch=True):
env = os.environ.copy(); env["MYSQL_PWD"] = DB_PASS
r = subprocess.run(["mysql","-h"+DB_HOST,"-P"+str(DB_PORT),"-u"+DB_USER,DB_NAME,"-N","-e",sql],
env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=600)
if r.returncode != 0:
log("MySQL ERROR: " + r.stderr.decode().strip(), "ERROR")
return None
if not fetch: return True
return r.stdout.decode().strip()
def dingtalk(level, title, body=""):
text = "**[%s] %s**\n\n%s\n\n> OA DUMMY @ %s" % (level, title, body, time.strftime("%Y-%m-%d %H:%M:%S"))
try:
urllib.request.urlopen(urllib.request.Request(DINGTALK, json.dumps({"msgtype":"markdown","markdown":{"title":title,"text":text}}).encode(), {"Content-Type":"application/json"}), timeout=5)
except Exception as e: log("钉钉: "+str(e), "WARN")
def check_http():
env = os.environ.copy()
cmd = ("python3 -c 'import paramiko; ssh=paramiko.SSHClient(); ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()); "
"ssh.connect(\"%s\",username=\"%s\",password=\"%s\",timeout=15); "
"_,o,_=ssh.exec_command(\"curl -s -o /dev/null -w %%{http_code} http://127.0.0.1:80/\"); "
"print(o.read().decode(\"utf-8\",\"replace\")); ssh.close()'" % (OA_HOST, OA_USER, OA_PASS))
r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=25, env=env)
return "200" in r.stdout.decode().strip()
def check_tr():
r = run_mysql("SHOW GLOBAL STATUS WHERE Variable_name='Threads_running';")
if not r: return 999
try: return int(r.split("\t")[-1].strip())
except: return 999
def wait_safe(max_wait=120):
for i in range(max_wait // 5):
if check_http() and check_tr() < 30: return True
log("等待恢复 (%ds) HTTP=%s TR=%d" % (i*5, check_http(), check_tr()), "WARN")
time.sleep(5)
return False
# === Phase 0 ===
log("===== Phase 0: 前置检查 =====")
DRY = "--dry-run" in sys.argv
if DRY: log("*** DRY RUN ***")
ok = check_http(); log("Resin HTTP 80 = %s" % ok)
if not ok: log("Resin 不通!中止!", "CRITICAL"); sys.exit(1)
tr = check_tr(); log("Threads_running = %d" % tr)
if tr >= 15:
wait_safe(180); tr = check_tr()
existing = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_FIX_%';")
log("现有 DUMMY_FIX = %s" % existing)
log("Phase 1: 备份表...")
run_mysql("CREATE TABLE IF NOT EXISTS ecology_message_info_read_bak_20261104 AS SELECT * FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';", fetch=False)
dingtalk("INFO", "DUMMY 补数据启动", "HTTP=200 ✅\nTR=%d\nDUMMY_FIX=%s" % (tr, existing))
# === Phase 2 ===
total = [0]
def insert_user(uid, prefix, col):
last_id = 0; batch = 0; user_total = 0
while True:
sql = ("INSERT INTO ecology_message_info_read "
"(ID, MESSAGEID, MESSAGETYPE, USERID, TARGETID, STATE, BIZSTATE, "
"CREATEDATE, CREATETIME, MESSAGEGROUPTYPE, BIZTITLE, BIZTYPE) "
"SELECT CONCAT('%s%s_', REQUESTID), 0, %d, %d, REQUESTID, %d, %d, "
"CURDATE(), CURTIME(), '%s', '%s', %d "
"FROM workflow_requestbase "
"WHERE %s = %d AND STATUS LIKE '%%归档%%' AND DELETED = 0 AND REQUESTID > %d "
"ORDER BY REQUESTID LIMIT %d;" %
(prefix, "%d", MESSAGETYPE, uid, STATE, BIZSTATE, GROUPTYPE, BIZTITLE, BIZTYPE,
col, uid, last_id, BATCH))
if DRY:
batch += 1; user_total += BATCH; last_id += BATCH
if batch >= 5: break
time.sleep(0.2); continue
run_mysql(sql, fetch=False)
rc = run_mysql("SELECT ROW_COUNT();")
inserted = int(rc) if rc and rc.isdigit() else 0
if inserted == 0: break
last_sql = "SELECT MAX(REQUESTID) FROM workflow_requestbase WHERE %s = %d AND REQUESTID > %d LIMIT %d;" % (col, uid, last_id, BATCH)
lv = run_mysql(last_sql)
if lv and lv.isdigit(): last_id = int(lv)
batch += 1; user_total += inserted; total[0] += inserted
log(" User=%d batch=%d +%d rows (total=%d)" % (uid, batch, inserted, total[0]))
if not check_http() or check_tr() >= 30:
log("异常!暂停...", "WARN"); wait_safe(10)
time.sleep(1)
if user_total > 0:
log("User=%d 完成: %d 条 %d 批" % (uid, user_total, batch))
# Phase 2a — 全量业务用户(不限 TOP 100)
log("===== Phase 2a: CREATER (全量) =====")
creaters_raw = run_mysql("SELECT CREATER, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND CREATER!=1 GROUP BY CREATER ORDER BY COUNT(*) DESC;")
creaters = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in creaters_raw.strip().split("\n") if l.strip()]
log("CREATER 用户=%d 归档总量=%d" % (len(creaters), sum(c for _,c in creaters)))
for i, (uid, tot) in enumerate(creaters, 1):
log("--- CREATER #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
insert_user(uid, "DUMMY_B_CREATER_", "CREATER")
if i % 10 == 0:
dingtalk("INFO", "CREATER %d/100" % i, "已插入=%d" % total[0])
# Phase 2b — 全量业务用户(不限 TOP 100)
log("===== Phase 2b: LASTOPERATOR (全量) =====")
ops_raw = run_mysql("SELECT LASTOPERATOR, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND LASTOPERATOR!=1 GROUP BY LASTOPERATOR ORDER BY COUNT(*) DESC;")
ops = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in ops_raw.strip().split("\n") if l.strip()]
log("OPERATOR 用户=%d 归档总量=%d" % (len(ops), sum(c for _,c in ops)))
for i, (uid, tot) in enumerate(ops, 1):
log("--- OPERATOR #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
insert_user(uid, "DUMMY_B_OPERATOR_", "LASTOPERATOR")
if i % 10 == 0:
dingtalk("INFO", "OPERATOR %d/100" % i, "已插入=%d" % total[0])
# Phase 3
log("===== Phase 3: 验证 =====")
fc = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';")
log("DUMMY_B 总数 = %s" % fc)
http_ok = check_http(); log("Resin HTTP = %s" % http_ok)
tr2 = check_tr(); log("Threads_running = %d" % tr2)
lm = run_mysql("SELECT MAX(CREATETIME) FROM ecology_message_log WHERE CREATEDATE=CURDATE();")
log("message_log 最新 = %s" % lm)
dingtalk("P0", "DUMMY 补数据完成 ✅",
"**DUMMY_B**: %s 条\n**HTTP**: %s\n**TR**: %d\n**最新消息**: %s\n\n回滚: DELETE FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%%';" %
(fc, http_ok, tr2, lm))
log("===== 全部完成 =====")