fix: check-alert v3 修复钉钉推送断档 P0 假阳性
根因:sqlite_exec 吞掉 stderr(2>/dev/null),查询失败返回空串, 0 默认为 0 导致误报 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验, 查询失败跳过检测并 warn 日志
This commit is contained in:
1 parent
7ab75364b1
commit
20fa48fb34
17 files changed
+1747
-26
No files matched your search
Executable
+161
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""fix-info-read-dummy.py — 补 DUMMY 根治 updateBizState 死循环 (Python 3.6 兼容)"""
|
||||
import sys, os, time, subprocess, json, urllib.request
|
||||
|
||||
DB_HOST, DB_PORT, DB_USER = "39.100.72.102", 3306, "ecology"
|
||||
DB_PASS, DB_NAME = "Z!BYT+jSDDyYHMdX+qFSLzJCYXW", "ecology"
|
||||
OA_HOST, OA_USER, OA_PASS = "39.99.236.138", "root", "KTOFKjVkVOGlNgvs8yUP1+PM1VM"
|
||||
DINGTALK = "https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249"
|
||||
BATCH = 5000
|
||||
LOG = "/opt/oa-monitor/logs/fix-info-read.log"
|
||||
|
||||
# === 正确值(从之前 DUMMY_FIX_ 样例提取)===
|
||||
MESSAGETYPE = 25 # int!不是 'archive'
|
||||
BIZTYPE = 86 # int!上次成功用的
|
||||
STATE = 2 # 已读
|
||||
BIZSTATE = 1 # 已更新(关键!让 updateBizState 不再重试)
|
||||
GROUPTYPE = "archive-fix"
|
||||
BIZTITLE = "dummy fix"
|
||||
|
||||
def log(msg, level="INFO"):
|
||||
line = "[%s] [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S"), level, msg)
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def run_mysql(sql, fetch=True):
|
||||
env = os.environ.copy(); env["MYSQL_PWD"] = DB_PASS
|
||||
r = subprocess.run(["mysql","-h"+DB_HOST,"-P"+str(DB_PORT),"-u"+DB_USER,DB_NAME,"-N","-e",sql],
|
||||
env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=600)
|
||||
if r.returncode != 0:
|
||||
log("MySQL ERROR: " + r.stderr.decode().strip(), "ERROR")
|
||||
return None
|
||||
if not fetch: return True
|
||||
return r.stdout.decode().strip()
|
||||
|
||||
def dingtalk(level, title, body=""):
|
||||
text = "**[%s] %s**\n\n%s\n\n> OA DUMMY @ %s" % (level, title, body, time.strftime("%Y-%m-%d %H:%M:%S"))
|
||||
try:
|
||||
urllib.request.urlopen(urllib.request.Request(DINGTALK, json.dumps({"msgtype":"markdown","markdown":{"title":title,"text":text}}).encode(), {"Content-Type":"application/json"}), timeout=5)
|
||||
except Exception as e: log("钉钉: "+str(e), "WARN")
|
||||
|
||||
def check_http():
|
||||
env = os.environ.copy()
|
||||
cmd = ("python3 -c 'import paramiko; ssh=paramiko.SSHClient(); ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()); "
|
||||
"ssh.connect(\"%s\",username=\"%s\",password=\"%s\",timeout=15); "
|
||||
"_,o,_=ssh.exec_command(\"curl -s -o /dev/null -w %%{http_code} http://127.0.0.1:80/\"); "
|
||||
"print(o.read().decode(\"utf-8\",\"replace\")); ssh.close()'" % (OA_HOST, OA_USER, OA_PASS))
|
||||
r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=25, env=env)
|
||||
return "200" in r.stdout.decode().strip()
|
||||
|
||||
def check_tr():
|
||||
r = run_mysql("SHOW GLOBAL STATUS WHERE Variable_name='Threads_running';")
|
||||
if not r: return 999
|
||||
try: return int(r.split("\t")[-1].strip())
|
||||
except: return 999
|
||||
|
||||
def wait_safe(max_wait=120):
|
||||
for i in range(max_wait // 5):
|
||||
if check_http() and check_tr() < 30: return True
|
||||
log("等待恢复 (%ds) HTTP=%s TR=%d" % (i*5, check_http(), check_tr()), "WARN")
|
||||
time.sleep(5)
|
||||
return False
|
||||
|
||||
# === Phase 0 ===
|
||||
log("===== Phase 0: 前置检查 =====")
|
||||
DRY = "--dry-run" in sys.argv
|
||||
if DRY: log("*** DRY RUN ***")
|
||||
|
||||
ok = check_http(); log("Resin HTTP 80 = %s" % ok)
|
||||
if not ok: log("Resin 不通!中止!", "CRITICAL"); sys.exit(1)
|
||||
|
||||
tr = check_tr(); log("Threads_running = %d" % tr)
|
||||
if tr >= 15:
|
||||
wait_safe(180); tr = check_tr()
|
||||
|
||||
existing = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_FIX_%';")
|
||||
log("现有 DUMMY_FIX = %s" % existing)
|
||||
|
||||
log("Phase 1: 备份表...")
|
||||
run_mysql("CREATE TABLE IF NOT EXISTS ecology_message_info_read_bak_20261104 AS SELECT * FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';", fetch=False)
|
||||
|
||||
dingtalk("INFO", "DUMMY 补数据启动", "HTTP=200 ✅\nTR=%d\nDUMMY_FIX=%s" % (tr, existing))
|
||||
|
||||
# === Phase 2 ===
|
||||
total = [0]
|
||||
|
||||
def insert_user(uid, prefix, col):
|
||||
last_id = 0; batch = 0; user_total = 0
|
||||
while True:
|
||||
sql = ("INSERT INTO ecology_message_info_read "
|
||||
"(ID, MESSAGEID, MESSAGETYPE, USERID, TARGETID, STATE, BIZSTATE, "
|
||||
"CREATEDATE, CREATETIME, MESSAGEGROUPTYPE, BIZTITLE, BIZTYPE) "
|
||||
"SELECT CONCAT('%s%s_', REQUESTID), 0, %d, %d, REQUESTID, %d, %d, "
|
||||
"CURDATE(), CURTIME(), '%s', '%s', %d "
|
||||
"FROM workflow_requestbase "
|
||||
"WHERE %s = %d AND STATUS LIKE '%%归档%%' AND DELETED = 0 AND REQUESTID > %d "
|
||||
"ORDER BY REQUESTID LIMIT %d;" %
|
||||
(prefix, "%d", MESSAGETYPE, uid, STATE, BIZSTATE, GROUPTYPE, BIZTITLE, BIZTYPE,
|
||||
col, uid, last_id, BATCH))
|
||||
if DRY:
|
||||
batch += 1; user_total += BATCH; last_id += BATCH
|
||||
if batch >= 5: break
|
||||
time.sleep(0.2); continue
|
||||
|
||||
run_mysql(sql, fetch=False)
|
||||
rc = run_mysql("SELECT ROW_COUNT();")
|
||||
inserted = int(rc) if rc and rc.isdigit() else 0
|
||||
if inserted == 0: break
|
||||
|
||||
last_sql = "SELECT MAX(REQUESTID) FROM workflow_requestbase WHERE %s = %d AND REQUESTID > %d LIMIT %d;" % (col, uid, last_id, BATCH)
|
||||
lv = run_mysql(last_sql)
|
||||
if lv and lv.isdigit(): last_id = int(lv)
|
||||
|
||||
batch += 1; user_total += inserted; total[0] += inserted
|
||||
log(" User=%d batch=%d +%d rows (total=%d)" % (uid, batch, inserted, total[0]))
|
||||
|
||||
if not check_http() or check_tr() >= 30:
|
||||
log("异常!暂停...", "WARN"); wait_safe(10)
|
||||
time.sleep(1)
|
||||
|
||||
if user_total > 0:
|
||||
log("User=%d 完成: %d 条 %d 批" % (uid, user_total, batch))
|
||||
|
||||
# Phase 2a — 全量业务用户(不限 TOP 100)
|
||||
log("===== Phase 2a: CREATER (全量) =====")
|
||||
creaters_raw = run_mysql("SELECT CREATER, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND CREATER!=1 GROUP BY CREATER ORDER BY COUNT(*) DESC;")
|
||||
creaters = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in creaters_raw.strip().split("\n") if l.strip()]
|
||||
log("CREATER 用户=%d 归档总量=%d" % (len(creaters), sum(c for _,c in creaters)))
|
||||
|
||||
for i, (uid, tot) in enumerate(creaters, 1):
|
||||
log("--- CREATER #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
|
||||
insert_user(uid, "DUMMY_B_CREATER_", "CREATER")
|
||||
if i % 10 == 0:
|
||||
dingtalk("INFO", "CREATER %d/100" % i, "已插入=%d" % total[0])
|
||||
|
||||
# Phase 2b — 全量业务用户(不限 TOP 100)
|
||||
log("===== Phase 2b: LASTOPERATOR (全量) =====")
|
||||
ops_raw = run_mysql("SELECT LASTOPERATOR, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND LASTOPERATOR!=1 GROUP BY LASTOPERATOR ORDER BY COUNT(*) DESC;")
|
||||
ops = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in ops_raw.strip().split("\n") if l.strip()]
|
||||
log("OPERATOR 用户=%d 归档总量=%d" % (len(ops), sum(c for _,c in ops)))
|
||||
|
||||
for i, (uid, tot) in enumerate(ops, 1):
|
||||
log("--- OPERATOR #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
|
||||
insert_user(uid, "DUMMY_B_OPERATOR_", "LASTOPERATOR")
|
||||
if i % 10 == 0:
|
||||
dingtalk("INFO", "OPERATOR %d/100" % i, "已插入=%d" % total[0])
|
||||
|
||||
# Phase 3
|
||||
log("===== Phase 3: 验证 =====")
|
||||
fc = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';")
|
||||
log("DUMMY_B 总数 = %s" % fc)
|
||||
http_ok = check_http(); log("Resin HTTP = %s" % http_ok)
|
||||
tr2 = check_tr(); log("Threads_running = %d" % tr2)
|
||||
lm = run_mysql("SELECT MAX(CREATETIME) FROM ecology_message_log WHERE CREATEDATE=CURDATE();")
|
||||
log("message_log 最新 = %s" % lm)
|
||||
|
||||
dingtalk("P0", "DUMMY 补数据完成 ✅",
|
||||
"**DUMMY_B**: %s 条\n**HTTP**: %s\n**TR**: %d\n**最新消息**: %s\n\n回滚: DELETE FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%%';" %
|
||||
(fc, http_ok, tr2, lm))
|
||||
log("===== 全部完成 =====")
|
||||
Reference in new issue
Block a user