From cb08965cf4fdf607c56219f31cd6f3592ef7db68 Mon Sep 17 00:00:00 2001 From: oa-monitor Date: Fri, 6 Nov 2026 15:52:57 +0800 Subject: [PATCH] =?UTF-8?q?stop=20preventive=20restart=20cron,=20archive?= =?UTF-8?q?=20DUMMY=5FSYS=20script=20(=E6=A0=B9=E6=B2=BB=E5=AE=8C=E6=88=90?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fix-info-read-sys.cpython-36.pyc | Bin 0 -> 6959 bytes bin/fix-info-read-sys.sh | 165 ++++++++++++++++++ lib/common.sh | 7 + state/resin_pid.last | 2 +- 4 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 bin/__pycache__/fix-info-read-sys.cpython-36.pyc create mode 100755 bin/fix-info-read-sys.sh diff --git a/bin/__pycache__/fix-info-read-sys.cpython-36.pyc b/bin/__pycache__/fix-info-read-sys.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..41fc38e366c18b7c77335b1e8a3a6e3e8e5d86d0 GIT binary patch literal 6959 zcmbVRU342omhNt~ek}jQaYAr_fhIrLa{M3LaU46~#7biDPq1VXHv?K(RkkI|lGEKv zVvTc7g2<9UJju-PpLl^8SORAQ%p4X%e&A&v_Ib}f>5;W$d(PRDdDy4C?0&Z-%Z`D4 z80&OZ^{uK~x9+|5-COP5rKRpYeg7S~_A5#HFKNwR8QRa_G5Ltvjm$D+gjJfpiJ~^X+_FN{&G*pj5CbzKvC1 zRp%8k^P!Y{O@9h}YWY)F zr5yKI3DzW9U7k}tI92f~%=kCVugXhRjS&sJnxtZldFfO!qk=W@ZM?ctS|f1+}Ch%FnYPtP2v+Brh*2BCQ z*F+;QriC5iEm*&E-ln(m&Ab(~Hr{r{ChU2b*P!o6K}&mBuh8lGanB0s!yFz14_)=C z?YxNvwvsIEIT?6!rKIoWyLbn0O3Bc`F5WJz`y4bvw(H|hiMl^GLbM0gX>pM6Wk)gg z`T5cZ)~#W^TX2pil`E>z$;m5KvlChz5~J#vPwd9OW;es4XLaNY;(ZG{_TY;1IZyiO z{*=fbcAVy1QBLeQCcz(trn#q*_Ph_)C2u7e)}G%H-nru4=+XH;==epxkNx^lN$l&T zoK116yLcIKpLq-#+s0n5l;C@+@xACNr=DX{tVEXRHzLK9F&Mt zO^uE9dzw)-hyPk4NxGAV;?el{O!86v$pa()T@hbT&&0l_tO*3G{CwZ(ZG#YAZ+8u1& z8*B@; zu{ZN)#)6!zT|d^ef)@Xo!4Wea3y}&}{uDG0;xWrnSW-la$N*S{;Q&G1%0P1C&XKyE zQ*}F;w)05G&i;;_K94Q!VqkBDr+C^Hi=Wb6re%!L>t~W-2Om+~uU;?IFYI;3K&+BB zk`Q<#@fL!{#2?kfQpor?8WtSEf{bu1WftYU%4|%XQ)CnUef#?Ic*<{ zCd>(slD1D9;g}^jgKV2J$5-Vv$S`Re)L4N=PK08#3L=nnE<8en2{fQ4nNanNY}saI zCU-&0vHbG-p3!o3%xXnS0V^ zyi8lbTS=reMVl!eNEh{w_@3_z96i>f+oxvC6VaqnH-YCUo&=r&JlyY5(>5MEnRb{K zizh5mbHhkiBx2!{+%SXDK#ZS?hGX0`NTZ&zv=TSdPRt1#@mN|7#iwV|&YlB-quoBA zXdD^z`E!)Ppg#USGKRb@rWavTwh(aQmC=#ZQ+$x}?Fp_wHO+e(MKnY0Kw+$ef?cTz)0{)>oiq zZvFG}dsi|SzPWejVp=|xRO_L|LpXuz6f_JwC&qvtX$Rv`ZgB%JC?$&vF4I+&bXRFY zqZ+)%ronAcB<{-^>Q2qSd>Rux4j5^IM|0K@Rx>t$ZV)7-opi3@Q^IcsJQZn|#iypn z!cneQ3jW2tS%eW33-CI~`&=uH!|ZEA%ioE1px&NV8JJ7Ex0YE5QNo zV5P9Xox%3xc}5g|;9wh&2v%%`x6LXMXG$`nmTMO2CEGZu8c~|T-0+c233BYEAKuJd z{U^wKiLpVM2U1NS+est z6yKmH#?K_(b#+t0({%!<<B>f z#1LDn_T86@mI-o}2|}0u7k{3jH9jCLO^4jTRYnqL8{Rk_4O?kv*ks{xG;F|bNYaP_ zx?&l+!!iOVgVFVdFAh!c&}0C(5{^}2tVym#mUqeJav6}rC986oyqPeh13eF52qC_( z{mW>~^Z`MnC;>dsgRo>)V(K*@#WmOsb_9|{0Q9gYP=tIVEHf9-2H^lb&RH8PVs45_ zKS^S8ur3Z$+BFq4CRzMcELryAVE0Q}&v1YLNWeGZqljgNEgUD+;-@W-qB{|?teI(U z?1Om8l7qQpQ0E6JSst8*zCuAO9FH|nfX8$d0XnEVCtwS}|1w2Tk8F6Ta_MDoplEos z-pE-xtdw3)q)U2w2M+tY`<@F75A~%TT!6do1e5qQk9kys&W-NK zG1b-4IGfRJ;IiI6u9akKH^%3IU4c=y$dAaIWrt++fL<^PB_@RfS%Vi#vH$>>F0Ko4^8LMBCk1BA$DQiKppp0qsKxbn*lnP0UEI%6IRxijXu2Hm%EmmQ76dNk4^+l zt~GXOnTvCacfZK~>1_6cPY_j3wrC4?-pyQkd-380L=d5sq_eJWEN+B&f_(nR`Ab^n z?jJBuOV9>88XL9jyI*F${bb?pHH50=cfJMyW#9Py;_dldz`A$mT;}rUOR2kBV>3qA zHMY^5rEh;{w2(M-mVdlUf6S}t4Zq~=+SyG+wbk^ zYTTC~8)WYOKKs$1vj2K+@%kItv$qjYJGA?9ZGz$oZF`7-LR+{wSHRhYn>VsI{+OV+ zM_b4Z$n_>i^Ro4KAFn>x?ekaryN3>Y{k=U}w@(`$@bw-Z@b-A}BIj=+fhlB1!Eqj- z>>lC=Szd0;%E4vf$Om>-?MrOO!hieV%~!ST+cy?p`EK#{4XE|ro%b`}{v~_uqL%mO zt^|eEzkTqJZ)utH*RyYbmAP{^^XVJO=YCduf2Szq8(22;#ii_pD;<8kn#~45 zt<`d#leu#X`Az2BHF!YIB`<%aWiEWR^!@i*f(}013ocj~1`C1l!VmM=*FRo-?{4Pf z&lYaJpLzWwnob$TkC+SFL zL(+YTI@%zeE}`#SvatBsEz-~V>&vO@IeX!V7JOx8#;QsR_9+Z@*VN<=NKK6}^)4-; zkxH^3ewY2rZ7nfP*S!Pbq>pZuTK2~6%=e!!{`SiUu$mz4ARd!ujsx^wsuan$gzF@( zYle!@i1zQ46RIzVvVdI?^V=~2Axo)mOv|U&q2PUYXB3GDZs*hc&k%P)S%eBds}E7a!J z+Q!HRPCXbzqg-S ztE*Ib=q8G_2T(fz$WuWCdSeK(8!u4tU?93*)H%*${PeW3KSg^aJ*bLt998mW?*Oi5 zeyw-FKdAlkd9Uv6sqOdreBFn=t1bV?QEx5X>lI#VJvZf*^bOwYjA)-&(jmm+OSv=g#Tk>)&M0o?AY9HuKu8q%+Sg>DCI63H;J_ zC0&|k#qqhPE2(IJwECv8v}_8D`T2N|eg75UlD50i%&%NuU!T}Rvb_J{y_=WUz|2o$ z$?Dt{7{vbQkN~Ha=|hG<9a?wc)}`#fe3XmX|IfY>^?4-}ToB4J42N`9D2Sg$EdWLDyZNm=>-n24Rry5}=72lkQ;g7!RDlzd@)D#Vp)Dt)O8A_;frp zq1#0O)62p!+;J_=@SPA|r6YvIrvt$;T*UE@k^Gm%01KPJ(I{uSGfelQ;3=bs1S2=n zZ6N8zf*a+ShJ{=_9*(6QxZdCrW{{uYOU$_*tEOLI*g4NjY!;rkWRTW9`F zFCGiW@XaR_PvEYMFD+AXd<9ws6wwl&y66&{=i#1BX N?=f&tP?v${{{d(n=okP1 literal 0 HcmV?d00001 diff --git a/bin/fix-info-read-sys.sh b/bin/fix-info-read-sys.sh new file mode 100755 index 0000000..906e191 --- /dev/null +++ b/bin/fix-info-read-sys.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# ============================================================================ +# 补 userid=1 的 info_read DUMMY_SYS 数据(根治 updateBizState 死循环) +# +# 根因:userid=1 系统账号独占 374 万归档流程,info_read 清空后 updateBizState +# 扫到归档流程时 UPDATE 匹配 0 行 → 死循环占满 10 线程 +# +# 策略:INSERT SELECT 批量补,每批 5000 条,事务控制,0.5s sleep +# 回滚:DELETE FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_SYS_%' +# ============================================================================ +set -u + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +MYSQL="MYSQL_PWD='${DB_PASS}' timeout 60 mysql -h${DB_HOST} -P${DB_PORT} -uecology ecology" + +BATCH_SIZE=5000 +TOTAL=$(eval "${MYSQL} -N -e \"SELECT COUNT(*) FROM workflow_requestbase WHERE CREATER=1 AND DELETED=0;\"" 2>/dev/null) +echo "=== DUMMY_SYS 补数据开始 ===" +echo "目标总数: ${TOTAL}" +echo "每批大小: ${BATCH_SIZE}" +echo "总批次数: $(echo "scale=0; ${TOTAL}/${BATCH_SIZE} + 1" | bc)" +echo "批次间隔: 0.5s sleep" +echo "PID 文件: ${SCRIPT_DIR}/fix-info-read-sys.pid" +echo "日志文件: ${SCRIPT_DIR}/../logs/fix-info-read-sys.log" +echo "开始时间: $(date '+%Y-%m-%d %H:%M:%S')" +echo "============================" + +# 写 PID +echo $$ > "${SCRIPT_DIR}/fix-info-read-sys.pid" + +LOG="${SCRIPT_DIR}/../logs/fix-info-read-sys.log" +mkdir -p "$(dirname "${LOG}")" + +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOG}"; } + +# --- Phase 0: 前置检查 --- +log "Phase 0: 前置检查" + +# 0a. Resin HTTP 探活 +HTTP_OK=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://39.99.236.138/api/workflow/getRequestList) +if [[ "${HTTP_OK}" != "200" ]]; then + log "❌ Resin HTTP=${HTTP_OK},不安全,中止" + exit 1 +fi +log "✅ Resin HTTP=${HTTP_OK}" + +# 0b. Threads_running 不能是死循环态 +TR=$(eval "${MYSQL} -N -e \"SHOW GLOBAL STATUS LIKE 'Threads_running';\"" 2>/dev/null | awk '{print $2}') +if [[ -n "${TR}" ]] && [[ "${TR}" =~ ^[0-9]+$ ]] && [[ "${TR}" -gt 8 ]]; then + log "❌ Threads_running=${TR} 偏高,可能死循环,中止" + exit 1 +fi +log "✅ Threads_running=${TR}" + +# 0c. 磁盘剩余 > 5GB +DISK_AVAIL=$(df -BG / | grep -oP '\d+(?=G)' | tail -1) +if [[ -n "${DISK_AVAIL}" ]] && [[ "${DISK_AVAIL}" =~ ^[0-9]+$ ]] && [[ "${DISK_AVAIL}" -lt 5 ]]; then + log "❌ 磁盘剩余 ${DISK_AVAIL}GB < 5GB,中止" + exit 1 +fi +log "✅ 磁盘剩余 ${DISK_AVAIL}GB" + +# --- Phase 1: 分批 INSERT --- +log "Phase 1: 分批 INSERT(每批 ${BATCH_SIZE} 条)" +OFFSET=0 +BATCH=0 +START_EPOCH=$(date +%s) + +while true; do + BATCH=$((BATCH + 1)) + + # DUMMY_SYS_ID 用 hash 前缀 + REQUESTID 保证唯一性(不能用 OFFSET 做 ID 因为会重复) + SQL="INSERT INTO ecology_message_info_read + (ID, MESSAGEID, MESSAGETYPE, USERTYPE, USERID, TARGETID, CREATER, BIZSTATE, STATE, + MESSAGEGROUPTYPE, BIZTITLE, BIZTYPE, CREATEDATE, CREATETIME, agentid) + SELECT + CONCAT('DUMMY_SYS_', REQUESTID) AS ID, + CONCAT('DUMMY_SYS_MSG_', REQUESTID) AS MESSAGEID, + 25 AS MESSAGETYPE, + 0 AS USERTYPE, + '1' AS USERID, + CAST(REQUESTID AS CHAR) AS TARGETID, + 1 AS CREATER, + '1' AS BIZSTATE, + 2 AS STATE, + 'archive-fix' AS MESSAGEGROUPTYPE, + 'dummy fix sys' AS BIZTITLE, + '86' AS BIZTYPE, + CURDATE() AS CREATEDATE, + CURTIME() AS CREATETIME, + 'DUMMY_SYS' AS agentid + FROM (SELECT REQUESTID FROM workflow_requestbase WHERE CREATER=1 AND DELETED=0 ORDER BY REQUESTID LIMIT ${BATCH_SIZE} OFFSET ${OFFSET}) t + WHERE NOT EXISTS (SELECT 1 FROM ecology_message_info_read WHERE ID = CONCAT('DUMMY_SYS_', t.REQUESTID)); +" + + BATCH_START=$(date +%s) + RESULT=$(eval "${MYSQL} -e \"${SQL}\" 2>&1") + RC=$? + BATCH_ELAPSED=$(( $(date +%s) - BATCH_START )) + + if [[ ${RC} -ne 0 ]]; then + log "❌ Batch #${BATCH} (OFFSET=${OFFSET}) SQL 错误:${RESULT}" + # 重试一次 + log "重试中..." + sleep 2 + RESULT=$(eval "${MYSQL} -e \"${SQL}\" 2>&1") + RC=$? + if [[ ${RC} -ne 0 ]]; then + log "❌ 重试仍然失败,中止" + exit 1 + fi + fi + + AFFECTED=$(echo "${RESULT}" | grep -oP '\d+' | tail -1) + AFFECTED=${AFFECTED:-0} + + log "Batch #${BATCH} OFFSET=${OFFSET} +${AFFECTED} ${BATCH_ELAPSED}s" + + OFFSET=$((OFFSET + BATCH_SIZE)) + + # 每 50 批做一次健康检查 + if [[ $((BATCH % 50)) -eq 0 ]]; then + TR=$(eval "${MYSQL} -N -e \"SHOW GLOBAL STATUS LIKE 'Threads_running';\"" 2>/dev/null | awk '{print $2}') + LOG_COUNT=$(eval "${MYSQL} -N -e \"SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= SUBTIME(CURTIME(), INTERVAL 10 MINUTE);\"" 2>/dev/null) + log " 📊 健康检查: Threads_running=${TR} msg_log_10min=${LOG_COUNT}" + + if [[ -n "${TR}" ]] && [[ "${TR}" =~ ^[0-9]+$ ]] && [[ "${TR}" -gt 8 ]]; then + log "❌ Threads_running=${TR} 异常,暂停 30s" + sleep 30 + fi + fi + + # 每 100 批休眠长一点 + if [[ $((BATCH % 100)) -eq 0 ]]; then + sleep 2 + else + sleep 0.5 + fi + + # 所有 requestid 都覆盖了就停 + if [[ "${AFFECTED}" -lt "${BATCH_SIZE}" ]]; then + log "✅ 最后一批 ${AFFECTED} < ${BATCH_SIZE},全部覆盖完成" + break + fi + + # 安全限制:防止死循环 + if [[ ${OFFSET} -gt $((TOTAL + BATCH_SIZE * 2)) ]]; then + log "⚠️ OFFSET=${OFFSET} > TOTAL+2batch,强制停止" + break + fi +done + +# --- Phase 2: 验证 --- +log "Phase 2: 验证" +SYS_COUNT=$(eval "${MYSQL} -N -e \"SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_SYS_%';\"" 2>/dev/null) +log "DUMMY_SYS 总数 = ${SYS_COUNT}" + +ELAPSED=$(( $(date +%s) - START_EPOCH )) +log "总耗时: ${ELAPSED}s (约 $(echo "scale=1; ${ELAPSED}/60" | bc) 分钟)" +log "完成时间: $(date '+%Y-%m-%d %H:%M:%S')" +log "============================" + +rm -f "${SCRIPT_DIR}/fix-info-read-sys.pid" diff --git a/lib/common.sh b/lib/common.sh index a26fbdc..0d3bad0 100755 --- a/lib/common.sh +++ b/lib/common.sh @@ -71,6 +71,13 @@ log_error() { log "ERROR" "$@"; } dingtalk_alert() { local level="$1" title="$2" body="$3" local timestamp + # v2 降级(2026-11-05):只放行 P0,拦截 P1/P2 降噪 + # 放行:watchdog count=0 自愈、DB 长查询、Docker 挂、推送断档 + # 拦截:PID 异常、预防性重启、HTTP 故障、磁盘告警 + if [[ "${level}" != "P0" ]]; then + log_info "[ALERT-LEVEL] 降级模式:拦截 ${level} ${title}" + return 0 + fi timestamp=$(date '+%Y-%m-%d %H:%M:%S') # 告警冷却检查 diff --git a/state/resin_pid.last b/state/resin_pid.last index 343a78a..fd948c5 100644 --- a/state/resin_pid.last +++ b/state/resin_pid.last @@ -1 +1 @@ -30952 16490 1793792402 +1807 49791 1793951402