diff --git a/bin/__pycache__/fix-info-read-sys.cpython-36.pyc b/bin/__pycache__/fix-info-read-sys.cpython-36.pyc new file mode 100644 index 0000000..41fc38e Binary files /dev/null and b/bin/__pycache__/fix-info-read-sys.cpython-36.pyc differ diff --git a/bin/fix-info-read-sys.sh b/bin/fix-info-read-sys.sh new file mode 100755 index 0000000..906e191 --- /dev/null +++ b/bin/fix-info-read-sys.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# ============================================================================ +# 补 userid=1 的 info_read DUMMY_SYS 数据(根治 updateBizState 死循环) +# +# 根因:userid=1 系统账号独占 374 万归档流程,info_read 清空后 updateBizState +# 扫到归档流程时 UPDATE 匹配 0 行 → 死循环占满 10 线程 +# +# 策略:INSERT SELECT 批量补,每批 5000 条,事务控制,0.5s sleep +# 回滚:DELETE FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_SYS_%' +# ============================================================================ +set -u + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +MYSQL="MYSQL_PWD='${DB_PASS}' timeout 60 mysql -h${DB_HOST} -P${DB_PORT} -uecology ecology" + +BATCH_SIZE=5000 +TOTAL=$(eval "${MYSQL} -N -e \"SELECT COUNT(*) FROM workflow_requestbase WHERE CREATER=1 AND DELETED=0;\"" 2>/dev/null) +echo "=== DUMMY_SYS 补数据开始 ===" +echo "目标总数: ${TOTAL}" +echo "每批大小: ${BATCH_SIZE}" +echo "总批次数: $(echo "scale=0; ${TOTAL}/${BATCH_SIZE} + 1" | bc)" +echo "批次间隔: 0.5s sleep" +echo "PID 文件: ${SCRIPT_DIR}/fix-info-read-sys.pid" +echo "日志文件: ${SCRIPT_DIR}/../logs/fix-info-read-sys.log" +echo "开始时间: $(date '+%Y-%m-%d %H:%M:%S')" +echo "============================" + +# 写 PID +echo $$ > "${SCRIPT_DIR}/fix-info-read-sys.pid" + +LOG="${SCRIPT_DIR}/../logs/fix-info-read-sys.log" +mkdir -p "$(dirname "${LOG}")" + +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOG}"; } + +# --- Phase 0: 前置检查 --- +log "Phase 0: 前置检查" + +# 0a. Resin HTTP 探活 +HTTP_OK=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://39.99.236.138/api/workflow/getRequestList) +if [[ "${HTTP_OK}" != "200" ]]; then + log "❌ Resin HTTP=${HTTP_OK},不安全,中止" + exit 1 +fi +log "✅ Resin HTTP=${HTTP_OK}" + +# 0b. Threads_running 不能是死循环态 +TR=$(eval "${MYSQL} -N -e \"SHOW GLOBAL STATUS LIKE 'Threads_running';\"" 2>/dev/null | awk '{print $2}') +if [[ -n "${TR}" ]] && [[ "${TR}" =~ ^[0-9]+$ ]] && [[ "${TR}" -gt 8 ]]; then + log "❌ Threads_running=${TR} 偏高,可能死循环,中止" + exit 1 +fi +log "✅ Threads_running=${TR}" + +# 0c. 磁盘剩余 > 5GB +DISK_AVAIL=$(df -BG / | grep -oP '\d+(?=G)' | tail -1) +if [[ -n "${DISK_AVAIL}" ]] && [[ "${DISK_AVAIL}" =~ ^[0-9]+$ ]] && [[ "${DISK_AVAIL}" -lt 5 ]]; then + log "❌ 磁盘剩余 ${DISK_AVAIL}GB < 5GB,中止" + exit 1 +fi +log "✅ 磁盘剩余 ${DISK_AVAIL}GB" + +# --- Phase 1: 分批 INSERT --- +log "Phase 1: 分批 INSERT(每批 ${BATCH_SIZE} 条)" +OFFSET=0 +BATCH=0 +START_EPOCH=$(date +%s) + +while true; do + BATCH=$((BATCH + 1)) + + # DUMMY_SYS_ID 用 hash 前缀 + REQUESTID 保证唯一性(不能用 OFFSET 做 ID 因为会重复) + SQL="INSERT INTO ecology_message_info_read + (ID, MESSAGEID, MESSAGETYPE, USERTYPE, USERID, TARGETID, CREATER, BIZSTATE, STATE, + MESSAGEGROUPTYPE, BIZTITLE, BIZTYPE, CREATEDATE, CREATETIME, agentid) + SELECT + CONCAT('DUMMY_SYS_', REQUESTID) AS ID, + CONCAT('DUMMY_SYS_MSG_', REQUESTID) AS MESSAGEID, + 25 AS MESSAGETYPE, + 0 AS USERTYPE, + '1' AS USERID, + CAST(REQUESTID AS CHAR) AS TARGETID, + 1 AS CREATER, + '1' AS BIZSTATE, + 2 AS STATE, + 'archive-fix' AS MESSAGEGROUPTYPE, + 'dummy fix sys' AS BIZTITLE, + '86' AS BIZTYPE, + CURDATE() AS CREATEDATE, + CURTIME() AS CREATETIME, + 'DUMMY_SYS' AS agentid + FROM (SELECT REQUESTID FROM workflow_requestbase WHERE CREATER=1 AND DELETED=0 ORDER BY REQUESTID LIMIT ${BATCH_SIZE} OFFSET ${OFFSET}) t + WHERE NOT EXISTS (SELECT 1 FROM ecology_message_info_read WHERE ID = CONCAT('DUMMY_SYS_', t.REQUESTID)); +" + + BATCH_START=$(date +%s) + RESULT=$(eval "${MYSQL} -e \"${SQL}\" 2>&1") + RC=$? + BATCH_ELAPSED=$(( $(date +%s) - BATCH_START )) + + if [[ ${RC} -ne 0 ]]; then + log "❌ Batch #${BATCH} (OFFSET=${OFFSET}) SQL 错误:${RESULT}" + # 重试一次 + log "重试中..." + sleep 2 + RESULT=$(eval "${MYSQL} -e \"${SQL}\" 2>&1") + RC=$? + if [[ ${RC} -ne 0 ]]; then + log "❌ 重试仍然失败,中止" + exit 1 + fi + fi + + AFFECTED=$(echo "${RESULT}" | grep -oP '\d+' | tail -1) + AFFECTED=${AFFECTED:-0} + + log "Batch #${BATCH} OFFSET=${OFFSET} +${AFFECTED} ${BATCH_ELAPSED}s" + + OFFSET=$((OFFSET + BATCH_SIZE)) + + # 每 50 批做一次健康检查 + if [[ $((BATCH % 50)) -eq 0 ]]; then + TR=$(eval "${MYSQL} -N -e \"SHOW GLOBAL STATUS LIKE 'Threads_running';\"" 2>/dev/null | awk '{print $2}') + LOG_COUNT=$(eval "${MYSQL} -N -e \"SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= SUBTIME(CURTIME(), INTERVAL 10 MINUTE);\"" 2>/dev/null) + log " 📊 健康检查: Threads_running=${TR} msg_log_10min=${LOG_COUNT}" + + if [[ -n "${TR}" ]] && [[ "${TR}" =~ ^[0-9]+$ ]] && [[ "${TR}" -gt 8 ]]; then + log "❌ Threads_running=${TR} 异常,暂停 30s" + sleep 30 + fi + fi + + # 每 100 批休眠长一点 + if [[ $((BATCH % 100)) -eq 0 ]]; then + sleep 2 + else + sleep 0.5 + fi + + # 所有 requestid 都覆盖了就停 + if [[ "${AFFECTED}" -lt "${BATCH_SIZE}" ]]; then + log "✅ 最后一批 ${AFFECTED} < ${BATCH_SIZE},全部覆盖完成" + break + fi + + # 安全限制:防止死循环 + if [[ ${OFFSET} -gt $((TOTAL + BATCH_SIZE * 2)) ]]; then + log "⚠️ OFFSET=${OFFSET} > TOTAL+2batch,强制停止" + break + fi +done + +# --- Phase 2: 验证 --- +log "Phase 2: 验证" +SYS_COUNT=$(eval "${MYSQL} -N -e \"SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_SYS_%';\"" 2>/dev/null) +log "DUMMY_SYS 总数 = ${SYS_COUNT}" + +ELAPSED=$(( $(date +%s) - START_EPOCH )) +log "总耗时: ${ELAPSED}s (约 $(echo "scale=1; ${ELAPSED}/60" | bc) 分钟)" +log "完成时间: $(date '+%Y-%m-%d %H:%M:%S')" +log "============================" + +rm -f "${SCRIPT_DIR}/fix-info-read-sys.pid" diff --git a/lib/common.sh b/lib/common.sh index a26fbdc..0d3bad0 100755 --- a/lib/common.sh +++ b/lib/common.sh @@ -71,6 +71,13 @@ log_error() { log "ERROR" "$@"; } dingtalk_alert() { local level="$1" title="$2" body="$3" local timestamp + # v2 降级(2026-11-05):只放行 P0,拦截 P1/P2 降噪 + # 放行:watchdog count=0 自愈、DB 长查询、Docker 挂、推送断档 + # 拦截:PID 异常、预防性重启、HTTP 故障、磁盘告警 + if [[ "${level}" != "P0" ]]; then + log_info "[ALERT-LEVEL] 降级模式:拦截 ${level} ${title}" + return 0 + fi timestamp=$(date '+%Y-%m-%d %H:%M:%S') # 告警冷却检查 diff --git a/state/resin_pid.last b/state/resin_pid.last index 343a78a..fd948c5 100644 --- a/state/resin_pid.last +++ b/state/resin_pid.last @@ -1 +1 @@ -30952 16490 1793792402 +1807 49791 1793951402