stop preventive restart cron, archive DUMMY_SYS script (根治完成)

This commit is contained in:
oa-monitor committed 2026-11-06 15:52:57 +08:00
1 parent 20fa48fb34
commit cb08965cf4
4 files changed
+173 -1

No files matched your search

Binary file not shown.
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env bash
# ============================================================================
# 补 userid=1 的 info_read DUMMY_SYS 数据(根治 updateBizState 死循环)
#
# 根因:userid=1 系统账号独占 374 万归档流程,info_read 清空后 updateBizState
# 扫到归档流程时 UPDATE 匹配 0 行 → 死循环占满 10 线程
#
# 策略:INSERT SELECT 批量补,每批 5000 条,事务控制,0.5s sleep
# 回滚:DELETE FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_SYS_%'
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
MYSQL="MYSQL_PWD='${DB_PASS}' timeout 60 mysql -h${DB_HOST} -P${DB_PORT} -uecology ecology"
BATCH_SIZE=5000
TOTAL=$(eval "${MYSQL} -N -e \"SELECT COUNT(*) FROM workflow_requestbase WHERE CREATER=1 AND DELETED=0;\"" 2>/dev/null)
echo "=== DUMMY_SYS 补数据开始 ==="
echo "目标总数: ${TOTAL}"
echo "每批大小: ${BATCH_SIZE}"
echo "总批次数: $(echo "scale=0; ${TOTAL}/${BATCH_SIZE} + 1" | bc)"
echo "批次间隔: 0.5s sleep"
echo "PID 文件: ${SCRIPT_DIR}/fix-info-read-sys.pid"
echo "日志文件: ${SCRIPT_DIR}/../logs/fix-info-read-sys.log"
echo "开始时间: $(date '+%Y-%m-%d %H:%M:%S')"
echo "============================"
# 写 PID
echo $$ > "${SCRIPT_DIR}/fix-info-read-sys.pid"
LOG="${SCRIPT_DIR}/../logs/fix-info-read-sys.log"
mkdir -p "$(dirname "${LOG}")"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOG}"; }
# --- Phase 0: 前置检查 ---
log "Phase 0: 前置检查"
# 0a. Resin HTTP 探活
HTTP_OK=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://39.99.236.138/api/workflow/getRequestList)
if [[ "${HTTP_OK}" != "200" ]]; then
log "❌ Resin HTTP=${HTTP_OK},不安全,中止"
exit 1
fi
log "✅ Resin HTTP=${HTTP_OK}"
# 0b. Threads_running 不能是死循环态
TR=$(eval "${MYSQL} -N -e \"SHOW GLOBAL STATUS LIKE 'Threads_running';\"" 2>/dev/null | awk '{print $2}')
if [[ -n "${TR}" ]] && [[ "${TR}" =~ ^[0-9]+$ ]] && [[ "${TR}" -gt 8 ]]; then
log "❌ Threads_running=${TR} 偏高,可能死循环,中止"
exit 1
fi
log "✅ Threads_running=${TR}"
# 0c. 磁盘剩余 > 5GB
DISK_AVAIL=$(df -BG / | grep -oP '\d+(?=G)' | tail -1)
if [[ -n "${DISK_AVAIL}" ]] && [[ "${DISK_AVAIL}" =~ ^[0-9]+$ ]] && [[ "${DISK_AVAIL}" -lt 5 ]]; then
log "❌ 磁盘剩余 ${DISK_AVAIL}GB < 5GB,中止"
exit 1
fi
log "✅ 磁盘剩余 ${DISK_AVAIL}GB"
# --- Phase 1: 分批 INSERT ---
log "Phase 1: 分批 INSERT(每批 ${BATCH_SIZE} 条)"
OFFSET=0
BATCH=0
START_EPOCH=$(date +%s)
while true; do
BATCH=$((BATCH + 1))
# DUMMY_SYS_ID 用 hash 前缀 + REQUESTID 保证唯一性(不能用 OFFSET 做 ID 因为会重复)
SQL="INSERT INTO ecology_message_info_read
(ID, MESSAGEID, MESSAGETYPE, USERTYPE, USERID, TARGETID, CREATER, BIZSTATE, STATE,
MESSAGEGROUPTYPE, BIZTITLE, BIZTYPE, CREATEDATE, CREATETIME, agentid)
SELECT
CONCAT('DUMMY_SYS_', REQUESTID) AS ID,
CONCAT('DUMMY_SYS_MSG_', REQUESTID) AS MESSAGEID,
25 AS MESSAGETYPE,
0 AS USERTYPE,
'1' AS USERID,
CAST(REQUESTID AS CHAR) AS TARGETID,
1 AS CREATER,
'1' AS BIZSTATE,
2 AS STATE,
'archive-fix' AS MESSAGEGROUPTYPE,
'dummy fix sys' AS BIZTITLE,
'86' AS BIZTYPE,
CURDATE() AS CREATEDATE,
CURTIME() AS CREATETIME,
'DUMMY_SYS' AS agentid
FROM (SELECT REQUESTID FROM workflow_requestbase WHERE CREATER=1 AND DELETED=0 ORDER BY REQUESTID LIMIT ${BATCH_SIZE} OFFSET ${OFFSET}) t
WHERE NOT EXISTS (SELECT 1 FROM ecology_message_info_read WHERE ID = CONCAT('DUMMY_SYS_', t.REQUESTID));
"
BATCH_START=$(date +%s)
RESULT=$(eval "${MYSQL} -e \"${SQL}\" 2>&1")
RC=$?
BATCH_ELAPSED=$(( $(date +%s) - BATCH_START ))
if [[ ${RC} -ne 0 ]]; then
log "❌ Batch #${BATCH} (OFFSET=${OFFSET}) SQL 错误:${RESULT}"
# 重试一次
log "重试中..."
sleep 2
RESULT=$(eval "${MYSQL} -e \"${SQL}\" 2>&1")
RC=$?
if [[ ${RC} -ne 0 ]]; then
log "❌ 重试仍然失败,中止"
exit 1
fi
fi
AFFECTED=$(echo "${RESULT}" | grep -oP '\d+' | tail -1)
AFFECTED=${AFFECTED:-0}
log "Batch #${BATCH} OFFSET=${OFFSET} +${AFFECTED} ${BATCH_ELAPSED}s"
OFFSET=$((OFFSET + BATCH_SIZE))
# 每 50 批做一次健康检查
if [[ $((BATCH % 50)) -eq 0 ]]; then
TR=$(eval "${MYSQL} -N -e \"SHOW GLOBAL STATUS LIKE 'Threads_running';\"" 2>/dev/null | awk '{print $2}')
LOG_COUNT=$(eval "${MYSQL} -N -e \"SELECT COUNT(*) FROM ecology_message_log WHERE CREATEDATE=CURDATE() AND CREATETIME >= SUBTIME(CURTIME(), INTERVAL 10 MINUTE);\"" 2>/dev/null)
log " 📊 健康检查: Threads_running=${TR} msg_log_10min=${LOG_COUNT}"
if [[ -n "${TR}" ]] && [[ "${TR}" =~ ^[0-9]+$ ]] && [[ "${TR}" -gt 8 ]]; then
log "❌ Threads_running=${TR} 异常,暂停 30s"
sleep 30
fi
fi
# 每 100 批休眠长一点
if [[ $((BATCH % 100)) -eq 0 ]]; then
sleep 2
else
sleep 0.5
fi
# 所有 requestid 都覆盖了就停
if [[ "${AFFECTED}" -lt "${BATCH_SIZE}" ]]; then
log "✅ 最后一批 ${AFFECTED} < ${BATCH_SIZE},全部覆盖完成"
break
fi
# 安全限制:防止死循环
if [[ ${OFFSET} -gt $((TOTAL + BATCH_SIZE * 2)) ]]; then
log "⚠️ OFFSET=${OFFSET} > TOTAL+2batch,强制停止"
break
fi
done
# --- Phase 2: 验证 ---
log "Phase 2: 验证"
SYS_COUNT=$(eval "${MYSQL} -N -e \"SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_SYS_%';\"" 2>/dev/null)
log "DUMMY_SYS 总数 = ${SYS_COUNT}"
ELAPSED=$(( $(date +%s) - START_EPOCH ))
log "总耗时: ${ELAPSED}s (约 $(echo "scale=1; ${ELAPSED}/60" | bc) 分钟)"
log "完成时间: $(date '+%Y-%m-%d %H:%M:%S')"
log "============================"
rm -f "${SCRIPT_DIR}/fix-info-read-sys.pid"
+7
View File
@@ -71,6 +71,13 @@ log_error() { log "ERROR" "$@"; }
dingtalk_alert() {
local level="$1" title="$2" body="$3"
local timestamp
# v2 降级(2026-11-05):只放行 P0,拦截 P1/P2 降噪
# 放行:watchdog count=0 自愈、DB 长查询、Docker 挂、推送断档
# 拦截:PID 异常、预防性重启、HTTP 故障、磁盘告警
if [[ "${level}" != "P0" ]]; then
log_info "[ALERT-LEVEL] 降级模式:拦截 ${level} ${title}"
return 0
fi
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
# 告警冷却检查
+1 -1
View File
@@ -1 +1 @@
30952 16490 1793792402
1807 49791 1793951402