fix: check-alert v3 修复钉钉推送断档 P0 假阳性

根因:sqlite_exec 吞掉 stderr(2>/dev/null),查询失败返回空串,
0 默认为 0 导致误报

修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,
查询失败跳过检测并 warn 日志
This commit is contained in:
oa-monitor committed 2026-11-04 19:41:18 +08:00
1 parent 7ab75364b1
commit 20fa48fb34
17 files changed
+1728 -7

No files matched your search

Executable
+281
View File
@@ -0,0 +1,281 @@
#!/usr/bin/env python3
# ============================================================================
# OA Monitor · Python HTTP API(纯标准库,无第三方依赖)
# 唯一职责:读 SQLite → 吐 JSON + 静态 Dashboard
# 只读,绝不写 SQLite;所有 POST/PUT/DELETE 返回 405
# ============================================================================
import http.server, json, sqlite3, os, sys, urllib.parse, base64
from datetime import datetime, timezone
DB_PATH = "/opt/oa-monitor/data/oa-monitor.db"
DASHBOARD_DIR = "/opt/oa-monitor/dashboard"
# ---- HTTP Basic Auth ----
# 只允许 oamonitor 用户
AUTH_USER = "oamonitor"
AUTH_PASS = "1mcAi/rCtrZZkz3To2BD"
AUTH_REALM = "OA Monitor"
def check_auth(headers):
"""检查 Authorization 头,返回 bool"""
auth = headers.get('Authorization', '')
if not auth.startswith('Basic '):
return False
try:
decoded = base64.b64decode(auth[6:]).decode()
user, _, pwd = decoded.partition(':')
return user == AUTH_USER and pwd == AUTH_PASS
except Exception:
return False
def send_401(handler):
body = b'Unauthorized: OA Monitor - HTTP Basic Auth required (user: oamonitor)'
handler.send_response(401)
handler.send_header('WWW-Authenticate', f'Basic realm="{AUTH_REALM}"')
handler.send_header('Content-Type', 'text/plain; charset=utf-8')
handler.send_header('Content-Length', str(len(body)))
handler.send_header('Access-Control-Allow-Origin', '*')
handler.end_headers()
handler.wfile.write(body)
def get_db():
return sqlite3.connect(DB_PATH, timeout=5)
def ts_to_iso(ts):
if not ts: return None
return datetime.fromtimestamp(int(ts)).strftime('%Y-%m-%d %H:%M:%S')
def fetch_all(sql, params=()):
db = get_db()
db.row_factory = sqlite3.Row
try:
rows = db.execute(sql, params).fetchall()
return [dict(r) for r in rows]
finally:
db.close()
def fetch_one(sql, params=()):
rows = fetch_all(sql, params)
return rows[0] if rows else None
class MonitorHandler(http.server.BaseHTTPRequestHandler):
# ---- CORS + JSON 头 ----
def _send_json(self, data, code=200):
body = json.dumps(data, ensure_ascii=False, default=str).encode()
self.send_response(code)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.send_header('Content-Length', str(len(body)))
self.send_header('Access-Control-Allow-Origin', '*')
self.send_header('Access-Control-Allow-Methods', 'GET, OPTIONS')
self.send_header('Access-Control-Allow-Headers', 'Content-Type')
self.end_headers()
self.wfile.write(body)
def do_OPTIONS(self):
self._send_json({})
# ---- 路由 ----
def do_GET(self):
# HTTP Basic Auth(全路径保护,包括 Dashboard 静态文件 + API)
if not check_auth(self.headers):
send_401(self)
return
parsed = urllib.parse.urlparse(self.path)
path = parsed.path.rstrip('/')
qs = urllib.parse.parse_qs(parsed.query)
try:
if path.startswith('/api/'):
self._route_api(path, qs)
else:
self._route_static(path)
except Exception as e:
self._send_json({"error": str(e)}, 500)
def _route_api(self, path, qs):
if path == '/api/health':
self._health()
elif path == '/api/overview':
self._overview()
elif path == '/api/db/recent':
self._db_recent(qs)
elif path == '/api/db/long_queries':
self._db_long_queries(qs)
elif path == '/api/http/recent':
self._http_recent(qs)
elif path == '/api/business/recent':
self._biz_recent(qs)
elif path == '/api/infra/recent':
self._infra_recent(qs)
elif path == '/api/alerts':
self._alerts(qs)
else:
self._send_json({"error": "not found", "path": path}, 404)
# ---- 静态文件服务 ----
def _route_static(self, path):
# 安全:禁止 .. 和 /etc 等
if '..' in path or path.startswith('/etc'):
self._send_json({"error": "forbidden"}, 403); return
# 根路径 → index.html
if path == '' or path == '/':
path = '/index.html'
filepath = os.path.join(DASHBOARD_DIR, path.lstrip('/'))
# 双重安全
if not filepath.startswith(DASHBOARD_DIR):
self._send_json({"error": "forbidden"}, 403); return
if not os.path.isfile(filepath):
self._send_json({"error": "not found", "path": path}, 404); return
with open(filepath, 'rb') as f:
content = f.read()
ext = os.path.splitext(filepath)[1].lower()
mime = {
'.html': 'text/html; charset=utf-8',
'.css': 'text/css; charset=utf-8',
'.js': 'application/javascript; charset=utf-8',
'.json': 'application/json; charset=utf-8',
'.svg': 'image/svg+xml',
'.png': 'image/png',
'.ico': 'image/x-icon',
}.get(ext, 'application/octet-stream')
self.send_response(200)
self.send_header('Content-Type', mime)
self.send_header('Content-Length', str(len(content)))
self.send_header('Access-Control-Allow-Origin', '*')
self.send_header('Cache-Control', 'no-cache')
self.end_headers()
self.wfile.write(content)
def do_POST(self): self._send_json({"error": "read-only"}, 405)
def do_PUT(self): self._send_json({"error": "read-only"}, 405)
def do_DELETE(self): self._send_json({"error": "read-only"}, 405)
def log_message(self, fmt, *args):
pass # 静默日志
# ---- Endpoint 实现 ----
def _health(self):
db_ok = os.path.exists(DB_PATH)
db_size = os.path.getsize(DB_PATH) if db_ok else 0
# 最新写入时间
last = fetch_one("SELECT MAX(ts) AS last_ts FROM db_metrics")
self._send_json({
"service": "oa-monitor-api",
"db_exists": db_ok,
"db_size_bytes": db_size,
"latest_data_ts": ts_to_iso(last['last_ts'] if last else None),
"now": ts_to_iso(int(datetime.now().timestamp()))
})
def _overview(self):
# 各层最新一条
db = fetch_one("SELECT * FROM db_metrics ORDER BY ts DESC LIMIT 1")
http_all = fetch_all("SELECT endpoint, status_code, resp_time_ms, healthy, ts FROM http_metrics WHERE ts > ? ORDER BY ts DESC", ((datetime.now().timestamp() - 120),))
biz = fetch_one("SELECT * FROM business_metrics ORDER BY ts DESC LIMIT 1")
infra = fetch_one("SELECT * FROM infra_metrics ORDER BY ts DESC LIMIT 1")
# 活跃告警
active_alerts = fetch_all("SELECT * FROM alerts WHERE ack = 0 ORDER BY ts DESC")
# HTTP 端点最新状态聚合
http_summary = []
for row in http_all:
http_summary.append({
"endpoint": row["endpoint"],
"status_code": row["status_code"],
"resp_time_ms": row["resp_time_ms"],
"healthy": row["healthy"],
"ts": ts_to_iso(row["ts"])
})
# DB 层最近长查询
long_q = fetch_all("SELECT * FROM db_processlist WHERE time_sec >= 30 ORDER BY time_sec DESC LIMIT 5")
self._send_json({
"db_latest": {
**({} if not db else db),
"ts_iso": ts_to_iso(db["ts"]) if db else None
},
"http_summary": http_summary,
"biz_latest": {
**({} if not biz else biz),
"ts_iso": ts_to_iso(biz["ts"]) if biz else None
},
"infra_latest": {
**({} if not infra else infra),
"ts_iso": ts_to_iso(infra["ts"]) if infra else None
},
"active_alerts": [{**a, "ts_iso": ts_to_iso(a["ts"])} for a in active_alerts],
"long_queries": [{**q, "ts_iso": ts_to_iso(q["ts"])} for q in long_q]
})
def _db_recent(self, qs):
minutes = int(qs.get('minutes', [30])[0])
since = int(datetime.now().timestamp()) - minutes * 60
rows = fetch_all(
"SELECT * FROM db_metrics WHERE ts >= ? ORDER BY ts ASC",
(since,)
)
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
def _db_long_queries(self, qs):
since_min = int(qs.get('since_minutes', [60])[0])
since = int(datetime.now().timestamp()) - since_min * 60
rows = fetch_all(
"SELECT * FROM db_processlist WHERE ts >= ? ORDER BY time_sec DESC LIMIT 100",
(since,)
)
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
def _http_recent(self, qs):
minutes = int(qs.get('minutes', [30])[0])
since = int(datetime.now().timestamp()) - minutes * 60
rows = fetch_all(
"SELECT * FROM http_metrics WHERE ts >= ? ORDER BY ts ASC",
(since,)
)
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
def _biz_recent(self, qs):
hours = int(qs.get('hours', [24])[0])
since = int(datetime.now().timestamp()) - hours * 3600
rows = fetch_all(
"SELECT * FROM business_metrics WHERE ts >= ? ORDER BY ts ASC",
(since,)
)
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
def _infra_recent(self, qs):
minutes = int(qs.get('minutes', [30])[0])
since = int(datetime.now().timestamp()) - minutes * 60
rows = fetch_all(
"SELECT * FROM infra_metrics WHERE ts >= ? ORDER BY ts ASC",
(since,)
)
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
def _alerts(self, qs):
limit = int(qs.get('limit', [100])[0])
only_unack = qs.get('unack', ['0'])[0]
sql = "SELECT * FROM alerts"
params = []
if only_unack == '1':
sql += " WHERE ack = 0"
sql += " ORDER BY ts DESC LIMIT ?"
params.append(limit)
rows = fetch_all(sql, tuple(params))
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
# ---- 启动 ----
if __name__ == '__main__':
port = int(sys.argv[1]) if len(sys.argv) > 1 else 8455
server = http.server.HTTPServer(('0.0.0.0', port), MonitorHandler)
print(f"OA Monitor API listening on :{port} (DB={DB_PATH})", flush=True)
server.serve_forever()
+27 -6
View File
@@ -4,6 +4,9 @@
# v2 修复(2026-10-28): # v2 修复(2026-10-28):
# - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警 # - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警
# - Step 3 业务层检测被 Step 0 结果门禁 # - Step 3 业务层检测被 Step 0 结果门禁
# v3 修复(2026-11-04):
# - 业务层 P0 推送断档假阳性:sqlite_exec 吞错误(2>/dev/null)→ 查询失败返回空串 → :-0 默认为 0 → 误报
# - 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,查询失败跳过检测并 warn
# ============================================================================ # ============================================================================
set -u set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -36,7 +39,7 @@ else
if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟 if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟
BIZ_FRESH=false BIZ_FRESH=false
HUMAN_AGE=$((BIZ_AGE / 60)) HUMAN_AGE=$((BIZ_AGE / 60))
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec \"SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')\") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec \"SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')\") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警" BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec "SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec "SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}" dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}"
log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)" log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)"
else else
@@ -94,11 +97,28 @@ fi
if [[ "${BIZ_FRESH}" != "true" ]]; then if [[ "${BIZ_FRESH}" != "true" ]]; then
log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)" log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)"
else else
# 数据新鲜 → 正常业务层判断 # v3 修复:显式调用 sqlite3 + 检查退出码 + 正则校验,区分"查询失败"和"结果为 0"
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1) MSG_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null)
ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1) MSG_RC=$?
HOUR_MSG=${HOUR_MSG:-0} ONGOING_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null)
ONGOING=${ONGOING:-0} ONGOING_RC=$?
# 双重校验:退出码必须为 0,且返回值非空且为纯数字
BIZ_QUERY_OK=true
if [[ ${MSG_RC} -ne 0 ]] || [[ -z "${MSG_RAW}" ]] || [[ ! "${MSG_RAW}" =~ ^[0-9]+$ ]]; then
log_warn "[ALERT] SQLite msg_log_hourly 查询异常 rc=${MSG_RC} raw='${MSG_RAW}',跳过业务层(防假阳性)"
BIZ_QUERY_OK=false
fi
if [[ ${ONGOING_RC} -ne 0 ]] || [[ -z "${ONGOING_RAW}" ]] || [[ ! "${ONGOING_RAW}" =~ ^[0-9]+$ ]]; then
log_warn "[ALERT] SQLite ongoing_requests 查询异常 rc=${ONGOING_RC} raw='${ONGOING_RAW}',跳过业务层(防假阳性)"
BIZ_QUERY_OK=false
fi
if [[ "${BIZ_QUERY_OK}" != "true" ]]; then
log_warn "[ALERT] ⏸️ 业务层数据查询异常,跳过钉钉推送断档 P0 检测"
else
HOUR_MSG=${MSG_RAW}
ONGOING=${ONGOING_RAW}
CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal
WORK_HOURS_ENABLED=false WORK_HOURS_ENABLED=false
@@ -124,6 +144,7 @@ else
log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警" log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警"
fi fi
fi fi
fi
# ============================================================================ # ============================================================================
# 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信) # 4. 基础设施层检测(infra_metrics 每分钟更新,数据可信)
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""fix-info-read-dummy.py — 补 DUMMY 根治 updateBizState 死循环 (Python 3.6 兼容)"""
import sys, os, time, subprocess, json, urllib.request
DB_HOST, DB_PORT, DB_USER = "39.100.72.102", 3306, "ecology"
DB_PASS, DB_NAME = "Z!BYT+jSDDyYHMdX+qFSLzJCYXW", "ecology"
OA_HOST, OA_USER, OA_PASS = "39.99.236.138", "root", "KTOFKjVkVOGlNgvs8yUP1+PM1VM"
DINGTALK = "https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249"
BATCH = 5000
LOG = "/opt/oa-monitor/logs/fix-info-read.log"
# === 正确值(从之前 DUMMY_FIX_ 样例提取)===
MESSAGETYPE = 25 # int!不是 'archive'
BIZTYPE = 86 # int!上次成功用的
STATE = 2 # 已读
BIZSTATE = 1 # 已更新(关键!让 updateBizState 不再重试)
GROUPTYPE = "archive-fix"
BIZTITLE = "dummy fix"
def log(msg, level="INFO"):
line = "[%s] [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S"), level, msg)
print(line, flush=True)
with open(LOG, "a") as f:
f.write(line + "\n")
def run_mysql(sql, fetch=True):
env = os.environ.copy(); env["MYSQL_PWD"] = DB_PASS
r = subprocess.run(["mysql","-h"+DB_HOST,"-P"+str(DB_PORT),"-u"+DB_USER,DB_NAME,"-N","-e",sql],
env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=600)
if r.returncode != 0:
log("MySQL ERROR: " + r.stderr.decode().strip(), "ERROR")
return None
if not fetch: return True
return r.stdout.decode().strip()
def dingtalk(level, title, body=""):
text = "**[%s] %s**\n\n%s\n\n> OA DUMMY @ %s" % (level, title, body, time.strftime("%Y-%m-%d %H:%M:%S"))
try:
urllib.request.urlopen(urllib.request.Request(DINGTALK, json.dumps({"msgtype":"markdown","markdown":{"title":title,"text":text}}).encode(), {"Content-Type":"application/json"}), timeout=5)
except Exception as e: log("钉钉: "+str(e), "WARN")
def check_http():
env = os.environ.copy()
cmd = ("python3 -c 'import paramiko; ssh=paramiko.SSHClient(); ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()); "
"ssh.connect(\"%s\",username=\"%s\",password=\"%s\",timeout=15); "
"_,o,_=ssh.exec_command(\"curl -s -o /dev/null -w %%{http_code} http://127.0.0.1:80/\"); "
"print(o.read().decode(\"utf-8\",\"replace\")); ssh.close()'" % (OA_HOST, OA_USER, OA_PASS))
r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=25, env=env)
return "200" in r.stdout.decode().strip()
def check_tr():
r = run_mysql("SHOW GLOBAL STATUS WHERE Variable_name='Threads_running';")
if not r: return 999
try: return int(r.split("\t")[-1].strip())
except: return 999
def wait_safe(max_wait=120):
for i in range(max_wait // 5):
if check_http() and check_tr() < 30: return True
log("等待恢复 (%ds) HTTP=%s TR=%d" % (i*5, check_http(), check_tr()), "WARN")
time.sleep(5)
return False
# === Phase 0 ===
log("===== Phase 0: 前置检查 =====")
DRY = "--dry-run" in sys.argv
if DRY: log("*** DRY RUN ***")
ok = check_http(); log("Resin HTTP 80 = %s" % ok)
if not ok: log("Resin 不通!中止!", "CRITICAL"); sys.exit(1)
tr = check_tr(); log("Threads_running = %d" % tr)
if tr >= 15:
wait_safe(180); tr = check_tr()
existing = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_FIX_%';")
log("现有 DUMMY_FIX = %s" % existing)
log("Phase 1: 备份表...")
run_mysql("CREATE TABLE IF NOT EXISTS ecology_message_info_read_bak_20261104 AS SELECT * FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';", fetch=False)
dingtalk("INFO", "DUMMY 补数据启动", "HTTP=200 ✅\nTR=%d\nDUMMY_FIX=%s" % (tr, existing))
# === Phase 2 ===
total = [0]
def insert_user(uid, prefix, col):
last_id = 0; batch = 0; user_total = 0
while True:
sql = ("INSERT INTO ecology_message_info_read "
"(ID, MESSAGEID, MESSAGETYPE, USERID, TARGETID, STATE, BIZSTATE, "
"CREATEDATE, CREATETIME, MESSAGEGROUPTYPE, BIZTITLE, BIZTYPE) "
"SELECT CONCAT('%s%s_', REQUESTID), 0, %d, %d, REQUESTID, %d, %d, "
"CURDATE(), CURTIME(), '%s', '%s', %d "
"FROM workflow_requestbase "
"WHERE %s = %d AND STATUS LIKE '%%归档%%' AND DELETED = 0 AND REQUESTID > %d "
"ORDER BY REQUESTID LIMIT %d;" %
(prefix, "%d", MESSAGETYPE, uid, STATE, BIZSTATE, GROUPTYPE, BIZTITLE, BIZTYPE,
col, uid, last_id, BATCH))
if DRY:
batch += 1; user_total += BATCH; last_id += BATCH
if batch >= 5: break
time.sleep(0.2); continue
run_mysql(sql, fetch=False)
rc = run_mysql("SELECT ROW_COUNT();")
inserted = int(rc) if rc and rc.isdigit() else 0
if inserted == 0: break
last_sql = "SELECT MAX(REQUESTID) FROM workflow_requestbase WHERE %s = %d AND REQUESTID > %d LIMIT %d;" % (col, uid, last_id, BATCH)
lv = run_mysql(last_sql)
if lv and lv.isdigit(): last_id = int(lv)
batch += 1; user_total += inserted; total[0] += inserted
log(" User=%d batch=%d +%d rows (total=%d)" % (uid, batch, inserted, total[0]))
if not check_http() or check_tr() >= 30:
log("异常!暂停...", "WARN"); wait_safe(10)
time.sleep(1)
if user_total > 0:
log("User=%d 完成: %d 条 %d 批" % (uid, user_total, batch))
# Phase 2a — 全量业务用户(不限 TOP 100)
log("===== Phase 2a: CREATER (全量) =====")
creaters_raw = run_mysql("SELECT CREATER, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND CREATER!=1 GROUP BY CREATER ORDER BY COUNT(*) DESC;")
creaters = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in creaters_raw.strip().split("\n") if l.strip()]
log("CREATER 用户=%d 归档总量=%d" % (len(creaters), sum(c for _,c in creaters)))
for i, (uid, tot) in enumerate(creaters, 1):
log("--- CREATER #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
insert_user(uid, "DUMMY_B_CREATER_", "CREATER")
if i % 10 == 0:
dingtalk("INFO", "CREATER %d/100" % i, "已插入=%d" % total[0])
# Phase 2b — 全量业务用户(不限 TOP 100)
log("===== Phase 2b: LASTOPERATOR (全量) =====")
ops_raw = run_mysql("SELECT LASTOPERATOR, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND LASTOPERATOR!=1 GROUP BY LASTOPERATOR ORDER BY COUNT(*) DESC;")
ops = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in ops_raw.strip().split("\n") if l.strip()]
log("OPERATOR 用户=%d 归档总量=%d" % (len(ops), sum(c for _,c in ops)))
for i, (uid, tot) in enumerate(ops, 1):
log("--- OPERATOR #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
insert_user(uid, "DUMMY_B_OPERATOR_", "LASTOPERATOR")
if i % 10 == 0:
dingtalk("INFO", "OPERATOR %d/100" % i, "已插入=%d" % total[0])
# Phase 3
log("===== Phase 3: 验证 =====")
fc = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';")
log("DUMMY_B 总数 = %s" % fc)
http_ok = check_http(); log("Resin HTTP = %s" % http_ok)
tr2 = check_tr(); log("Threads_running = %d" % tr2)
lm = run_mysql("SELECT MAX(CREATETIME) FROM ecology_message_log WHERE CREATEDATE=CURDATE();")
log("message_log 最新 = %s" % lm)
dingtalk("P0", "DUMMY 补数据完成 ✅",
"**DUMMY_B**: %s 条\n**HTTP**: %s\n**TR**: %d\n**最新消息**: %s\n\n回滚: DELETE FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%%';" %
(fc, http_ok, tr2, lm))
log("===== 全部完成 =====")
+120
View File
@@ -0,0 +1,120 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 告警检测(每 1 分钟)
# 遍历所有指标,超过阈值触发钉钉告警
# 告警冷却由 common.sh 的 alert_cooldown 表处理(10 分钟同类型不重复)
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
NOW=$(date +%s)
log_info "[ALERT] ========== 告警检测开始 =========="
# ============================================================================
# 1. DB 层检测
# ============================================================================
# Threads_running 过高
TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${TR}" ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then
BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;')\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;')"
dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');"
fi
# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的)
LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;")
if [[ -n "${LONG_QUERIES}" ]]; then
COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ')
DETAIL=""
while IFS='|' read -r pid time user state info; do
DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`"
done <<< "${LONG_QUERIES}"
BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理"
dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');"
fi
# ============================================================================
# 2. HTTP 端点检测
# ============================================================================
# 最近 3 次的失败次数
FAIL_COUNT=0
FAIL_DETAIL=""
for endpoint in "${OA_ENDPOINTS[@]}"; do
FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;")
FAIL_HERE=${FAIL_HERE:-0}
if [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then
FAIL_COUNT=$((FAIL_COUNT + 1))
LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;")
FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}"
fi
done
if [[ ${FAIL_COUNT} -gt 0 ]]; then
BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态"
dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');"
fi
# ============================================================================
# 3. 业务层检测
# ============================================================================
# message_log 小时增量过低(钉钉推送可能断了)
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${HOUR_MSG}" ]] && [[ ${HOUR_MSG} -lt ${ALERT_MSG_LOG_HOUR_MIN} ]] && [[ ${HOUR_MSG} -gt 0 ]]; then
BODY="**ecology_message_log 小时增量仅 ${HOUR_MSG} 条**(阈值 ${ALERT_MSG_LOG_HOUR_MIN})\n\n> 钉钉推送可能阻塞。泛微云桥 e-Bridge 为黑盒线程,需登录 OA 后台检查「消息推送」日志或用 SHOW PROCESSLIST 看 active 线程"
dingtalk_alert "P1" "钉钉推送疑似停滞" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'business', '钉钉推送疑似停滞', 'msg_hourly=${HOUR_MSG}');"
fi
# ============================================================================
# 4. 基础设施层检测
# ============================================================================
# Docker daemon 挂了
DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;")
if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then
BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> ⚠️ 今天的 rm -rf /ipaas 事故中 Docker daemon 重启过。请立即登录 DSH 检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`"
dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');"
fi
# 容器数突变(对比最近 30 分钟的平均数)
CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;")
if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]]; then
DELTA=$(( CURR_COUNT - ${AVG_COUNT%%.*} ))
ABS_DELTA=${DELTA#-}
if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then
BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a"
dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');"
fi
fi
# 磁盘
DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${DISK_PCT}" ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then
BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)\n\n> 请检查:df -h → du -sh /var/lib/docker/* → pm2 logs 清理 → 旧 SQLite 数据归档"
dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');"
fi
# inode
INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${INODE_PCT}" ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then
BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)\n\n> 可能有大量小文件,检查 find / -type f | wc -l"
dingtalk_alert "P1" "DSH inode 告警" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');"
fi
# Docker root 写入突增
ROOT_WRITE=$(sqlite_exec "SELECT docker_root_write_mb FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
if [[ -n "${ROOT_WRITE}" ]] && (( ${ROOT_WRITE%%.*} >= ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN )); then
BODY="**Docker data-root 1 分钟内写入 ${ROOT_WRITE}MB**(阈值 ${ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN}MB)\n\n> 可能有容器在疯狂写数据,检查:docker stats → docker logs"
dingtalk_alert "P2" "Docker data-root 写入突增" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P2', 'infra', 'Docker root 写入突增', 'write_mb=${ROOT_WRITE}');"
fi
log_info "[ALERT] ========== 告警检测完成 =========="
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 数据清理(每天 03:00)
# 保留最近 7 天指标,alerts 和 alert_cooldown 保留 30 天
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
NOW=$(date +%s)
RETENTION_7D=$(( NOW - 7 * 86400 ))
RETENTION_30D=$(( NOW - 30 * 86400 ))
log_info "[CLEANUP] 开始清理,阈值 ts < ${RETENTION_7D}(7天),alerts < ${RETENTION_30D}(30天)"
# 各层指标表
for table in db_metrics http_metrics business_metrics infra_metrics db_processlist; do
BEFORE=$(sqlite_exec "SELECT COUNT(*) FROM ${table};")
sqlite_exec "DELETE FROM ${table} WHERE ts < ${RETENTION_7D};"
AFTER=$(sqlite_exec "SELECT COUNT(*) FROM ${table};")
log_info "[CLEANUP] ${table}: ${BEFORE} → ${AFTER}(删了 $((BEFORE - AFTER)) 条)"
done
# alerts 保留 30 天
BEFORE=$(sqlite_exec "SELECT COUNT(*) FROM alerts;")
sqlite_exec "DELETE FROM alerts WHERE ts < ${RETENTION_30D};"
AFTER=$(sqlite_exec "SELECT COUNT(*) FROM alerts;")
log_info "[CLEANUP] alerts: ${BEFORE} → ${AFTER}(删了 $((BEFORE - AFTER)) 条)"
# VACUUM 压缩(可选,不频繁)
if (( NOW % (86400 * 7) < 3600 )); then
log_info "[CLEANUP] 每周 VACUUM SQLite..."
sqlite_exec "VACUUM;"
fi
log_info "[CLEANUP] 完成"
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 业务指标采集(每 1 小时)
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
NOW=$(date +%s)
db_exec() {
local sql="$1"
MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -e "${sql}" 2>/dev/null
}
MSG_TOTAL=$(db_exec "SELECT COUNT(*) FROM ecology_message_log;")
MSG_TOTAL=${MSG_TOTAL:-0}
MSG_FAIL_HOUR=$(db_exec "SELECT COUNT(*) FROM ecology_message_log WHERE retcode != 0 AND operTime >= DATE_SUB(NOW(), INTERVAL 1 HOUR);")
MSG_FAIL_HOUR=${MSG_FAIL_HOUR:-0}
ONGOING=$(db_exec "SELECT COUNT(*) FROM workflow_requestbase WHERE status = 2 OR (status IS NULL AND currentnodeid > 0);")
ONGOING=${ONGOING:-0}
OVERDUE=$(db_exec "SELECT COUNT(*) FROM workflow_requestlog WHERE timeout = 1 AND operatedate >= DATE_SUB(NOW(), INTERVAL 1 DAY);")
OVERDUE=${OVERDUE:-0}
if [[ ${MSG_TOTAL} -gt 0 ]]; then
FAIL_RATE=$(echo "scale=4; ${MSG_FAIL_HOUR} / (${MSG_TOTAL} + 0.001);" | bc 2>/dev/null || echo "0.0")
else
FAIL_RATE="0.0"
fi
sqlite_exec "INSERT INTO business_metrics VALUES (
${NOW}, ${MSG_TOTAL}, ${FAIL_RATE}, ${ONGOING}, ${OVERDUE}
);"
log_info "[BIZ] ✅ msg_total=${MSG_TOTAL} fail_hour=${MSG_FAIL_HOUR} fail_rate=${FAIL_RATE} ongoing=${ONGOING} overdue=${OVERDUE}"
+94
View File
@@ -0,0 +1,94 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · DB 层采集(每 30 秒)
# 在 DSH 宿主机上直接 mysql 连 OA DB(DSH 本身就是跳板,不需要再 ssh)
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
NOW=$(date +%s)
# ---- MySQL 直连(DSH 上直接连 OA DB 公网映射)----
DB_MYSQL() {
MYSQL_PWD="${DB_PASS}" timeout 12 mysql -h39.100.72.102 -P3306 -uecology ecology 2>/dev/null
}
db_exec() {
DB_MYSQL -e "$1"
}
# ---- 1. SHOW STATUS ----
log_info "[DB] SHOW STATUS"
STATUS=$(DB_MYSQL -e "SHOW GLOBAL STATUS" | grep -E '^(Threads_running|Threads_connected|Threads_idle|Uptime|Innodb_rows_read|Innodb_rows_inserted|Innodb_rows_updated|Innodb_rows_deleted|Slow_queries)\t')
THREADS_RUNNING=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_running"{print $2}')
THREADS_CONNECTED=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_connected"{print $2}')
THREADS_IDLE=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_idle"{print $2}')
UPTIME=$(echo "${STATUS}" | awk -F'\t' '$1=="Uptime"{print $2}')
INNODB_READ=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_read"{print $2}')
INNODB_INS=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_inserted"{print $2}')
INNODB_UPD=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_updated"{print $2}')
INNODB_DEL=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_deleted"{print $2}')
SLOW_QUERIES=$(echo "${STATUS}" | awk -F'\t' '$1=="Slow_queries"{print $2}')
INNODB_READ=${INNODB_READ:-0}
INNODB_INS=${INNODB_INS:-0}
INNODB_UPD=${INNODB_UPD:-0}
INNODB_DEL=${INNODB_DEL:-0}
INNODB_WRITE=$(( INNODB_INS + INNODB_UPD + INNODB_DEL ))
THREADS_RUNNING=${THREADS_RUNNING:-0}
THREADS_CONNECTED=${THREADS_CONNECTED:-0}
THREADS_IDLE=${THREADS_IDLE:-0}
UPTIME=${UPTIME:-0}
SLOW_QUERIES=${SLOW_QUERIES:-0}
# ---- 2. 上一次快照 → 差值 ----
PREV=$(sqlite_exec "SELECT * FROM db_metrics ORDER BY ts DESC LIMIT 1;")
QPS_READ=0; QPS_WRITE=0; SLOW_DELTA=0
if [[ -n "${PREV}" ]]; then
PREV_UPTIME=$(echo "${PREV}" | cut -d'|' -f5)
PREV_READ=$(echo "${PREV}" | cut -d'|' -f6)
PREV_WRITE=$(echo "${PREV}" | cut -d'|' -f7)
PREV_SLOW=$(echo "${PREV}" | cut -d'|' -f9)
PREV_UPTIME=${PREV_UPTIME:-0}; PREV_READ=${PREV_READ:-0}
PREV_WRITE=${PREV_WRITE:-0}; PREV_SLOW=${PREV_SLOW:-0}
DIFF_SEC=$(( NOW - PREV_UPTIME ))
if [[ ${DIFF_SEC} -gt 0 ]]; then
QPS_READ=$(( (INNODB_READ - PREV_READ) / DIFF_SEC ))
QPS_WRITE=$(( (INNODB_WRITE - PREV_WRITE) / DIFF_SEC ))
fi
SLOW_DELTA=$(( SLOW_QUERIES - PREV_SLOW ))
fi
# ---- 3. 写 SQLite ----
sqlite_exec "INSERT INTO db_metrics VALUES (
${NOW}, ${THREADS_RUNNING}, ${THREADS_CONNECTED}, ${THREADS_IDLE},
${UPTIME}, ${QPS_READ}, ${QPS_WRITE}, ${SLOW_DELTA}
);"
log_info "[DB] ✅ 写入:Threads_running=${THREADS_RUNNING} QPS_read=${QPS_READ} QPS_write=${QPS_WRITE}"
# ---- 4. 长查询 processlist ----
log_info "[DB] processlist"
PLIST=$(DB_MYSQL -e 'SHOW PROCESSLIST;')
echo "${PLIST}" | awk -F'\t' '
NR==1 { next }
NF < 8 { next }
{
pid=$1; user=$2; state=$6; time=$7; info=$8
if (time+0 >= 5 && pid+0 > 0) {
gsub(/\t/, " ", info)
printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300)
}
}
' | while IFS=$'\t' read -r pid time user state info; do
[[ -z "${pid}" ]] && continue
info_safe=$(echo "${info}" | sed "s/'/\"/g")
sqlite_exec "INSERT INTO db_processlist (ts, pid, user, state, time_sec, info)
VALUES (${NOW}, ${pid}, '${user}', '${state}', ${time}, '${info_safe}');"
done
log_info "[DB] ✅ 完成"
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · HTTP 端点健康检查(每 30 秒)
# 在 DSH 上直接 curl
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
NOW=$(date +%s)
for endpoint in "${OA_ENDPOINTS[@]}"; do
result=$(curl -sk -o /dev/null -w '%{http_code}|%{time_total}' --max-time 10 "${endpoint}" 2>/dev/null)
status=$(echo "${result}" | cut -d'|' -f1)
time_ms=$(echo "${result}" | cut -d'|' -f2 | awk '{printf "%d", $1*1000}')
status=${status:-0}
time_ms=${time_ms:-0}
case "${status}" in
000) healthy=0 ;;
2*) healthy=1 ;;
3*) healthy=1 ;;
*) healthy=0 ;;
esac
sqlite_exec "INSERT INTO http_metrics (ts, endpoint, status_code, resp_time_ms, healthy)
VALUES (${NOW}, '${endpoint}', ${status}, ${time_ms}, ${healthy});"
log_info "[HTTP] ${endpoint} status=${status} time=${time_ms}ms healthy=${healthy}"
done
log_info "[HTTP] 全部完成"
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 基础设施层采集(每 30 秒)
# 在 DSH 上直接执行 docker / df / du
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
NOW=$(date +%s)
DOCKER_ROOT="/var/lib/docker"
# ---- Docker daemon + 容器数 ----
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ')
if [[ "${CONTAINER_COUNT}" =~ ^[0-9]+$ ]] && [[ ${CONTAINER_COUNT} -ge 0 ]]; then
DOCKER_OK=1
else
DOCKER_OK=0
CONTAINER_COUNT=0
fi
# ---- Docker root 写入量(du overlay2 增量)----
PREV_ROOT_WRITE=$(sqlite_exec "SELECT COALESCE(docker_root_write_mb, 0) FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
CURRENT_ROOT_MB=$(du -sm "${DOCKER_ROOT}/overlay2" 2>/dev/null | cut -f1 | tr -d ' ')
CURRENT_ROOT_MB=${CURRENT_ROOT_MB:-0}
PREV_ROOT_WRITE=${PREV_ROOT_WRITE:-0}
ROOT_WRITE_DELTA_MB=0
if [[ -n "${PREV_ROOT_WRITE}" ]] && [[ "${PREV_ROOT_WRITE}" =~ ^[0-9.]+$ ]]; then
DELTA=$(( CURRENT_ROOT_MB - ${PREV_ROOT_WRITE%%.*} ))
if [[ ${DELTA} -gt 0 ]]; then
ROOT_WRITE_DELTA_MB="${DELTA}"
fi
fi
# ---- 磁盘 / inode / load ----
DISK_PCT=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
INODE_PCT=$(df -i / | tail -1 | awk '{print $5}' | tr -d '%')
LOADAVG=$(cat /proc/loadavg | awk '{print $1}')
DISK_PCT=${DISK_PCT:-0}
INODE_PCT=${INODE_PCT:-0}
LOADAVG=${LOADAVG:-0}
sqlite_exec "INSERT INTO infra_metrics VALUES (
${NOW}, ${DOCKER_OK}, ${CONTAINER_COUNT}, ${ROOT_WRITE_DELTA_MB},
${DISK_PCT}, ${INODE_PCT}, ${LOADAVG}
);"
log_info "[INFRA] ✅ docker_ok=${DOCKER_OK} containers=${CONTAINER_COUNT} root_write_mb=${ROOT_WRITE_DELTA_MB} disk=${DISK_PCT}% inode=${INODE_PCT}% loadavg=${LOADAVG}"
+106
View File
@@ -0,0 +1,106 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 全局配置与公共函数
# 所有采集/告警脚本 source 此文件,只修改这里的变量即可调整行为
# ============================================================================
set -u # 未定义变量报错
# 注意:不写 set -e,采集脚本单条命令失败不应整体中断
# ---- 路径 ----
export MONITOR_ROOT="/opt/oa-monitor"
export MONITOR_DATA="${MONITOR_ROOT}/data"
export MONITOR_LOGS="${MONITOR_ROOT}/logs"
export MONITOR_DB="${MONITOR_DATA}/oa-monitor.db"
mkdir -p "${MONITOR_DATA}" "${MONITOR_LOGS}"
# ---- 工具 ----
export MYSQL_CMD="mysql -h39.100.72.102 -P3306 -uecology"
export DB_NAME="ecology"
export DB_PASS='Z!BYT+jSDDyYHMdX+qFSLzJCYXW'
export SSH_DSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i ~/.ssh/id_ed25519 root@47.108.21.167"
# ---- OA 端点(DSH 上访问 OA 应用服务器内网 IP,更稳)----
OA_HTTP="http://39.99.236.138"
OA_ENDPOINTS=(
"${OA_HTTP}:80/wui/main/Login.jsp"
"${OA_HTTP}:8444/api/portal/portalinfo"
"${OA_HTTP}:8444/api/portal/sso/getLoginInfo.jsp"
"${OA_HTTP}/api/workflow/getRequestList"
)
# ---- 钉钉 Webhook(群 723387268,走 DSH 本地转发)----
export DINGTALK_WEBHOOK="https://dsh.gs-robot.com/dingtalk/webhook"
export DINGTALK_SECRET=""
# ---- 告警阈值 ----
export ALERT_THREADS_RUNNING_HIGH=50 # 当前活跃线程 > 50 告警
export ALERT_SLOW_QUERY_SEC=30 # 慢查询 > 30s 告警
export ALERT_SLOW_QUERY_COUNT=5 # 同进程号出现 5 次以上告警
export ALERT_HTTP_FAIL_COUNT=3 # HTTP 端点连续失败 3 次告警
export ALERT_MSG_LOG_HOUR_MIN=100 # 1 小时消息增量 < 100 条告警(可能钉钉推送断了)
export ALERT_DOCKER_CONTAINER_DELTA=3 # 容器数突变 > ±3 告警
export ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN=1024 # Docker data-root 1 分钟写 > 1GB 告警
export ALERT_DISK_USAGE_PCT=85 # 磁盘使用率 > 85% 告警
export ALERT_INODE_USAGE_PCT=80 # inode 使用率 > 80% 告警
export ALERT_DOCKER_DAEMON_FAIL_COUNT=2 # Docker daemon 连续失败 2 次告警
export ALERT_LONG_QUERY_SEC=300 # 长查询 > 300s 触发 KILL(dry-run 默认)
# ---- 告警抑制 ----
export ALERT_COOLDOWN_SEC=600 # 同一告警类型 10 分钟内不重复发
# ---- SQLite 工具函数 ----
sqlite_exec() {
sqlite3 "${MONITOR_DB}" "$@" 2>/dev/null
}
# ---- 日志函数 ----
log() {
local level="$1"; shift
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MONITOR_LOGS}/oa-monitor.log"
}
log_info() { log "INFO" "$@"; }
log_warn() { log "WARN" "$@"; }
log_error() { log "ERROR" "$@"; }
# ---- 钉钉告警(markdown 格式)----
# 用法:dingtalk_alert <level> <title> <markdown_body>
# level: P0 / P1 / P2
dingtalk_alert() {
local level="$1" title="$2" body="$3"
local timestamp
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
# 告警冷却检查
local key="${level}:${title}"
local last_ts
last_ts=$(sqlite_exec "SELECT COALESCE(last_alert_ts, 0) FROM alert_cooldown WHERE alert_key='${key}';")
local now_epoch
now_epoch=$(date +%s)
if [[ "${last_ts}" =~ ^[0-9]+$ ]] && (( now_epoch - last_ts < ALERT_COOLDOWN_SEC )); then
log_info "告警抑制:${key}(距上次 ${now_epoch}s < ${ALERT_COOLDOWN_SEC}s)"
return 0
fi
# 构造 JSON
local json
json=$(cat <<JSON
{"msgtype":"markdown","markdown":{"title":"[OA监控] ${level} ${title}","text":"### [OA监控] **${level}** ${title}\n\n**时间**:${timestamp}\n\n**主机**:DSH (47.108.21.167)\n\n---\n\n${body}\n\n---\n\n> OA 运维监控平台 · $(basename "$0")"}}
JSON
)
# 发送
local http_code
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Content-Type: application/json" \
-d "${json}" "${DINGTALK_WEBHOOK}" 2>/dev/null)
if [[ "${http_code}" == "200" ]]; then
log_info "钉钉告警发送成功:[${level}] ${title}"
sqlite_exec "INSERT OR REPLACE INTO alert_cooldown VALUES ('${key}', ${now_epoch});"
else
log_error "钉钉告警发送失败(HTTP ${http_code}):[${level}] ${title}"
fi
}
+102
View File
@@ -0,0 +1,102 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>OA 运维监控 · 告警历史</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: -apple-system, 'PingFang SC', sans-serif; background: #0d1117; color: #c9d1d9; padding: 16px; }
h1 { font-size: 20px; color: #58a6ff; margin-bottom: 16px; }
.filter-bar { margin-bottom: 16px; display: flex; gap: 12px; align-items: center; flex-wrap: wrap; }
.filter-bar button { background: #21262d; color: #c9d1d9; border: 1px solid #30363d; padding: 6px 14px; border-radius: 6px; cursor: pointer; font-size: 13px; }
.filter-bar button.active { background: #1f6feb; border-color: #1f6feb; }
.filter-bar button:hover { border-color: #58a6ff; }
.stats { display: flex; gap: 16px; margin-bottom: 16px; }
.stat { background: #161b22; border: 1px solid #30363d; padding: 12px 16px; border-radius: 8px; }
.stat .num { font-size: 24px; font-weight: 700; }
.stat .label { color: #8b949e; font-size: 12px; margin-top: 2px; }
.tag { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 11px; font-weight: 600; }
.tag-p0 { background: rgba(248,81,73,0.25); color: #ff7b72; }
.tag-p1 { background: rgba(210,153,34,0.25); color: #d29922; }
.tag-p2 { background: rgba(56,139,253,0.25); color: #58a6ff; }
table { width: 100%; border-collapse: collapse; font-size: 13px; background: #161b22; border-radius: 8px; overflow: hidden; }
th { background: #21262d; color: #8b949e; padding: 10px 12px; text-align: left; font-weight: 600; position: sticky; top: 0; }
td { padding: 9px 12px; border-top: 1px solid #21262d; }
tr:hover td { background: #1f2428; }
.btn-ack { background: #1f6feb; color: white; border: none; padding: 4px 10px; border-radius: 4px; cursor: pointer; font-size: 12px; }
.btn-resolve { background: #238636; color: white; border: none; padding: 4px 10px; border-radius: 4px; cursor: pointer; font-size: 12px; }
.ack-badge { color: #3fb950; font-weight: 600; font-size: 11px; }
</style>
</head>
<body>
<h1>🚨 告警历史</h1>
<div class="filter-bar">
<button data-filter="all" class="active">全部</button>
<button data-filter="P0">P0</button>
<button data-filter="P1">P1</button>
<button data-filter="P2">P2</button>
<button data-filter="unack">未确认</button>
</div>
<div class="stats">
<div class="stat"><div class="num" id="statTotal">0</div><div class="label">告警总数</div></div>
<div class="stat"><div class="num" id="statP0">0</div><div class="label">P0 告警</div></div>
<div class="stat"><div class="num" id="statUnack">0</div><div class="label">未确认</div></div>
</div>
<table id="alertsTable">
<thead><tr><th>级别</th><th>时间</th><th>类别</th><th>标题</th><th>详情</th><th>状态</th></tr></thead>
<tbody id="alertsBody"></tbody>
</table>
<script>
const API = window.location.origin + '/monitor';
let currentFilter = 'all';
let allAlerts = [];
document.querySelectorAll('.filter-bar button').forEach(b => {
b.addEventListener('click', () => {
document.querySelectorAll('.filter-bar button').forEach(x => x.classList.remove('active'));
b.classList.add('active');
currentFilter = b.dataset.filter;
render();
});
});
async function refresh() {
const res = await fetch(API + '/api/alerts?limit=200');
allAlerts = await res.json();
render();
}
function render() {
let filtered = allAlerts;
if (currentFilter === 'unack') filtered = filtered.filter(a => !a.ack);
else if (['P0','P1','P2'].includes(currentFilter)) filtered = filtered.filter(a => a.level === currentFilter);
// stats
document.getElementById('statTotal').textContent = allAlerts.length;
document.getElementById('statP0').textContent = allAlerts.filter(a => a.level === 'P0').length;
document.getElementById('statUnack').textContent = allAlerts.filter(a => !a.ack).length;
const tbody = document.getElementById('alertsBody');
tbody.innerHTML = filtered.map(a => {
const cls = a.level === 'P0' ? 'tag-p0' : (a.level === 'P1' ? 'tag-p1' : 'tag-p2');
const ack = a.ack ? '<span class="ack-badge">已确认</span>' : '<span style="color:#d29922;">未确认</span>';
const reso = a.resolved ? '✅ 已恢复' : (a.ack ? '' : '');
return `<tr>
<td><span class="tag ${cls}">${a.level}</span></td>
<td>${a.ts_iso}</td>
<td>${a.category}</td>
<td><b>${a.title}</b></td>
<td style="max-width:300px;">${a.detail||''}</td>
<td>${ack} ${reso}</td>
</tr>`;
}).join('');
}
refresh();
setInterval(refresh, 30000);
</script>
</body>
</html>
+283
View File
@@ -0,0 +1,283 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>OA 运维监控平台 · 总览</title>
<script src="https://cdn.jsdelivr.net/npm/echarts@5.5.0/dist/echarts.min.js"></script>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'PingFang SC', 'Microsoft YaHei', sans-serif; background: #0d1117; color: #c9d1d9; padding: 16px; }
h1 { font-size: 20px; color: #58a6ff; margin-bottom: 16px; }
.status-bar { display: flex; gap: 12px; margin-bottom: 16px; flex-wrap: wrap; }
.status-chip { background: #161b22; border-radius: 6px; padding: 10px 14px; border: 1px solid #30363d; display: flex; align-items: center; gap: 8px; font-size: 13px; }
.status-chip.ok { border-color: #238636; background: rgba(35,134,54,0.1); }
.status-chip.warn { border-color: #d29922; background: rgba(210,153,34,0.1); }
.status-chip.err { border-color: #f85149; background: rgba(248,81,73,0.1); animation: pulse 2s infinite; }
.status-chip .dot { width: 10px; height: 10px; border-radius: 50%; }
.status-chip.ok .dot { background: #238636; }
.status-chip.warn .dot { background: #d29922; }
.status-chip.err .dot { background: #f85149; }
@keyframes pulse { 0%,100% { opacity: 1; } 50% { opacity: 0.5; } }
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(480px, 1fr)); gap: 16px; }
.card { background: #161b22; border-radius: 8px; border: 1px solid #30363d; padding: 14px; }
.card h2 { font-size: 14px; color: #8b949e; margin-bottom: 10px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; }
.chart { width: 100%; height: 220px; }
table { width: 100%; border-collapse: collapse; font-size: 12px; }
th { background: #21262d; color: #8b949e; padding: 6px 8px; text-align: left; font-weight: 600; }
td { padding: 5px 8px; border-top: 1px solid #21262d; word-break: break-all; }
.tag { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 11px; font-weight: 600; }
.tag-ok { background: rgba(35,134,54,0.2); color: #3fb950; }
.tag-fail { background: rgba(248,81,73,0.2); color: #f85149; }
.tag-p0 { background: rgba(248,81,73,0.25); color: #ff7b72; }
.tag-p1 { background: rgba(210,153,34,0.25); color: #d29922; }
.tag-p2 { background: rgba(56,139,253,0.25); color: #58a6ff; }
.empty { text-align: center; color: #484f58; padding: 20px; font-size: 13px; }
.endpoints-table td:first-child { max-width: 220px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.endpoints-table td.status-ok { color: #3fb950; font-weight: 600; }
.endpoints-table td.status-fail { color: #f85149; font-weight: 600; }
.ts { color: #6e7681; font-size: 11px; }
</style>
</head>
<body>
<h1>🛡️ OA 运维监控平台 · 总览</h1>
<div class="status-bar" id="statusBar">
<div class="status-chip" id="chip-db"><span class="dot"></span><span>DB 层</span><span class="ts" id="ts-db"></span></div>
<div class="status-chip" id="chip-http"><span class="dot"></span><span>HTTP 端点</span><span class="ts" id="ts-http"></span></div>
<div class="status-chip" id="chip-business"><span class="dot"></span><span>业务/钉钉推送</span><span class="ts" id="ts-biz"></span></div>
<div class="status-chip" id="chip-infra"><span class="dot"></span><span>基础设施</span><span class="ts" id="ts-infra"></span></div>
<div class="status-chip err" id="chip-active" style="display:none;"><span class="dot"></span><span id="activeCount"></span> 活跃告警</div>
</div>
<div class="grid">
<div class="card">
<h2>DB · Threads Running / 30s</h2>
<div id="chart-threads" class="chart"></div>
</div>
<div class="card">
<h2>DB · QPS Read / Write</h2>
<div id="chart-qps" class="chart"></div>
</div>
<div class="card">
<h2>DB · 当前活跃长查询(>30s)</h2>
<div id="longQueriesTable"></div>
</div>
<div class="card">
<h2>HTTP · 端点状态(最近 2 分钟)</h2>
<div id="httpTable"></div>
</div>
<div class="card">
<h2>业务 · 钉钉推送小时增量</h2>
<div id="chart-msg" class="chart"></div>
</div>
<div class="card">
<h2>基础设施 · 容器数 / 磁盘 / Docker</h2>
<div id="chart-infra" class="chart"></div>
</div>
<div class="card" style="grid-column: 1/-1;">
<h2>🔴 活跃告警</h2>
<div id="alertsTable"></div>
</div>
</div>
<script>
const API = window.location.origin + '/monitor';
const REFRESH_MS = 15000;
// ---- Charts ----
const chartThreads = echarts.init(document.getElementById('chart-threads'), 'dark');
const chartQps = echarts.init(document.getElementById('chart-qps'), 'dark');
const chartMsg = echarts.init(document.getElementById('chart-msg'), 'dark');
const chartInfra = echarts.init(document.getElementById('chart-infra'), 'dark');
function formatTime(ts) {
const d = new Date(ts * 1000);
return d.toLocaleTimeString('zh-CN', { hour12: false });
}
async function fetchJSON(url) {
const res = await fetch(API + url, { cache: 'no-cache' });
return res.json();
}
async function refresh() {
try {
const overview = await fetchJSON('/api/overview');
const db30 = await fetchJSON('/api/db/recent?minutes=30');
const biz24 = await fetchJSON('/api/business/recent?hours=24');
const infra30 = await fetchJSON('/api/infra/recent?minutes=30');
renderStatus(overview);
renderCharts(db30, biz24, infra30);
renderLongQueries(overview.long_queries);
renderHttpTable(overview.http_summary);
renderAlerts(overview.active_alerts);
} catch (e) {
console.error('refresh fail:', e);
}
}
function renderStatus(o) {
// DB
const dbOk = o.db_latest && o.db_latest.threads_running >= 0;
updateChip('chip-db', 'ts-db', dbOk ? 'ok' : 'err', o.db_latest?.ts_iso);
// HTTP
const httpAll = o.http_summary || [];
const httpOk = httpAll.length > 0 && httpAll.every(e => e.healthy === 1);
updateChip('chip-http', 'ts-http', httpOk ? 'ok' : (httpAll.some(e => !e.healthy) ? 'warn' : 'err'), httpAll[0]?.ts);
// Business
const bizOk = o.biz_latest && o.biz_latest.msg_log_hourly >= 0;
updateChip('chip-business', 'ts-biz', bizOk ? 'ok' : 'warn', o.biz_latest?.ts_iso);
// Infra
const infraOk = o.infra_latest && o.infra_latest.docker_daemon_ok === 1;
const infraClass = infraOk ? 'ok' : 'err';
updateChip('chip-infra', 'ts-infra', infraClass, o.infra_latest?.ts_iso);
// Active alerts
const aa = o.active_alerts || [];
const chipActive = document.getElementById('chip-active');
if (aa.length > 0) {
chipActive.style.display = 'flex';
document.getElementById('activeCount').textContent = aa.length + ' 活跃告警';
} else {
chipActive.style.display = 'none';
}
}
function updateChip(id, tsId, cls, ts) {
const el = document.getElementById(id);
el.className = 'status-chip ' + cls;
if (ts) document.getElementById(tsId).textContent = ts;
}
function renderCharts(db, biz, infra) {
const dbX = db.map(r => formatTime(r.ts));
const dbThreads = db.map(r => r.threads_running);
const dbQpsR = db.map(r => r.qps_read);
const dbQpsW = db.map(r => r.qps_write);
chartThreads.setOption({
tooltip: { trigger: 'axis' },
grid: { left: 40, right: 20, top: 20, bottom: 30 },
xAxis: { type: 'category', data: dbX, axisLine: { lineStyle: { color: '#30363d' } } },
yAxis: { type: 'value', splitLine: { lineStyle: { color: '#21262d' } } },
series: [{ name: 'Threads_running', type: 'line', data: dbThreads, smooth: true,
lineStyle: { color: '#58a6ff' }, areaStyle: { color: 'rgba(88,166,255,0.15)' },
markLine: { data: [{ yAxis: 50, label: '阈值50', lineStyle: { color: '#f85149', type: 'dashed' } }] }
}]
});
chartQps.setOption({
tooltip: { trigger: 'axis' },
legend: { data: ['Read QPS', 'Write QPS'], textStyle: { color: '#c9d1d9' } },
grid: { left: 40, right: 20, top: 30, bottom: 30 },
xAxis: { type: 'category', data: dbX, axisLine: { lineStyle: { color: '#30363d' } } },
yAxis: { type: 'value', splitLine: { lineStyle: { color: '#21262d' } } },
series: [
{ name: 'Read QPS', type: 'line', data: dbQpsR, smooth: true, lineStyle: { color: '#3fb950' } },
{ name: 'Write QPS', type: 'line', data: dbQpsW, smooth: true, lineStyle: { color: '#d29922' } }
]
});
// Biz chart
const bizX = biz.map(r => r.ts_iso?.substring(11,16) || r.ts);
const bizMsg = biz.map(r => r.msg_log_hourly);
chartMsg.setOption({
tooltip: { trigger: 'axis' },
grid: { left: 60, right: 20, top: 20, bottom: 30 },
xAxis: { type: 'category', data: bizX, axisLine: { lineStyle: { color: '#30363d' } } },
yAxis: { type: 'value', splitLine: { lineStyle: { color: '#21262d' } } },
series: [{ name: '小时增量', type: 'bar', data: bizMsg, itemStyle: { color: '#a371f7' },
markLine: { data: [{ yAxis: 100, label: '阈值100', lineStyle: { color: '#f85149', type: 'dashed' } }] }
}]
});
// Infra chart
const infraX = infra.map(r => formatTime(r.ts));
const infraC = infra.map(r => r.docker_container_count);
const infraD = infra.map(r => r.disk_usage_pct);
chartInfra.setOption({
tooltip: { trigger: 'axis' },
legend: { data: ['容器数', '磁盘%'], textStyle: { color: '#c9d1d9' } },
grid: { left: 40, right: 50, top: 30, bottom: 30 },
xAxis: { type: 'category', data: infraX, axisLine: { lineStyle: { color: '#30363d' } } },
yAxis: [
{ type: 'value', name: '容器数', splitLine: { lineStyle: { color: '#21262d' } } },
{ type: 'value', name: '磁盘%', min: 0, max: 100, splitLine: { show: false } }
],
series: [
{ name: '容器数', type: 'line', data: infraC, smooth: true, lineStyle: { color: '#3fb950' }, areaStyle: { color: 'rgba(63,185,80,0.1)' } },
{ name: '磁盘%', type: 'line', yAxisIndex: 1, data: infraD, smooth: true, lineStyle: { color: '#d29922' },
markLine: { data: [{ yAxis: 85, label: '85%', lineStyle: { color: '#f85149', type: 'dashed' } }] }
}
]
});
}
function renderLongQueries(queries) {
const el = document.getElementById('longQueriesTable');
if (!queries || queries.length === 0) {
el.innerHTML = '<div class="empty">✅ 当前无活跃长查询(>30s)</div>';
return;
}
let html = '<table><tr><th>PID</th><th>Time(s)</th><th>User</th><th>State</th><th>SQL</th></tr>';
queries.forEach(q => {
html += `<tr><td>${q.pid}</td><td style="color:#f85149;font-weight:600;">${q.time_sec}</td><td>${q.user}</td><td>${q.state||'-'}</td><td style="max-width:260px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="${q.info||''}">${q.info||''}</td></tr>`;
});
html += '</table>';
el.innerHTML = html;
}
function renderHttpTable(list) {
const el = document.getElementById('httpTable');
if (!list || list.length === 0) {
el.innerHTML = '<div class="empty">等待数据...</div>';
return;
}
let html = '<table class="endpoints-table"><tr><th>端点</th><th>状态</th><th>耗时(ms)</th><th>时间</th></tr>';
const seen = new Set();
list.sort((a, b) => (b.healthy - a.healthy) || (a.endpoint.localeCompare(b.endpoint)));
list.forEach(e => {
if (seen.has(e.endpoint)) return;
seen.add(e.endpoint);
const ok = e.healthy === 1;
html += `<tr><td title="${e.endpoint}">${e.endpoint}</td>
<td class="${ok?'status-ok':'status-fail'}">${ok?'OK ('+e.status_code+')':'FAIL ('+e.status_code+')'}</td>
<td>${e.resp_time_ms}</td><td class="ts">${e.ts}</td></tr>`;
});
html += '</table>';
el.innerHTML = html;
}
function renderAlerts(alerts) {
const el = document.getElementById('alertsTable');
if (!alerts || alerts.length === 0) {
el.innerHTML = '<div class="empty">🎉 当前无活跃告警</div>';
return;
}
let html = '<table><tr><th>级别</th><th>时间</th><th>类别</th><th>标题</th><th>详情</th></tr>';
alerts.forEach(a => {
const cls = a.level === 'P0' ? 'tag-p0' : (a.level === 'P1' ? 'tag-p1' : 'tag-p2');
html += `<tr><td><span class="tag ${cls}">${a.level}</span></td>
<td class="ts">${a.ts_iso}</td>
<td>${a.category}</td>
<td><b>${a.title}</b></td>
<td>${a.detail||''}</td></tr>`;
});
html += '</table>';
el.innerHTML = html;
}
refresh();
setInterval(refresh, REFRESH_MS);
window.addEventListener('resize', () => {
chartThreads.resize(); chartQps.resize(); chartMsg.resize(); chartInfra.resize();
});
</script>
</body>
</html>
+84
View File
@@ -0,0 +1,84 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · SQLite 数据库初始化
# 只建表,不删数据;幂等可重复执行
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
log_info "初始化 SQLite 数据库:${MONITOR_DB}"
sqlite_exec "CREATE TABLE IF NOT EXISTS db_metrics (
ts INTEGER PRIMARY KEY,
threads_running INTEGER,
threads_connected INTEGER,
threads_idle INTEGER,
uptime_sec INTEGER,
qps_read INTEGER,
qps_write INTEGER,
slow_queries INTEGER -- 近 30s 慢查询数(>30s)
);"
sqlite_exec "CREATE TABLE IF NOT EXISTS db_processlist (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts INTEGER,
pid INTEGER,
user TEXT,
state TEXT,
time_sec INTEGER,
info TEXT
);"
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_db_proc_ts ON db_processlist(ts);"
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_db_proc_time ON db_processlist(time_sec);"
sqlite_exec "CREATE TABLE IF NOT EXISTS http_metrics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts INTEGER,
endpoint TEXT,
status_code INTEGER,
resp_time_ms INTEGER,
healthy INTEGER -- 0/1
);"
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_http_ts_ep ON http_metrics(ts, endpoint);"
sqlite_exec "CREATE TABLE IF NOT EXISTS business_metrics (
ts INTEGER PRIMARY KEY,
msg_log_hourly INTEGER, -- 本小时 ecology_message_log 增量
msg_log_fail_rate REAL, -- 失败率
ongoing_requests INTEGER, -- 进行中流程数
overdue_tasks INTEGER -- 超时待办数
);"
sqlite_exec "CREATE TABLE IF NOT EXISTS infra_metrics (
ts INTEGER PRIMARY KEY,
docker_daemon_ok INTEGER, -- 0/1
docker_container_count INTEGER,
docker_root_write_mb REAL, -- 1 分钟 Docker data-root 写入量 MB
disk_usage_pct REAL,
inode_usage_pct REAL,
load_avg REAL
);"
sqlite_exec "CREATE TABLE IF NOT EXISTS alerts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts INTEGER,
level TEXT, -- P0/P1/P2
category TEXT, -- db/http/business/infra
title TEXT,
detail TEXT,
ack INTEGER DEFAULT 0, -- 0=未确认 1=已确认
resolved INTEGER DEFAULT 0 -- 0=未恢复 1=已恢复
);"
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_alerts_ts ON alerts(ts);"
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_alerts_ack ON alerts(ack, resolved);"
sqlite_exec "CREATE TABLE IF NOT EXISTS alert_cooldown (
alert_key TEXT PRIMARY KEY,
last_alert_ts INTEGER
);"
log_info "SQLite schema 初始化完成"
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env bash
# ============================================================================
# OA Monitor · 长查询检测 dry-run KILL(每 10 分钟)
# 默认 DRY_RUN=1(只报告,不真 KILL)。--execute 才真的 KILL
# ============================================================================
set -u
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/../lib/common.sh"
DRY_RUN=1
for arg in "$@"; do
[[ "${arg}" == "--execute" ]] && DRY_RUN=0
[[ "${arg}" == "--dry-run" ]] && DRY_RUN=1
done
NOW=$(date +%s)
db_exec() {
local sql="$1"
MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -e "${sql}" 2>/dev/null
}
log_info "[LONG-CHECK] 开始(DRY_RUN=${DRY_RUN},阈值 ${ALERT_LONG_QUERY_SEC}s)"
RESULT=$(db_exec "SHOW FULL PROCESSLIST;" | awk -F'\t' -v t="${ALERT_LONG_QUERY_SEC}" '
NR==1 { next }
NF < 10 { next }
{
pid=$1; user=$2; state=$6; time=$7; info=$9
if (time+0 >= t+0 && time+0 > 0) {
gsub(/\t/, " ", info)
printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300)
}
}
')
if [[ -z "${RESULT}" ]]; then
log_info "[LONG-CHECK] ✅ 无长查询"
exit 0
fi
COUNT=$(echo "${RESULT}" | wc -l | tr -d ' ')
log_warn "[LONG-CHECK] ⚠️ ${COUNT} 条长查询"
DETAIL=""
while IFS=$'\t' read -r pid time user state info; do
[[ -z "${pid}" ]] && continue
DETAIL="${DETAIL}\n- PID \`${pid}\` **${time}s** \`${user}\` ${state}"
DETAIL="${DETAIL}\n \`\`\`\n ${info}\n \`\`\`"
done <<< "${RESULT}"
MODE_LABEL=$([[ ${DRY_RUN} -eq 1 ]] && echo "🔒 DRY-RUN" || echo "🚨 执行 KILL")
BODY="**${COUNT} 条长查询**(${ALERT_LONG_QUERY_SEC}s 阈值)· ${MODE_LABEL}\n\n${DETAIL}"
dingtalk_alert "P0" "长查询阻塞需处理" "${BODY}"
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询', '${COUNT} 条, dry_run=${DRY_RUN}');"
if [[ ${DRY_RUN} -eq 0 ]]; then
log_warn "[LONG-CHECK] 执行 KILL!"
while IFS=$'\t' read -r pid time user state info; do
[[ -z "${pid}" ]] && continue
db_exec "KILL ${pid};"
log_warn "[LONG-CHECK] KILL ${pid} (${time}s, ${user})"
done <<< "${RESULT}"
else
log_info "[LONG-CHECK] DRY-RUN,不执行 KILL"
fi
+248
View File
@@ -0,0 +1,248 @@
#!/usr/bin/env python3
# ============================================================================
# OA Monitor · Python HTTP API(纯标准库,无第三方依赖)
# 唯一职责:读 SQLite → 吐 JSON
# 只读,绝不写 SQLite;所有 POST/PUT/DELETE 返回 405
# ============================================================================
import http.server, json, sqlite3, os, sys, urllib.parse
from datetime import datetime, timezone
DB_PATH = "/opt/oa-monitor/data/oa-monitor.db"
DASHBOARD_DIR = "/opt/oa-monitor/dashboard"
def get_db():
return sqlite3.connect(DB_PATH, timeout=5)
def ts_to_iso(ts):
if not ts: return None
return datetime.fromtimestamp(int(ts)).strftime('%Y-%m-%d %H:%M:%S')
def fetch_all(sql, params=()):
db = get_db()
db.row_factory = sqlite3.Row
try:
rows = db.execute(sql, params).fetchall()
return [dict(r) for r in rows]
finally:
db.close()
def fetch_one(sql, params=()):
rows = fetch_all(sql, params)
return rows[0] if rows else None
class MonitorHandler(http.server.BaseHTTPRequestHandler):
# ---- CORS + JSON 头 ----
def _send_json(self, data, code=200):
body = json.dumps(data, ensure_ascii=False, default=str).encode()
self.send_response(code)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.send_header('Content-Length', str(len(body)))
self.send_header('Access-Control-Allow-Origin', '*')
self.send_header('Access-Control-Allow-Methods', 'GET, OPTIONS')
self.send_header('Access-Control-Allow-Headers', 'Content-Type')
self.end_headers()
self.wfile.write(body)
def do_OPTIONS(self):
self._send_json({})
# ---- 路由 ----
def do_GET(self):
parsed = urllib.parse.urlparse(self.path)
path = parsed.path.rstrip('/')
qs = urllib.parse.parse_qs(parsed.query)
try:
if path.startswith('/api/'):
self._route_api(path, qs)
else:
self._route_static(path)
except Exception as e:
self._send_json({"error": str(e)}, 500)
def _route_api(self, path, qs):
if path == '/api/health':
self._health()
elif path == '/api/overview':
self._overview()
elif path == '/api/db/recent':
self._db_recent(qs)
elif path == '/api/db/long_queries':
self._db_long_queries(qs)
elif path == '/api/http/recent':
self._http_recent(qs)
elif path == '/api/business/recent':
self._biz_recent(qs)
elif path == '/api/infra/recent':
self._infra_recent(qs)
elif path == '/api/alerts':
self._alerts(qs)
else:
self._send_json({"error": "not found", "path": path}, 404)
# ---- 静态文件服务 ----
def _route_static(self, path):
# 安全:禁止 .. 和 /etc 等
if '..' in path or path.startswith('/etc'):
self._send_json({"error": "forbidden"}, 403); return
# 根路径 → index.html
if path == '' or path == '/':
path = '/index.html'
filepath = os.path.join(DASHBOARD_DIR, path.lstrip('/'))
# 双重安全
if not filepath.startswith(DASHBOARD_DIR):
self._send_json({"error": "forbidden"}, 403); return
if not os.path.isfile(filepath):
self._send_json({"error": "not found", "path": path}, 404); return
with open(filepath, 'rb') as f:
content = f.read()
ext = os.path.splitext(filepath)[1].lower()
mime = {
'.html': 'text/html; charset=utf-8',
'.css': 'text/css; charset=utf-8',
'.js': 'application/javascript; charset=utf-8',
'.json': 'application/json; charset=utf-8',
'.svg': 'image/svg+xml',
'.png': 'image/png',
'.ico': 'image/x-icon',
}.get(ext, 'application/octet-stream')
self.send_response(200)
self.send_header('Content-Type', mime)
self.send_header('Content-Length', str(len(content)))
self.send_header('Access-Control-Allow-Origin', '*')
self.send_header('Cache-Control', 'no-cache')
self.end_headers()
self.wfile.write(content)
def do_POST(self): self._send_json({"error": "read-only"}, 405)
def do_PUT(self): self._send_json({"error": "read-only"}, 405)
def do_DELETE(self): self._send_json({"error": "read-only"}, 405)
def log_message(self, fmt, *args):
pass # 静默日志
# ---- Endpoint 实现 ----
def _health(self):
db_ok = os.path.exists(DB_PATH)
db_size = os.path.getsize(DB_PATH) if db_ok else 0
# 最新写入时间
last = fetch_one("SELECT MAX(ts) AS last_ts FROM db_metrics")
self._send_json({
"service": "oa-monitor-api",
"db_exists": db_ok,
"db_size_bytes": db_size,
"latest_data_ts": ts_to_iso(last['last_ts'] if last else None),
"now": ts_to_iso(int(datetime.now().timestamp()))
})
def _overview(self):
# 各层最新一条
db = fetch_one("SELECT * FROM db_metrics ORDER BY ts DESC LIMIT 1")
http_all = fetch_all("SELECT endpoint, status_code, resp_time_ms, healthy, ts FROM http_metrics WHERE ts > ? ORDER BY ts DESC", ((datetime.now().timestamp() - 120),))
biz = fetch_one("SELECT * FROM business_metrics ORDER BY ts DESC LIMIT 1")
infra = fetch_one("SELECT * FROM infra_metrics ORDER BY ts DESC LIMIT 1")
# 活跃告警
active_alerts = fetch_all("SELECT * FROM alerts WHERE ack = 0 ORDER BY ts DESC")
# HTTP 端点最新状态聚合
http_summary = []
for row in http_all:
http_summary.append({
"endpoint": row["endpoint"],
"status_code": row["status_code"],
"resp_time_ms": row["resp_time_ms"],
"healthy": row["healthy"],
"ts": ts_to_iso(row["ts"])
})
# DB 层最近长查询
long_q = fetch_all("SELECT * FROM db_processlist WHERE time_sec >= 30 ORDER BY time_sec DESC LIMIT 5")
self._send_json({
"db_latest": {
**({} if not db else db),
"ts_iso": ts_to_iso(db["ts"]) if db else None
},
"http_summary": http_summary,
"biz_latest": {
**({} if not biz else biz),
"ts_iso": ts_to_iso(biz["ts"]) if biz else None
},
"infra_latest": {
**({} if not infra else infra),
"ts_iso": ts_to_iso(infra["ts"]) if infra else None
},
"active_alerts": [{**a, "ts_iso": ts_to_iso(a["ts"])} for a in active_alerts],
"long_queries": [{**q, "ts_iso": ts_to_iso(q["ts"])} for q in long_q]
})
def _db_recent(self, qs):
minutes = int(qs.get('minutes', [30])[0])
since = int(datetime.now().timestamp()) - minutes * 60
rows = fetch_all(
"SELECT * FROM db_metrics WHERE ts >= ? ORDER BY ts ASC",
(since,)
)
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
def _db_long_queries(self, qs):
since_min = int(qs.get('since_minutes', [60])[0])
since = int(datetime.now().timestamp()) - since_min * 60
rows = fetch_all(
"SELECT * FROM db_processlist WHERE ts >= ? ORDER BY time_sec DESC LIMIT 100",
(since,)
)
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
def _http_recent(self, qs):
minutes = int(qs.get('minutes', [30])[0])
since = int(datetime.now().timestamp()) - minutes * 60
rows = fetch_all(
"SELECT * FROM http_metrics WHERE ts >= ? ORDER BY ts ASC",
(since,)
)
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
def _biz_recent(self, qs):
hours = int(qs.get('hours', [24])[0])
since = int(datetime.now().timestamp()) - hours * 3600
rows = fetch_all(
"SELECT * FROM business_metrics WHERE ts >= ? ORDER BY ts ASC",
(since,)
)
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
def _infra_recent(self, qs):
minutes = int(qs.get('minutes', [30])[0])
since = int(datetime.now().timestamp()) - minutes * 60
rows = fetch_all(
"SELECT * FROM infra_metrics WHERE ts >= ? ORDER BY ts ASC",
(since,)
)
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
def _alerts(self, qs):
limit = int(qs.get('limit', [100])[0])
only_unack = qs.get('unack', ['0'])[0]
sql = "SELECT * FROM alerts"
params = []
if only_unack == '1':
sql += " WHERE ack = 0"
sql += " ORDER BY ts DESC LIMIT ?"
params.append(limit)
rows = fetch_all(sql, tuple(params))
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
# ---- 启动 ----
if __name__ == '__main__':
port = int(sys.argv[1]) if len(sys.argv) > 1 else 8455
server = http.server.HTTPServer(('0.0.0.0', port), MonitorHandler)
print(f"OA Monitor API listening on :{port} (DB={DB_PATH})", flush=True)
server.serve_forever()
+1
View File
@@ -0,0 +1 @@
1231
+1 -1
View File
@@ -1 +1 @@
13286 9890 1793773202 30952 16490 1793792402