diff --git a/api/server.py b/api/server.py new file mode 100755 index 0000000..1e947a3 --- /dev/null +++ b/api/server.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +# ============================================================================ +# OA Monitor · Python HTTP API(纯标准库,无第三方依赖) +# 唯一职责:读 SQLite → 吐 JSON + 静态 Dashboard +# 只读,绝不写 SQLite;所有 POST/PUT/DELETE 返回 405 +# ============================================================================ +import http.server, json, sqlite3, os, sys, urllib.parse, base64 +from datetime import datetime, timezone + +DB_PATH = "/opt/oa-monitor/data/oa-monitor.db" +DASHBOARD_DIR = "/opt/oa-monitor/dashboard" + +# ---- HTTP Basic Auth ---- +# 只允许 oamonitor 用户 +AUTH_USER = "oamonitor" +AUTH_PASS = "1mcAi/rCtrZZkz3To2BD" +AUTH_REALM = "OA Monitor" + +def check_auth(headers): + """检查 Authorization 头,返回 bool""" + auth = headers.get('Authorization', '') + if not auth.startswith('Basic '): + return False + try: + decoded = base64.b64decode(auth[6:]).decode() + user, _, pwd = decoded.partition(':') + return user == AUTH_USER and pwd == AUTH_PASS + except Exception: + return False + +def send_401(handler): + body = b'Unauthorized: OA Monitor - HTTP Basic Auth required (user: oamonitor)' + handler.send_response(401) + handler.send_header('WWW-Authenticate', f'Basic realm="{AUTH_REALM}"') + handler.send_header('Content-Type', 'text/plain; charset=utf-8') + handler.send_header('Content-Length', str(len(body))) + handler.send_header('Access-Control-Allow-Origin', '*') + handler.end_headers() + handler.wfile.write(body) + +def get_db(): + return sqlite3.connect(DB_PATH, timeout=5) + +def ts_to_iso(ts): + if not ts: return None + return datetime.fromtimestamp(int(ts)).strftime('%Y-%m-%d %H:%M:%S') + +def fetch_all(sql, params=()): + db = get_db() + db.row_factory = sqlite3.Row + try: + rows = db.execute(sql, params).fetchall() + return [dict(r) for r in rows] + finally: + db.close() + +def fetch_one(sql, params=()): + rows = fetch_all(sql, params) + return rows[0] if rows else None + +class MonitorHandler(http.server.BaseHTTPRequestHandler): + # ---- CORS + JSON 头 ---- + def _send_json(self, data, code=200): + body = json.dumps(data, ensure_ascii=False, default=str).encode() + self.send_response(code) + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.send_header('Content-Length', str(len(body))) + self.send_header('Access-Control-Allow-Origin', '*') + self.send_header('Access-Control-Allow-Methods', 'GET, OPTIONS') + self.send_header('Access-Control-Allow-Headers', 'Content-Type') + self.end_headers() + self.wfile.write(body) + + def do_OPTIONS(self): + self._send_json({}) + + # ---- 路由 ---- + def do_GET(self): + # HTTP Basic Auth(全路径保护,包括 Dashboard 静态文件 + API) + if not check_auth(self.headers): + send_401(self) + return + + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip('/') + qs = urllib.parse.parse_qs(parsed.query) + + try: + if path.startswith('/api/'): + self._route_api(path, qs) + else: + self._route_static(path) + except Exception as e: + self._send_json({"error": str(e)}, 500) + + def _route_api(self, path, qs): + if path == '/api/health': + self._health() + elif path == '/api/overview': + self._overview() + elif path == '/api/db/recent': + self._db_recent(qs) + elif path == '/api/db/long_queries': + self._db_long_queries(qs) + elif path == '/api/http/recent': + self._http_recent(qs) + elif path == '/api/business/recent': + self._biz_recent(qs) + elif path == '/api/infra/recent': + self._infra_recent(qs) + elif path == '/api/alerts': + self._alerts(qs) + else: + self._send_json({"error": "not found", "path": path}, 404) + + # ---- 静态文件服务 ---- + def _route_static(self, path): + # 安全:禁止 .. 和 /etc 等 + if '..' in path or path.startswith('/etc'): + self._send_json({"error": "forbidden"}, 403); return + + # 根路径 → index.html + if path == '' or path == '/': + path = '/index.html' + + filepath = os.path.join(DASHBOARD_DIR, path.lstrip('/')) + # 双重安全 + if not filepath.startswith(DASHBOARD_DIR): + self._send_json({"error": "forbidden"}, 403); return + + if not os.path.isfile(filepath): + self._send_json({"error": "not found", "path": path}, 404); return + + with open(filepath, 'rb') as f: + content = f.read() + + ext = os.path.splitext(filepath)[1].lower() + mime = { + '.html': 'text/html; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.js': 'application/javascript; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.svg': 'image/svg+xml', + '.png': 'image/png', + '.ico': 'image/x-icon', + }.get(ext, 'application/octet-stream') + + self.send_response(200) + self.send_header('Content-Type', mime) + self.send_header('Content-Length', str(len(content))) + self.send_header('Access-Control-Allow-Origin', '*') + self.send_header('Cache-Control', 'no-cache') + self.end_headers() + self.wfile.write(content) + + def do_POST(self): self._send_json({"error": "read-only"}, 405) + def do_PUT(self): self._send_json({"error": "read-only"}, 405) + def do_DELETE(self): self._send_json({"error": "read-only"}, 405) + + def log_message(self, fmt, *args): + pass # 静默日志 + + # ---- Endpoint 实现 ---- + def _health(self): + db_ok = os.path.exists(DB_PATH) + db_size = os.path.getsize(DB_PATH) if db_ok else 0 + # 最新写入时间 + last = fetch_one("SELECT MAX(ts) AS last_ts FROM db_metrics") + self._send_json({ + "service": "oa-monitor-api", + "db_exists": db_ok, + "db_size_bytes": db_size, + "latest_data_ts": ts_to_iso(last['last_ts'] if last else None), + "now": ts_to_iso(int(datetime.now().timestamp())) + }) + + def _overview(self): + # 各层最新一条 + db = fetch_one("SELECT * FROM db_metrics ORDER BY ts DESC LIMIT 1") + http_all = fetch_all("SELECT endpoint, status_code, resp_time_ms, healthy, ts FROM http_metrics WHERE ts > ? ORDER BY ts DESC", ((datetime.now().timestamp() - 120),)) + biz = fetch_one("SELECT * FROM business_metrics ORDER BY ts DESC LIMIT 1") + infra = fetch_one("SELECT * FROM infra_metrics ORDER BY ts DESC LIMIT 1") + + # 活跃告警 + active_alerts = fetch_all("SELECT * FROM alerts WHERE ack = 0 ORDER BY ts DESC") + + # HTTP 端点最新状态聚合 + http_summary = [] + for row in http_all: + http_summary.append({ + "endpoint": row["endpoint"], + "status_code": row["status_code"], + "resp_time_ms": row["resp_time_ms"], + "healthy": row["healthy"], + "ts": ts_to_iso(row["ts"]) + }) + + # DB 层最近长查询 + long_q = fetch_all("SELECT * FROM db_processlist WHERE time_sec >= 30 ORDER BY time_sec DESC LIMIT 5") + + self._send_json({ + "db_latest": { + **({} if not db else db), + "ts_iso": ts_to_iso(db["ts"]) if db else None + }, + "http_summary": http_summary, + "biz_latest": { + **({} if not biz else biz), + "ts_iso": ts_to_iso(biz["ts"]) if biz else None + }, + "infra_latest": { + **({} if not infra else infra), + "ts_iso": ts_to_iso(infra["ts"]) if infra else None + }, + "active_alerts": [{**a, "ts_iso": ts_to_iso(a["ts"])} for a in active_alerts], + "long_queries": [{**q, "ts_iso": ts_to_iso(q["ts"])} for q in long_q] + }) + + def _db_recent(self, qs): + minutes = int(qs.get('minutes', [30])[0]) + since = int(datetime.now().timestamp()) - minutes * 60 + rows = fetch_all( + "SELECT * FROM db_metrics WHERE ts >= ? ORDER BY ts ASC", + (since,) + ) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + + def _db_long_queries(self, qs): + since_min = int(qs.get('since_minutes', [60])[0]) + since = int(datetime.now().timestamp()) - since_min * 60 + rows = fetch_all( + "SELECT * FROM db_processlist WHERE ts >= ? ORDER BY time_sec DESC LIMIT 100", + (since,) + ) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + + def _http_recent(self, qs): + minutes = int(qs.get('minutes', [30])[0]) + since = int(datetime.now().timestamp()) - minutes * 60 + rows = fetch_all( + "SELECT * FROM http_metrics WHERE ts >= ? ORDER BY ts ASC", + (since,) + ) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + + def _biz_recent(self, qs): + hours = int(qs.get('hours', [24])[0]) + since = int(datetime.now().timestamp()) - hours * 3600 + rows = fetch_all( + "SELECT * FROM business_metrics WHERE ts >= ? ORDER BY ts ASC", + (since,) + ) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + + def _infra_recent(self, qs): + minutes = int(qs.get('minutes', [30])[0]) + since = int(datetime.now().timestamp()) - minutes * 60 + rows = fetch_all( + "SELECT * FROM infra_metrics WHERE ts >= ? ORDER BY ts ASC", + (since,) + ) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + + def _alerts(self, qs): + limit = int(qs.get('limit', [100])[0]) + only_unack = qs.get('unack', ['0'])[0] + sql = "SELECT * FROM alerts" + params = [] + if only_unack == '1': + sql += " WHERE ack = 0" + sql += " ORDER BY ts DESC LIMIT ?" + params.append(limit) + rows = fetch_all(sql, tuple(params)) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + +# ---- 启动 ---- +if __name__ == '__main__': + port = int(sys.argv[1]) if len(sys.argv) > 1 else 8455 + server = http.server.HTTPServer(('0.0.0.0', port), MonitorHandler) + print(f"OA Monitor API listening on :{port} (DB={DB_PATH})", flush=True) + server.serve_forever() diff --git a/bin/check-alert.sh b/bin/check-alert.sh index 2f0e87b..73244c3 100755 --- a/bin/check-alert.sh +++ b/bin/check-alert.sh @@ -4,6 +4,9 @@ # v2 修复(2026-10-28): # - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警 # - Step 3 业务层检测被 Step 0 结果门禁 +# v3 修复(2026-11-04): +# - 业务层 P0 推送断档假阳性:sqlite_exec 吞错误(2>/dev/null)→ 查询失败返回空串 → :-0 默认为 0 → 误报 +# - 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,查询失败跳过检测并 warn # ============================================================================ set -u SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -36,7 +39,7 @@ else if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟 BIZ_FRESH=false HUMAN_AGE=$((BIZ_AGE / 60)) - BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec \"SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')\") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec \"SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')\") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警" + BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec "SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec "SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警" dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}" log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)" else @@ -94,34 +97,52 @@ fi if [[ "${BIZ_FRESH}" != "true" ]]; then log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)" else - # 数据新鲜 → 正常业务层判断 - HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1) - ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1) - HOUR_MSG=${HOUR_MSG:-0} - ONGOING=${ONGOING:-0} + # v3 修复:显式调用 sqlite3 + 检查退出码 + 正则校验,区分"查询失败"和"结果为 0" + MSG_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null) + MSG_RC=$? + ONGOING_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null) + ONGOING_RC=$? - CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal - WORK_HOURS_ENABLED=false - if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then - WORK_HOURS_ENABLED=true + # 双重校验:退出码必须为 0,且返回值非空且为纯数字 + BIZ_QUERY_OK=true + if [[ ${MSG_RC} -ne 0 ]] || [[ -z "${MSG_RAW}" ]] || [[ ! "${MSG_RAW}" =~ ^[0-9]+$ ]]; then + log_warn "[ALERT] SQLite msg_log_hourly 查询异常 rc=${MSG_RC} raw='${MSG_RAW}',跳过业务层(防假阳性)" + BIZ_QUERY_OK=false + fi + if [[ ${ONGOING_RC} -ne 0 ]] || [[ -z "${ONGOING_RAW}" ]] || [[ ! "${ONGOING_RAW}" =~ ^[0-9]+$ ]]; then + log_warn "[ALERT] SQLite ongoing_requests 查询异常 rc=${ONGOING_RC} raw='${ONGOING_RAW}',跳过业务层(防假阳性)" + BIZ_QUERY_OK=false fi - if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then - if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then - BODY="**🔴 P0 钉钉推送完全断档!**\n\n" - BODY="${BODY}| 指标 | 值 |\n|---|---|\n" - BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n" - BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n" - BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n" - BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n" - BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n" - BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n" - BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**" - dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}" - sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');" - fi + if [[ "${BIZ_QUERY_OK}" != "true" ]]; then + log_warn "[ALERT] ⏸️ 业务层数据查询异常,跳过钉钉推送断档 P0 检测" else - log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警" + HOUR_MSG=${MSG_RAW} + ONGOING=${ONGOING_RAW} + + CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal + WORK_HOURS_ENABLED=false + if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then + WORK_HOURS_ENABLED=true + fi + + if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then + if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then + BODY="**🔴 P0 钉钉推送完全断档!**\n\n" + BODY="${BODY}| 指标 | 值 |\n|---|---|\n" + BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n" + BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n" + BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n" + BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n" + BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n" + BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n" + BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**" + dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');" + fi + else + log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警" + fi fi fi diff --git a/bin/fix-info-read-dummy.py b/bin/fix-info-read-dummy.py new file mode 100755 index 0000000..18f4772 --- /dev/null +++ b/bin/fix-info-read-dummy.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""fix-info-read-dummy.py — 补 DUMMY 根治 updateBizState 死循环 (Python 3.6 兼容)""" +import sys, os, time, subprocess, json, urllib.request + +DB_HOST, DB_PORT, DB_USER = "39.100.72.102", 3306, "ecology" +DB_PASS, DB_NAME = "Z!BYT+jSDDyYHMdX+qFSLzJCYXW", "ecology" +OA_HOST, OA_USER, OA_PASS = "39.99.236.138", "root", "KTOFKjVkVOGlNgvs8yUP1+PM1VM" +DINGTALK = "https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249" +BATCH = 5000 +LOG = "/opt/oa-monitor/logs/fix-info-read.log" + +# === 正确值(从之前 DUMMY_FIX_ 样例提取)=== +MESSAGETYPE = 25 # int!不是 'archive' +BIZTYPE = 86 # int!上次成功用的 +STATE = 2 # 已读 +BIZSTATE = 1 # 已更新(关键!让 updateBizState 不再重试) +GROUPTYPE = "archive-fix" +BIZTITLE = "dummy fix" + +def log(msg, level="INFO"): + line = "[%s] [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S"), level, msg) + print(line, flush=True) + with open(LOG, "a") as f: + f.write(line + "\n") + +def run_mysql(sql, fetch=True): + env = os.environ.copy(); env["MYSQL_PWD"] = DB_PASS + r = subprocess.run(["mysql","-h"+DB_HOST,"-P"+str(DB_PORT),"-u"+DB_USER,DB_NAME,"-N","-e",sql], + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=600) + if r.returncode != 0: + log("MySQL ERROR: " + r.stderr.decode().strip(), "ERROR") + return None + if not fetch: return True + return r.stdout.decode().strip() + +def dingtalk(level, title, body=""): + text = "**[%s] %s**\n\n%s\n\n> OA DUMMY @ %s" % (level, title, body, time.strftime("%Y-%m-%d %H:%M:%S")) + try: + urllib.request.urlopen(urllib.request.Request(DINGTALK, json.dumps({"msgtype":"markdown","markdown":{"title":title,"text":text}}).encode(), {"Content-Type":"application/json"}), timeout=5) + except Exception as e: log("钉钉: "+str(e), "WARN") + +def check_http(): + env = os.environ.copy() + cmd = ("python3 -c 'import paramiko; ssh=paramiko.SSHClient(); ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()); " + "ssh.connect(\"%s\",username=\"%s\",password=\"%s\",timeout=15); " + "_,o,_=ssh.exec_command(\"curl -s -o /dev/null -w %%{http_code} http://127.0.0.1:80/\"); " + "print(o.read().decode(\"utf-8\",\"replace\")); ssh.close()'" % (OA_HOST, OA_USER, OA_PASS)) + r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=25, env=env) + return "200" in r.stdout.decode().strip() + +def check_tr(): + r = run_mysql("SHOW GLOBAL STATUS WHERE Variable_name='Threads_running';") + if not r: return 999 + try: return int(r.split("\t")[-1].strip()) + except: return 999 + +def wait_safe(max_wait=120): + for i in range(max_wait // 5): + if check_http() and check_tr() < 30: return True + log("等待恢复 (%ds) HTTP=%s TR=%d" % (i*5, check_http(), check_tr()), "WARN") + time.sleep(5) + return False + +# === Phase 0 === +log("===== Phase 0: 前置检查 =====") +DRY = "--dry-run" in sys.argv +if DRY: log("*** DRY RUN ***") + +ok = check_http(); log("Resin HTTP 80 = %s" % ok) +if not ok: log("Resin 不通!中止!", "CRITICAL"); sys.exit(1) + +tr = check_tr(); log("Threads_running = %d" % tr) +if tr >= 15: + wait_safe(180); tr = check_tr() + +existing = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_FIX_%';") +log("现有 DUMMY_FIX = %s" % existing) + +log("Phase 1: 备份表...") +run_mysql("CREATE TABLE IF NOT EXISTS ecology_message_info_read_bak_20261104 AS SELECT * FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';", fetch=False) + +dingtalk("INFO", "DUMMY 补数据启动", "HTTP=200 ✅\nTR=%d\nDUMMY_FIX=%s" % (tr, existing)) + +# === Phase 2 === +total = [0] + +def insert_user(uid, prefix, col): + last_id = 0; batch = 0; user_total = 0 + while True: + sql = ("INSERT INTO ecology_message_info_read " + "(ID, MESSAGEID, MESSAGETYPE, USERID, TARGETID, STATE, BIZSTATE, " + "CREATEDATE, CREATETIME, MESSAGEGROUPTYPE, BIZTITLE, BIZTYPE) " + "SELECT CONCAT('%s%s_', REQUESTID), 0, %d, %d, REQUESTID, %d, %d, " + "CURDATE(), CURTIME(), '%s', '%s', %d " + "FROM workflow_requestbase " + "WHERE %s = %d AND STATUS LIKE '%%归档%%' AND DELETED = 0 AND REQUESTID > %d " + "ORDER BY REQUESTID LIMIT %d;" % + (prefix, "%d", MESSAGETYPE, uid, STATE, BIZSTATE, GROUPTYPE, BIZTITLE, BIZTYPE, + col, uid, last_id, BATCH)) + if DRY: + batch += 1; user_total += BATCH; last_id += BATCH + if batch >= 5: break + time.sleep(0.2); continue + + run_mysql(sql, fetch=False) + rc = run_mysql("SELECT ROW_COUNT();") + inserted = int(rc) if rc and rc.isdigit() else 0 + if inserted == 0: break + + last_sql = "SELECT MAX(REQUESTID) FROM workflow_requestbase WHERE %s = %d AND REQUESTID > %d LIMIT %d;" % (col, uid, last_id, BATCH) + lv = run_mysql(last_sql) + if lv and lv.isdigit(): last_id = int(lv) + + batch += 1; user_total += inserted; total[0] += inserted + log(" User=%d batch=%d +%d rows (total=%d)" % (uid, batch, inserted, total[0])) + + if not check_http() or check_tr() >= 30: + log("异常!暂停...", "WARN"); wait_safe(10) + time.sleep(1) + + if user_total > 0: + log("User=%d 完成: %d 条 %d 批" % (uid, user_total, batch)) + +# Phase 2a — 全量业务用户(不限 TOP 100) +log("===== Phase 2a: CREATER (全量) =====") +creaters_raw = run_mysql("SELECT CREATER, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND CREATER!=1 GROUP BY CREATER ORDER BY COUNT(*) DESC;") +creaters = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in creaters_raw.strip().split("\n") if l.strip()] +log("CREATER 用户=%d 归档总量=%d" % (len(creaters), sum(c for _,c in creaters))) + +for i, (uid, tot) in enumerate(creaters, 1): + log("--- CREATER #%d/100 uid=%d 归档=%d ---" % (i, uid, tot)) + insert_user(uid, "DUMMY_B_CREATER_", "CREATER") + if i % 10 == 0: + dingtalk("INFO", "CREATER %d/100" % i, "已插入=%d" % total[0]) + +# Phase 2b — 全量业务用户(不限 TOP 100) +log("===== Phase 2b: LASTOPERATOR (全量) =====") +ops_raw = run_mysql("SELECT LASTOPERATOR, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND LASTOPERATOR!=1 GROUP BY LASTOPERATOR ORDER BY COUNT(*) DESC;") +ops = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in ops_raw.strip().split("\n") if l.strip()] +log("OPERATOR 用户=%d 归档总量=%d" % (len(ops), sum(c for _,c in ops))) + +for i, (uid, tot) in enumerate(ops, 1): + log("--- OPERATOR #%d/100 uid=%d 归档=%d ---" % (i, uid, tot)) + insert_user(uid, "DUMMY_B_OPERATOR_", "LASTOPERATOR") + if i % 10 == 0: + dingtalk("INFO", "OPERATOR %d/100" % i, "已插入=%d" % total[0]) + +# Phase 3 +log("===== Phase 3: 验证 =====") +fc = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';") +log("DUMMY_B 总数 = %s" % fc) +http_ok = check_http(); log("Resin HTTP = %s" % http_ok) +tr2 = check_tr(); log("Threads_running = %d" % tr2) +lm = run_mysql("SELECT MAX(CREATETIME) FROM ecology_message_log WHERE CREATEDATE=CURDATE();") +log("message_log 最新 = %s" % lm) + +dingtalk("P0", "DUMMY 补数据完成 ✅", + "**DUMMY_B**: %s 条\n**HTTP**: %s\n**TR**: %d\n**最新消息**: %s\n\n回滚: DELETE FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%%';" % + (fc, http_ok, tr2, lm)) +log("===== 全部完成 =====") diff --git a/check-alert.sh b/check-alert.sh new file mode 100644 index 0000000..2cb8738 --- /dev/null +++ b/check-alert.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 告警检测(每 1 分钟) +# 遍历所有指标,超过阈值触发钉钉告警 +# 告警冷却由 common.sh 的 alert_cooldown 表处理(10 分钟同类型不重复) +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) +log_info "[ALERT] ========== 告警检测开始 ==========" + +# ============================================================================ +# 1. DB 层检测 +# ============================================================================ +# Threads_running 过高 +TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;") +if [[ -n "${TR}" ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then + BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;')\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;')" + dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');" +fi + +# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的) +LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;") +if [[ -n "${LONG_QUERIES}" ]]; then + COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ') + DETAIL="" + while IFS='|' read -r pid time user state info; do + DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`" + done <<< "${LONG_QUERIES}" + BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理" + dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');" +fi + +# ============================================================================ +# 2. HTTP 端点检测 +# ============================================================================ +# 最近 3 次的失败次数 +FAIL_COUNT=0 +FAIL_DETAIL="" +for endpoint in "${OA_ENDPOINTS[@]}"; do + FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;") + FAIL_HERE=${FAIL_HERE:-0} + if [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then + FAIL_COUNT=$((FAIL_COUNT + 1)) + LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;") + FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}" + fi +done + +if [[ ${FAIL_COUNT} -gt 0 ]]; then + BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态" + dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');" +fi + +# ============================================================================ +# 3. 业务层检测 +# ============================================================================ +# message_log 小时增量过低(钉钉推送可能断了) +HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;") +if [[ -n "${HOUR_MSG}" ]] && [[ ${HOUR_MSG} -lt ${ALERT_MSG_LOG_HOUR_MIN} ]] && [[ ${HOUR_MSG} -gt 0 ]]; then + BODY="**ecology_message_log 小时增量仅 ${HOUR_MSG} 条**(阈值 ${ALERT_MSG_LOG_HOUR_MIN})\n\n> 钉钉推送可能阻塞。泛微云桥 e-Bridge 为黑盒线程,需登录 OA 后台检查「消息推送」日志或用 SHOW PROCESSLIST 看 active 线程" + dingtalk_alert "P1" "钉钉推送疑似停滞" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'business', '钉钉推送疑似停滞', 'msg_hourly=${HOUR_MSG}');" +fi + +# ============================================================================ +# 4. 基础设施层检测 +# ============================================================================ +# Docker daemon 挂了 +DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;") +if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then + BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> ⚠️ 今天的 rm -rf /ipaas 事故中 Docker daemon 重启过。请立即登录 DSH 检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`" + dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');" +fi + +# 容器数突变(对比最近 30 分钟的平均数) +CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;") +if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]]; then + DELTA=$(( CURR_COUNT - ${AVG_COUNT%%.*} )) + ABS_DELTA=${DELTA#-} + if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then + BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a" + dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');" + fi +fi + +# 磁盘 +DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +if [[ -n "${DISK_PCT}" ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then + BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)\n\n> 请检查:df -h → du -sh /var/lib/docker/* → pm2 logs 清理 → 旧 SQLite 数据归档" + dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');" +fi + +# inode +INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +if [[ -n "${INODE_PCT}" ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then + BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)\n\n> 可能有大量小文件,检查 find / -type f | wc -l" + dingtalk_alert "P1" "DSH inode 告警" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');" +fi + +# Docker root 写入突增 +ROOT_WRITE=$(sqlite_exec "SELECT docker_root_write_mb FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +if [[ -n "${ROOT_WRITE}" ]] && (( ${ROOT_WRITE%%.*} >= ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN )); then + BODY="**Docker data-root 1 分钟内写入 ${ROOT_WRITE}MB**(阈值 ${ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN}MB)\n\n> 可能有容器在疯狂写数据,检查:docker stats → docker logs" + dingtalk_alert "P2" "Docker data-root 写入突增" "${BODY}" + sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P2', 'infra', 'Docker root 写入突增', 'write_mb=${ROOT_WRITE}');" +fi + +log_info "[ALERT] ========== 告警检测完成 ==========" diff --git a/cleanup.sh b/cleanup.sh new file mode 100644 index 0000000..3332145 --- /dev/null +++ b/cleanup.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 数据清理(每天 03:00) +# 保留最近 7 天指标,alerts 和 alert_cooldown 保留 30 天 +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) +RETENTION_7D=$(( NOW - 7 * 86400 )) +RETENTION_30D=$(( NOW - 30 * 86400 )) + +log_info "[CLEANUP] 开始清理,阈值 ts < ${RETENTION_7D}(7天),alerts < ${RETENTION_30D}(30天)" + +# 各层指标表 +for table in db_metrics http_metrics business_metrics infra_metrics db_processlist; do + BEFORE=$(sqlite_exec "SELECT COUNT(*) FROM ${table};") + sqlite_exec "DELETE FROM ${table} WHERE ts < ${RETENTION_7D};" + AFTER=$(sqlite_exec "SELECT COUNT(*) FROM ${table};") + log_info "[CLEANUP] ${table}: ${BEFORE} → ${AFTER}(删了 $((BEFORE - AFTER)) 条)" +done + +# alerts 保留 30 天 +BEFORE=$(sqlite_exec "SELECT COUNT(*) FROM alerts;") +sqlite_exec "DELETE FROM alerts WHERE ts < ${RETENTION_30D};" +AFTER=$(sqlite_exec "SELECT COUNT(*) FROM alerts;") +log_info "[CLEANUP] alerts: ${BEFORE} → ${AFTER}(删了 $((BEFORE - AFTER)) 条)" + +# VACUUM 压缩(可选,不频繁) +if (( NOW % (86400 * 7) < 3600 )); then + log_info "[CLEANUP] 每周 VACUUM SQLite..." + sqlite_exec "VACUUM;" +fi + +log_info "[CLEANUP] 完成" diff --git a/collect-business.sh b/collect-business.sh new file mode 100644 index 0000000..153eda2 --- /dev/null +++ b/collect-business.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 业务指标采集(每 1 小时) +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) + +db_exec() { + local sql="$1" + MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -e "${sql}" 2>/dev/null +} + +MSG_TOTAL=$(db_exec "SELECT COUNT(*) FROM ecology_message_log;") +MSG_TOTAL=${MSG_TOTAL:-0} +MSG_FAIL_HOUR=$(db_exec "SELECT COUNT(*) FROM ecology_message_log WHERE retcode != 0 AND operTime >= DATE_SUB(NOW(), INTERVAL 1 HOUR);") +MSG_FAIL_HOUR=${MSG_FAIL_HOUR:-0} +ONGOING=$(db_exec "SELECT COUNT(*) FROM workflow_requestbase WHERE status = 2 OR (status IS NULL AND currentnodeid > 0);") +ONGOING=${ONGOING:-0} +OVERDUE=$(db_exec "SELECT COUNT(*) FROM workflow_requestlog WHERE timeout = 1 AND operatedate >= DATE_SUB(NOW(), INTERVAL 1 DAY);") +OVERDUE=${OVERDUE:-0} + +if [[ ${MSG_TOTAL} -gt 0 ]]; then + FAIL_RATE=$(echo "scale=4; ${MSG_FAIL_HOUR} / (${MSG_TOTAL} + 0.001);" | bc 2>/dev/null || echo "0.0") +else + FAIL_RATE="0.0" +fi + +sqlite_exec "INSERT INTO business_metrics VALUES ( + ${NOW}, ${MSG_TOTAL}, ${FAIL_RATE}, ${ONGOING}, ${OVERDUE} +);" + +log_info "[BIZ] ✅ msg_total=${MSG_TOTAL} fail_hour=${MSG_FAIL_HOUR} fail_rate=${FAIL_RATE} ongoing=${ONGOING} overdue=${OVERDUE}" diff --git a/collect-db.sh b/collect-db.sh new file mode 100644 index 0000000..8d5673f --- /dev/null +++ b/collect-db.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · DB 层采集(每 30 秒) +# 在 DSH 宿主机上直接 mysql 连 OA DB(DSH 本身就是跳板,不需要再 ssh) +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) + +# ---- MySQL 直连(DSH 上直接连 OA DB 公网映射)---- +DB_MYSQL() { + MYSQL_PWD="${DB_PASS}" timeout 12 mysql -h39.100.72.102 -P3306 -uecology ecology 2>/dev/null +} + +db_exec() { + DB_MYSQL -e "$1" +} + +# ---- 1. SHOW STATUS ---- +log_info "[DB] SHOW STATUS" +STATUS=$(DB_MYSQL -e "SHOW GLOBAL STATUS" | grep -E '^(Threads_running|Threads_connected|Threads_idle|Uptime|Innodb_rows_read|Innodb_rows_inserted|Innodb_rows_updated|Innodb_rows_deleted|Slow_queries)\t') + +THREADS_RUNNING=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_running"{print $2}') +THREADS_CONNECTED=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_connected"{print $2}') +THREADS_IDLE=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_idle"{print $2}') +UPTIME=$(echo "${STATUS}" | awk -F'\t' '$1=="Uptime"{print $2}') +INNODB_READ=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_read"{print $2}') +INNODB_INS=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_inserted"{print $2}') +INNODB_UPD=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_updated"{print $2}') +INNODB_DEL=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_deleted"{print $2}') +SLOW_QUERIES=$(echo "${STATUS}" | awk -F'\t' '$1=="Slow_queries"{print $2}') + +INNODB_READ=${INNODB_READ:-0} +INNODB_INS=${INNODB_INS:-0} +INNODB_UPD=${INNODB_UPD:-0} +INNODB_DEL=${INNODB_DEL:-0} +INNODB_WRITE=$(( INNODB_INS + INNODB_UPD + INNODB_DEL )) +THREADS_RUNNING=${THREADS_RUNNING:-0} +THREADS_CONNECTED=${THREADS_CONNECTED:-0} +THREADS_IDLE=${THREADS_IDLE:-0} +UPTIME=${UPTIME:-0} +SLOW_QUERIES=${SLOW_QUERIES:-0} + +# ---- 2. 上一次快照 → 差值 ---- +PREV=$(sqlite_exec "SELECT * FROM db_metrics ORDER BY ts DESC LIMIT 1;") +QPS_READ=0; QPS_WRITE=0; SLOW_DELTA=0 + +if [[ -n "${PREV}" ]]; then + PREV_UPTIME=$(echo "${PREV}" | cut -d'|' -f5) + PREV_READ=$(echo "${PREV}" | cut -d'|' -f6) + PREV_WRITE=$(echo "${PREV}" | cut -d'|' -f7) + PREV_SLOW=$(echo "${PREV}" | cut -d'|' -f9) + PREV_UPTIME=${PREV_UPTIME:-0}; PREV_READ=${PREV_READ:-0} + PREV_WRITE=${PREV_WRITE:-0}; PREV_SLOW=${PREV_SLOW:-0} + DIFF_SEC=$(( NOW - PREV_UPTIME )) + if [[ ${DIFF_SEC} -gt 0 ]]; then + QPS_READ=$(( (INNODB_READ - PREV_READ) / DIFF_SEC )) + QPS_WRITE=$(( (INNODB_WRITE - PREV_WRITE) / DIFF_SEC )) + fi + SLOW_DELTA=$(( SLOW_QUERIES - PREV_SLOW )) +fi + +# ---- 3. 写 SQLite ---- +sqlite_exec "INSERT INTO db_metrics VALUES ( + ${NOW}, ${THREADS_RUNNING}, ${THREADS_CONNECTED}, ${THREADS_IDLE}, + ${UPTIME}, ${QPS_READ}, ${QPS_WRITE}, ${SLOW_DELTA} +);" + +log_info "[DB] ✅ 写入:Threads_running=${THREADS_RUNNING} QPS_read=${QPS_READ} QPS_write=${QPS_WRITE}" + +# ---- 4. 长查询 processlist ---- +log_info "[DB] processlist" +PLIST=$(DB_MYSQL -e 'SHOW PROCESSLIST;') + +echo "${PLIST}" | awk -F'\t' ' + NR==1 { next } + NF < 8 { next } + { + pid=$1; user=$2; state=$6; time=$7; info=$8 + if (time+0 >= 5 && pid+0 > 0) { + gsub(/\t/, " ", info) + printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300) + } + } +' | while IFS=$'\t' read -r pid time user state info; do + [[ -z "${pid}" ]] && continue + info_safe=$(echo "${info}" | sed "s/'/\"/g") + sqlite_exec "INSERT INTO db_processlist (ts, pid, user, state, time_sec, info) + VALUES (${NOW}, ${pid}, '${user}', '${state}', ${time}, '${info_safe}');" +done + +log_info "[DB] ✅ 完成" diff --git a/collect-http.sh b/collect-http.sh new file mode 100644 index 0000000..793cc16 --- /dev/null +++ b/collect-http.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · HTTP 端点健康检查(每 30 秒) +# 在 DSH 上直接 curl +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) + +for endpoint in "${OA_ENDPOINTS[@]}"; do + result=$(curl -sk -o /dev/null -w '%{http_code}|%{time_total}' --max-time 10 "${endpoint}" 2>/dev/null) + status=$(echo "${result}" | cut -d'|' -f1) + time_ms=$(echo "${result}" | cut -d'|' -f2 | awk '{printf "%d", $1*1000}') + status=${status:-0} + time_ms=${time_ms:-0} + + case "${status}" in + 000) healthy=0 ;; + 2*) healthy=1 ;; + 3*) healthy=1 ;; + *) healthy=0 ;; + esac + + sqlite_exec "INSERT INTO http_metrics (ts, endpoint, status_code, resp_time_ms, healthy) + VALUES (${NOW}, '${endpoint}', ${status}, ${time_ms}, ${healthy});" + + log_info "[HTTP] ${endpoint} status=${status} time=${time_ms}ms healthy=${healthy}" +done + +log_info "[HTTP] 全部完成" diff --git a/collect-infra.sh b/collect-infra.sh new file mode 100644 index 0000000..e6afcd8 --- /dev/null +++ b/collect-infra.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 基础设施层采集(每 30 秒) +# 在 DSH 上直接执行 docker / df / du +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +NOW=$(date +%s) +DOCKER_ROOT="/var/lib/docker" + +# ---- Docker daemon + 容器数 ---- +CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ') +if [[ "${CONTAINER_COUNT}" =~ ^[0-9]+$ ]] && [[ ${CONTAINER_COUNT} -ge 0 ]]; then + DOCKER_OK=1 +else + DOCKER_OK=0 + CONTAINER_COUNT=0 +fi + +# ---- Docker root 写入量(du overlay2 增量)---- +PREV_ROOT_WRITE=$(sqlite_exec "SELECT COALESCE(docker_root_write_mb, 0) FROM infra_metrics ORDER BY ts DESC LIMIT 1;") +CURRENT_ROOT_MB=$(du -sm "${DOCKER_ROOT}/overlay2" 2>/dev/null | cut -f1 | tr -d ' ') +CURRENT_ROOT_MB=${CURRENT_ROOT_MB:-0} +PREV_ROOT_WRITE=${PREV_ROOT_WRITE:-0} + +ROOT_WRITE_DELTA_MB=0 +if [[ -n "${PREV_ROOT_WRITE}" ]] && [[ "${PREV_ROOT_WRITE}" =~ ^[0-9.]+$ ]]; then + DELTA=$(( CURRENT_ROOT_MB - ${PREV_ROOT_WRITE%%.*} )) + if [[ ${DELTA} -gt 0 ]]; then + ROOT_WRITE_DELTA_MB="${DELTA}" + fi +fi + +# ---- 磁盘 / inode / load ---- +DISK_PCT=$(df / | tail -1 | awk '{print $5}' | tr -d '%') +INODE_PCT=$(df -i / | tail -1 | awk '{print $5}' | tr -d '%') +LOADAVG=$(cat /proc/loadavg | awk '{print $1}') + +DISK_PCT=${DISK_PCT:-0} +INODE_PCT=${INODE_PCT:-0} +LOADAVG=${LOADAVG:-0} + +sqlite_exec "INSERT INTO infra_metrics VALUES ( + ${NOW}, ${DOCKER_OK}, ${CONTAINER_COUNT}, ${ROOT_WRITE_DELTA_MB}, + ${DISK_PCT}, ${INODE_PCT}, ${LOADAVG} +);" + +log_info "[INFRA] ✅ docker_ok=${DOCKER_OK} containers=${CONTAINER_COUNT} root_write_mb=${ROOT_WRITE_DELTA_MB} disk=${DISK_PCT}% inode=${INODE_PCT}% loadavg=${LOADAVG}" diff --git a/common.sh b/common.sh new file mode 100644 index 0000000..cf7263e --- /dev/null +++ b/common.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 全局配置与公共函数 +# 所有采集/告警脚本 source 此文件,只修改这里的变量即可调整行为 +# ============================================================================ +set -u # 未定义变量报错 +# 注意:不写 set -e,采集脚本单条命令失败不应整体中断 + +# ---- 路径 ---- +export MONITOR_ROOT="/opt/oa-monitor" +export MONITOR_DATA="${MONITOR_ROOT}/data" +export MONITOR_LOGS="${MONITOR_ROOT}/logs" +export MONITOR_DB="${MONITOR_DATA}/oa-monitor.db" + +mkdir -p "${MONITOR_DATA}" "${MONITOR_LOGS}" + +# ---- 工具 ---- +export MYSQL_CMD="mysql -h39.100.72.102 -P3306 -uecology" +export DB_NAME="ecology" +export DB_PASS='Z!BYT+jSDDyYHMdX+qFSLzJCYXW' + +export SSH_DSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i ~/.ssh/id_ed25519 root@47.108.21.167" + +# ---- OA 端点(DSH 上访问 OA 应用服务器内网 IP,更稳)---- +OA_HTTP="http://39.99.236.138" +OA_ENDPOINTS=( + "${OA_HTTP}:80/wui/main/Login.jsp" + "${OA_HTTP}:8444/api/portal/portalinfo" + "${OA_HTTP}:8444/api/portal/sso/getLoginInfo.jsp" + "${OA_HTTP}/api/workflow/getRequestList" +) + +# ---- 钉钉 Webhook(群 723387268,走 DSH 本地转发)---- +export DINGTALK_WEBHOOK="https://dsh.gs-robot.com/dingtalk/webhook" +export DINGTALK_SECRET="" + +# ---- 告警阈值 ---- +export ALERT_THREADS_RUNNING_HIGH=50 # 当前活跃线程 > 50 告警 +export ALERT_SLOW_QUERY_SEC=30 # 慢查询 > 30s 告警 +export ALERT_SLOW_QUERY_COUNT=5 # 同进程号出现 5 次以上告警 +export ALERT_HTTP_FAIL_COUNT=3 # HTTP 端点连续失败 3 次告警 +export ALERT_MSG_LOG_HOUR_MIN=100 # 1 小时消息增量 < 100 条告警(可能钉钉推送断了) +export ALERT_DOCKER_CONTAINER_DELTA=3 # 容器数突变 > ±3 告警 +export ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN=1024 # Docker data-root 1 分钟写 > 1GB 告警 +export ALERT_DISK_USAGE_PCT=85 # 磁盘使用率 > 85% 告警 +export ALERT_INODE_USAGE_PCT=80 # inode 使用率 > 80% 告警 +export ALERT_DOCKER_DAEMON_FAIL_COUNT=2 # Docker daemon 连续失败 2 次告警 +export ALERT_LONG_QUERY_SEC=300 # 长查询 > 300s 触发 KILL(dry-run 默认) + +# ---- 告警抑制 ---- +export ALERT_COOLDOWN_SEC=600 # 同一告警类型 10 分钟内不重复发 + +# ---- SQLite 工具函数 ---- +sqlite_exec() { + sqlite3 "${MONITOR_DB}" "$@" 2>/dev/null +} + +# ---- 日志函数 ---- +log() { + local level="$1"; shift + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MONITOR_LOGS}/oa-monitor.log" +} + +log_info() { log "INFO" "$@"; } +log_warn() { log "WARN" "$@"; } +log_error() { log "ERROR" "$@"; } + +# ---- 钉钉告警(markdown 格式)---- +# 用法:dingtalk_alert <markdown_body> +# level: P0 / P1 / P2 +dingtalk_alert() { + local level="$1" title="$2" body="$3" + local timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + + # 告警冷却检查 + local key="${level}:${title}" + local last_ts + last_ts=$(sqlite_exec "SELECT COALESCE(last_alert_ts, 0) FROM alert_cooldown WHERE alert_key='${key}';") + local now_epoch + now_epoch=$(date +%s) + if [[ "${last_ts}" =~ ^[0-9]+$ ]] && (( now_epoch - last_ts < ALERT_COOLDOWN_SEC )); then + log_info "告警抑制:${key}(距上次 ${now_epoch}s < ${ALERT_COOLDOWN_SEC}s)" + return 0 + fi + + # 构造 JSON + local json + json=$(cat <<JSON +{"msgtype":"markdown","markdown":{"title":"[OA监控] ${level} ${title}","text":"### [OA监控] **${level}** ${title}\n\n**时间**:${timestamp}\n\n**主机**:DSH (47.108.21.167)\n\n---\n\n${body}\n\n---\n\n> OA 运维监控平台 · $(basename "$0")"}} +JSON +) + + # 发送 + local http_code + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Content-Type: application/json" \ + -d "${json}" "${DINGTALK_WEBHOOK}" 2>/dev/null) + + if [[ "${http_code}" == "200" ]]; then + log_info "钉钉告警发送成功:[${level}] ${title}" + sqlite_exec "INSERT OR REPLACE INTO alert_cooldown VALUES ('${key}', ${now_epoch});" + else + log_error "钉钉告警发送失败(HTTP ${http_code}):[${level}] ${title}" + fi +} diff --git a/dashboard/alerts.html b/dashboard/alerts.html new file mode 100644 index 0000000..682e501 --- /dev/null +++ b/dashboard/alerts.html @@ -0,0 +1,102 @@ +<!DOCTYPE html> +<html lang="zh-CN"> +<head> +<meta charset="UTF-8"> +<title>OA 运维监控 · 告警历史 + + + +

🚨 告警历史

+
+ + + + + +
+ +
+
0
告警总数
+
0
P0 告警
+
0
未确认
+
+ + + + +
级别时间类别标题详情状态
+ + + + diff --git a/dashboard/index.html b/dashboard/index.html new file mode 100644 index 0000000..db4836c --- /dev/null +++ b/dashboard/index.html @@ -0,0 +1,283 @@ + + + + + +OA 运维监控平台 · 总览 + + + + +

🛡️ OA 运维监控平台 · 总览

+ +
+
DB 层
+
HTTP 端点
+
业务/钉钉推送
+
基础设施
+ +
+ +
+
+

DB · Threads Running / 30s

+
+
+
+

DB · QPS Read / Write

+
+
+
+

DB · 当前活跃长查询(>30s)

+
+
+
+

HTTP · 端点状态(最近 2 分钟)

+
+
+
+

业务 · 钉钉推送小时增量

+
+
+
+

基础设施 · 容器数 / 磁盘 / Docker

+
+
+
+

🔴 活跃告警

+
+
+
+ + + + diff --git a/init-db.sh b/init-db.sh new file mode 100644 index 0000000..0ea0c1b --- /dev/null +++ b/init-db.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · SQLite 数据库初始化 +# 只建表,不删数据;幂等可重复执行 +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +log_info "初始化 SQLite 数据库:${MONITOR_DB}" + +sqlite_exec "CREATE TABLE IF NOT EXISTS db_metrics ( + ts INTEGER PRIMARY KEY, + threads_running INTEGER, + threads_connected INTEGER, + threads_idle INTEGER, + uptime_sec INTEGER, + qps_read INTEGER, + qps_write INTEGER, + slow_queries INTEGER -- 近 30s 慢查询数(>30s) +);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS db_processlist ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER, + pid INTEGER, + user TEXT, + state TEXT, + time_sec INTEGER, + info TEXT +);" + +sqlite_exec "CREATE INDEX IF NOT EXISTS idx_db_proc_ts ON db_processlist(ts);" +sqlite_exec "CREATE INDEX IF NOT EXISTS idx_db_proc_time ON db_processlist(time_sec);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS http_metrics ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER, + endpoint TEXT, + status_code INTEGER, + resp_time_ms INTEGER, + healthy INTEGER -- 0/1 +);" + +sqlite_exec "CREATE INDEX IF NOT EXISTS idx_http_ts_ep ON http_metrics(ts, endpoint);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS business_metrics ( + ts INTEGER PRIMARY KEY, + msg_log_hourly INTEGER, -- 本小时 ecology_message_log 增量 + msg_log_fail_rate REAL, -- 失败率 + ongoing_requests INTEGER, -- 进行中流程数 + overdue_tasks INTEGER -- 超时待办数 +);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS infra_metrics ( + ts INTEGER PRIMARY KEY, + docker_daemon_ok INTEGER, -- 0/1 + docker_container_count INTEGER, + docker_root_write_mb REAL, -- 1 分钟 Docker data-root 写入量 MB + disk_usage_pct REAL, + inode_usage_pct REAL, + load_avg REAL +);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS alerts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER, + level TEXT, -- P0/P1/P2 + category TEXT, -- db/http/business/infra + title TEXT, + detail TEXT, + ack INTEGER DEFAULT 0, -- 0=未确认 1=已确认 + resolved INTEGER DEFAULT 0 -- 0=未恢复 1=已恢复 +);" + +sqlite_exec "CREATE INDEX IF NOT EXISTS idx_alerts_ts ON alerts(ts);" +sqlite_exec "CREATE INDEX IF NOT EXISTS idx_alerts_ack ON alerts(ack, resolved);" + +sqlite_exec "CREATE TABLE IF NOT EXISTS alert_cooldown ( + alert_key TEXT PRIMARY KEY, + last_alert_ts INTEGER +);" + +log_info "SQLite schema 初始化完成" diff --git a/run-long-check.sh b/run-long-check.sh new file mode 100644 index 0000000..525daa8 --- /dev/null +++ b/run-long-check.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# ============================================================================ +# OA Monitor · 长查询检测 dry-run KILL(每 10 分钟) +# 默认 DRY_RUN=1(只报告,不真 KILL)。--execute 才真的 KILL +# ============================================================================ +set -u +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/../lib/common.sh" + +DRY_RUN=1 +for arg in "$@"; do + [[ "${arg}" == "--execute" ]] && DRY_RUN=0 + [[ "${arg}" == "--dry-run" ]] && DRY_RUN=1 +done + +NOW=$(date +%s) + +db_exec() { + local sql="$1" + MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -e "${sql}" 2>/dev/null +} + +log_info "[LONG-CHECK] 开始(DRY_RUN=${DRY_RUN},阈值 ${ALERT_LONG_QUERY_SEC}s)" + +RESULT=$(db_exec "SHOW FULL PROCESSLIST;" | awk -F'\t' -v t="${ALERT_LONG_QUERY_SEC}" ' + NR==1 { next } + NF < 10 { next } + { + pid=$1; user=$2; state=$6; time=$7; info=$9 + if (time+0 >= t+0 && time+0 > 0) { + gsub(/\t/, " ", info) + printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300) + } + } +') + +if [[ -z "${RESULT}" ]]; then + log_info "[LONG-CHECK] ✅ 无长查询" + exit 0 +fi + +COUNT=$(echo "${RESULT}" | wc -l | tr -d ' ') +log_warn "[LONG-CHECK] ⚠️ ${COUNT} 条长查询" + +DETAIL="" +while IFS=$'\t' read -r pid time user state info; do + [[ -z "${pid}" ]] && continue + DETAIL="${DETAIL}\n- PID \`${pid}\` **${time}s** \`${user}\` ${state}" + DETAIL="${DETAIL}\n \`\`\`\n ${info}\n \`\`\`" +done <<< "${RESULT}" + +MODE_LABEL=$([[ ${DRY_RUN} -eq 1 ]] && echo "🔒 DRY-RUN" || echo "🚨 执行 KILL") +BODY="**${COUNT} 条长查询**(${ALERT_LONG_QUERY_SEC}s 阈值)· ${MODE_LABEL}\n\n${DETAIL}" +dingtalk_alert "P0" "长查询阻塞需处理" "${BODY}" + +sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询', '${COUNT} 条, dry_run=${DRY_RUN}');" + +if [[ ${DRY_RUN} -eq 0 ]]; then + log_warn "[LONG-CHECK] 执行 KILL!" + while IFS=$'\t' read -r pid time user state info; do + [[ -z "${pid}" ]] && continue + db_exec "KILL ${pid};" + log_warn "[LONG-CHECK] KILL ${pid} (${time}s, ${user})" + done <<< "${RESULT}" +else + log_info "[LONG-CHECK] DRY-RUN,不执行 KILL" +fi diff --git a/server.py b/server.py new file mode 100644 index 0000000..2bade3e --- /dev/null +++ b/server.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# ============================================================================ +# OA Monitor · Python HTTP API(纯标准库,无第三方依赖) +# 唯一职责:读 SQLite → 吐 JSON +# 只读,绝不写 SQLite;所有 POST/PUT/DELETE 返回 405 +# ============================================================================ +import http.server, json, sqlite3, os, sys, urllib.parse +from datetime import datetime, timezone + +DB_PATH = "/opt/oa-monitor/data/oa-monitor.db" +DASHBOARD_DIR = "/opt/oa-monitor/dashboard" + +def get_db(): + return sqlite3.connect(DB_PATH, timeout=5) + +def ts_to_iso(ts): + if not ts: return None + return datetime.fromtimestamp(int(ts)).strftime('%Y-%m-%d %H:%M:%S') + +def fetch_all(sql, params=()): + db = get_db() + db.row_factory = sqlite3.Row + try: + rows = db.execute(sql, params).fetchall() + return [dict(r) for r in rows] + finally: + db.close() + +def fetch_one(sql, params=()): + rows = fetch_all(sql, params) + return rows[0] if rows else None + +class MonitorHandler(http.server.BaseHTTPRequestHandler): + # ---- CORS + JSON 头 ---- + def _send_json(self, data, code=200): + body = json.dumps(data, ensure_ascii=False, default=str).encode() + self.send_response(code) + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.send_header('Content-Length', str(len(body))) + self.send_header('Access-Control-Allow-Origin', '*') + self.send_header('Access-Control-Allow-Methods', 'GET, OPTIONS') + self.send_header('Access-Control-Allow-Headers', 'Content-Type') + self.end_headers() + self.wfile.write(body) + + def do_OPTIONS(self): + self._send_json({}) + + # ---- 路由 ---- + def do_GET(self): + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip('/') + qs = urllib.parse.parse_qs(parsed.query) + + try: + if path.startswith('/api/'): + self._route_api(path, qs) + else: + self._route_static(path) + except Exception as e: + self._send_json({"error": str(e)}, 500) + + def _route_api(self, path, qs): + if path == '/api/health': + self._health() + elif path == '/api/overview': + self._overview() + elif path == '/api/db/recent': + self._db_recent(qs) + elif path == '/api/db/long_queries': + self._db_long_queries(qs) + elif path == '/api/http/recent': + self._http_recent(qs) + elif path == '/api/business/recent': + self._biz_recent(qs) + elif path == '/api/infra/recent': + self._infra_recent(qs) + elif path == '/api/alerts': + self._alerts(qs) + else: + self._send_json({"error": "not found", "path": path}, 404) + + # ---- 静态文件服务 ---- + def _route_static(self, path): + # 安全:禁止 .. 和 /etc 等 + if '..' in path or path.startswith('/etc'): + self._send_json({"error": "forbidden"}, 403); return + + # 根路径 → index.html + if path == '' or path == '/': + path = '/index.html' + + filepath = os.path.join(DASHBOARD_DIR, path.lstrip('/')) + # 双重安全 + if not filepath.startswith(DASHBOARD_DIR): + self._send_json({"error": "forbidden"}, 403); return + + if not os.path.isfile(filepath): + self._send_json({"error": "not found", "path": path}, 404); return + + with open(filepath, 'rb') as f: + content = f.read() + + ext = os.path.splitext(filepath)[1].lower() + mime = { + '.html': 'text/html; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.js': 'application/javascript; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.svg': 'image/svg+xml', + '.png': 'image/png', + '.ico': 'image/x-icon', + }.get(ext, 'application/octet-stream') + + self.send_response(200) + self.send_header('Content-Type', mime) + self.send_header('Content-Length', str(len(content))) + self.send_header('Access-Control-Allow-Origin', '*') + self.send_header('Cache-Control', 'no-cache') + self.end_headers() + self.wfile.write(content) + + def do_POST(self): self._send_json({"error": "read-only"}, 405) + def do_PUT(self): self._send_json({"error": "read-only"}, 405) + def do_DELETE(self): self._send_json({"error": "read-only"}, 405) + + def log_message(self, fmt, *args): + pass # 静默日志 + + # ---- Endpoint 实现 ---- + def _health(self): + db_ok = os.path.exists(DB_PATH) + db_size = os.path.getsize(DB_PATH) if db_ok else 0 + # 最新写入时间 + last = fetch_one("SELECT MAX(ts) AS last_ts FROM db_metrics") + self._send_json({ + "service": "oa-monitor-api", + "db_exists": db_ok, + "db_size_bytes": db_size, + "latest_data_ts": ts_to_iso(last['last_ts'] if last else None), + "now": ts_to_iso(int(datetime.now().timestamp())) + }) + + def _overview(self): + # 各层最新一条 + db = fetch_one("SELECT * FROM db_metrics ORDER BY ts DESC LIMIT 1") + http_all = fetch_all("SELECT endpoint, status_code, resp_time_ms, healthy, ts FROM http_metrics WHERE ts > ? ORDER BY ts DESC", ((datetime.now().timestamp() - 120),)) + biz = fetch_one("SELECT * FROM business_metrics ORDER BY ts DESC LIMIT 1") + infra = fetch_one("SELECT * FROM infra_metrics ORDER BY ts DESC LIMIT 1") + + # 活跃告警 + active_alerts = fetch_all("SELECT * FROM alerts WHERE ack = 0 ORDER BY ts DESC") + + # HTTP 端点最新状态聚合 + http_summary = [] + for row in http_all: + http_summary.append({ + "endpoint": row["endpoint"], + "status_code": row["status_code"], + "resp_time_ms": row["resp_time_ms"], + "healthy": row["healthy"], + "ts": ts_to_iso(row["ts"]) + }) + + # DB 层最近长查询 + long_q = fetch_all("SELECT * FROM db_processlist WHERE time_sec >= 30 ORDER BY time_sec DESC LIMIT 5") + + self._send_json({ + "db_latest": { + **({} if not db else db), + "ts_iso": ts_to_iso(db["ts"]) if db else None + }, + "http_summary": http_summary, + "biz_latest": { + **({} if not biz else biz), + "ts_iso": ts_to_iso(biz["ts"]) if biz else None + }, + "infra_latest": { + **({} if not infra else infra), + "ts_iso": ts_to_iso(infra["ts"]) if infra else None + }, + "active_alerts": [{**a, "ts_iso": ts_to_iso(a["ts"])} for a in active_alerts], + "long_queries": [{**q, "ts_iso": ts_to_iso(q["ts"])} for q in long_q] + }) + + def _db_recent(self, qs): + minutes = int(qs.get('minutes', [30])[0]) + since = int(datetime.now().timestamp()) - minutes * 60 + rows = fetch_all( + "SELECT * FROM db_metrics WHERE ts >= ? ORDER BY ts ASC", + (since,) + ) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + + def _db_long_queries(self, qs): + since_min = int(qs.get('since_minutes', [60])[0]) + since = int(datetime.now().timestamp()) - since_min * 60 + rows = fetch_all( + "SELECT * FROM db_processlist WHERE ts >= ? ORDER BY time_sec DESC LIMIT 100", + (since,) + ) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + + def _http_recent(self, qs): + minutes = int(qs.get('minutes', [30])[0]) + since = int(datetime.now().timestamp()) - minutes * 60 + rows = fetch_all( + "SELECT * FROM http_metrics WHERE ts >= ? ORDER BY ts ASC", + (since,) + ) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + + def _biz_recent(self, qs): + hours = int(qs.get('hours', [24])[0]) + since = int(datetime.now().timestamp()) - hours * 3600 + rows = fetch_all( + "SELECT * FROM business_metrics WHERE ts >= ? ORDER BY ts ASC", + (since,) + ) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + + def _infra_recent(self, qs): + minutes = int(qs.get('minutes', [30])[0]) + since = int(datetime.now().timestamp()) - minutes * 60 + rows = fetch_all( + "SELECT * FROM infra_metrics WHERE ts >= ? ORDER BY ts ASC", + (since,) + ) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + + def _alerts(self, qs): + limit = int(qs.get('limit', [100])[0]) + only_unack = qs.get('unack', ['0'])[0] + sql = "SELECT * FROM alerts" + params = [] + if only_unack == '1': + sql += " WHERE ack = 0" + sql += " ORDER BY ts DESC LIMIT ?" + params.append(limit) + rows = fetch_all(sql, tuple(params)) + self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows]) + +# ---- 启动 ---- +if __name__ == '__main__': + port = int(sys.argv[1]) if len(sys.argv) > 1 else 8455 + server = http.server.HTTPServer(('0.0.0.0', port), MonitorHandler) + print(f"OA Monitor API listening on :{port} (DB={DB_PATH})", flush=True) + server.serve_forever() diff --git a/state/fix-info-read.pid b/state/fix-info-read.pid new file mode 100644 index 0000000..53d108b --- /dev/null +++ b/state/fix-info-read.pid @@ -0,0 +1 @@ +1231 diff --git a/state/resin_pid.last b/state/resin_pid.last index 757643f..343a78a 100644 --- a/state/resin_pid.last +++ b/state/resin_pid.last @@ -1 +1 @@ -13286 9890 1793773202 +30952 16490 1793792402