fix: check-alert v3 修复钉钉推送断档 P0 假阳性
根因:sqlite_exec 吞掉 stderr(2>/dev/null),查询失败返回空串, 0 默认为 0 导致误报 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验, 查询失败跳过检测并 warn 日志
This commit is contained in:
1 parent
7ab75364b1
commit
20fa48fb34
17 files changed
+1747
-26
No files matched your search
Executable
+281
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env python3
|
||||
# ============================================================================
|
||||
# OA Monitor · Python HTTP API(纯标准库,无第三方依赖)
|
||||
# 唯一职责:读 SQLite → 吐 JSON + 静态 Dashboard
|
||||
# 只读,绝不写 SQLite;所有 POST/PUT/DELETE 返回 405
|
||||
# ============================================================================
|
||||
import http.server, json, sqlite3, os, sys, urllib.parse, base64
|
||||
from datetime import datetime, timezone
|
||||
|
||||
DB_PATH = "/opt/oa-monitor/data/oa-monitor.db"
|
||||
DASHBOARD_DIR = "/opt/oa-monitor/dashboard"
|
||||
|
||||
# ---- HTTP Basic Auth ----
|
||||
# 只允许 oamonitor 用户
|
||||
AUTH_USER = "oamonitor"
|
||||
AUTH_PASS = "1mcAi/rCtrZZkz3To2BD"
|
||||
AUTH_REALM = "OA Monitor"
|
||||
|
||||
def check_auth(headers):
|
||||
"""检查 Authorization 头,返回 bool"""
|
||||
auth = headers.get('Authorization', '')
|
||||
if not auth.startswith('Basic '):
|
||||
return False
|
||||
try:
|
||||
decoded = base64.b64decode(auth[6:]).decode()
|
||||
user, _, pwd = decoded.partition(':')
|
||||
return user == AUTH_USER and pwd == AUTH_PASS
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def send_401(handler):
|
||||
body = b'Unauthorized: OA Monitor - HTTP Basic Auth required (user: oamonitor)'
|
||||
handler.send_response(401)
|
||||
handler.send_header('WWW-Authenticate', f'Basic realm="{AUTH_REALM}"')
|
||||
handler.send_header('Content-Type', 'text/plain; charset=utf-8')
|
||||
handler.send_header('Content-Length', str(len(body)))
|
||||
handler.send_header('Access-Control-Allow-Origin', '*')
|
||||
handler.end_headers()
|
||||
handler.wfile.write(body)
|
||||
|
||||
def get_db():
|
||||
return sqlite3.connect(DB_PATH, timeout=5)
|
||||
|
||||
def ts_to_iso(ts):
|
||||
if not ts: return None
|
||||
return datetime.fromtimestamp(int(ts)).strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
def fetch_all(sql, params=()):
|
||||
db = get_db()
|
||||
db.row_factory = sqlite3.Row
|
||||
try:
|
||||
rows = db.execute(sql, params).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def fetch_one(sql, params=()):
|
||||
rows = fetch_all(sql, params)
|
||||
return rows[0] if rows else None
|
||||
|
||||
class MonitorHandler(http.server.BaseHTTPRequestHandler):
|
||||
# ---- CORS + JSON 头 ----
|
||||
def _send_json(self, data, code=200):
|
||||
body = json.dumps(data, ensure_ascii=False, default=str).encode()
|
||||
self.send_response(code)
|
||||
self.send_header('Content-Type', 'application/json; charset=utf-8')
|
||||
self.send_header('Content-Length', str(len(body)))
|
||||
self.send_header('Access-Control-Allow-Origin', '*')
|
||||
self.send_header('Access-Control-Allow-Methods', 'GET, OPTIONS')
|
||||
self.send_header('Access-Control-Allow-Headers', 'Content-Type')
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_OPTIONS(self):
|
||||
self._send_json({})
|
||||
|
||||
# ---- 路由 ----
|
||||
def do_GET(self):
|
||||
# HTTP Basic Auth(全路径保护,包括 Dashboard 静态文件 + API)
|
||||
if not check_auth(self.headers):
|
||||
send_401(self)
|
||||
return
|
||||
|
||||
parsed = urllib.parse.urlparse(self.path)
|
||||
path = parsed.path.rstrip('/')
|
||||
qs = urllib.parse.parse_qs(parsed.query)
|
||||
|
||||
try:
|
||||
if path.startswith('/api/'):
|
||||
self._route_api(path, qs)
|
||||
else:
|
||||
self._route_static(path)
|
||||
except Exception as e:
|
||||
self._send_json({"error": str(e)}, 500)
|
||||
|
||||
def _route_api(self, path, qs):
|
||||
if path == '/api/health':
|
||||
self._health()
|
||||
elif path == '/api/overview':
|
||||
self._overview()
|
||||
elif path == '/api/db/recent':
|
||||
self._db_recent(qs)
|
||||
elif path == '/api/db/long_queries':
|
||||
self._db_long_queries(qs)
|
||||
elif path == '/api/http/recent':
|
||||
self._http_recent(qs)
|
||||
elif path == '/api/business/recent':
|
||||
self._biz_recent(qs)
|
||||
elif path == '/api/infra/recent':
|
||||
self._infra_recent(qs)
|
||||
elif path == '/api/alerts':
|
||||
self._alerts(qs)
|
||||
else:
|
||||
self._send_json({"error": "not found", "path": path}, 404)
|
||||
|
||||
# ---- 静态文件服务 ----
|
||||
def _route_static(self, path):
|
||||
# 安全:禁止 .. 和 /etc 等
|
||||
if '..' in path or path.startswith('/etc'):
|
||||
self._send_json({"error": "forbidden"}, 403); return
|
||||
|
||||
# 根路径 → index.html
|
||||
if path == '' or path == '/':
|
||||
path = '/index.html'
|
||||
|
||||
filepath = os.path.join(DASHBOARD_DIR, path.lstrip('/'))
|
||||
# 双重安全
|
||||
if not filepath.startswith(DASHBOARD_DIR):
|
||||
self._send_json({"error": "forbidden"}, 403); return
|
||||
|
||||
if not os.path.isfile(filepath):
|
||||
self._send_json({"error": "not found", "path": path}, 404); return
|
||||
|
||||
with open(filepath, 'rb') as f:
|
||||
content = f.read()
|
||||
|
||||
ext = os.path.splitext(filepath)[1].lower()
|
||||
mime = {
|
||||
'.html': 'text/html; charset=utf-8',
|
||||
'.css': 'text/css; charset=utf-8',
|
||||
'.js': 'application/javascript; charset=utf-8',
|
||||
'.json': 'application/json; charset=utf-8',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.png': 'image/png',
|
||||
'.ico': 'image/x-icon',
|
||||
}.get(ext, 'application/octet-stream')
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', mime)
|
||||
self.send_header('Content-Length', str(len(content)))
|
||||
self.send_header('Access-Control-Allow-Origin', '*')
|
||||
self.send_header('Cache-Control', 'no-cache')
|
||||
self.end_headers()
|
||||
self.wfile.write(content)
|
||||
|
||||
def do_POST(self): self._send_json({"error": "read-only"}, 405)
|
||||
def do_PUT(self): self._send_json({"error": "read-only"}, 405)
|
||||
def do_DELETE(self): self._send_json({"error": "read-only"}, 405)
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
pass # 静默日志
|
||||
|
||||
# ---- Endpoint 实现 ----
|
||||
def _health(self):
|
||||
db_ok = os.path.exists(DB_PATH)
|
||||
db_size = os.path.getsize(DB_PATH) if db_ok else 0
|
||||
# 最新写入时间
|
||||
last = fetch_one("SELECT MAX(ts) AS last_ts FROM db_metrics")
|
||||
self._send_json({
|
||||
"service": "oa-monitor-api",
|
||||
"db_exists": db_ok,
|
||||
"db_size_bytes": db_size,
|
||||
"latest_data_ts": ts_to_iso(last['last_ts'] if last else None),
|
||||
"now": ts_to_iso(int(datetime.now().timestamp()))
|
||||
})
|
||||
|
||||
def _overview(self):
|
||||
# 各层最新一条
|
||||
db = fetch_one("SELECT * FROM db_metrics ORDER BY ts DESC LIMIT 1")
|
||||
http_all = fetch_all("SELECT endpoint, status_code, resp_time_ms, healthy, ts FROM http_metrics WHERE ts > ? ORDER BY ts DESC", ((datetime.now().timestamp() - 120),))
|
||||
biz = fetch_one("SELECT * FROM business_metrics ORDER BY ts DESC LIMIT 1")
|
||||
infra = fetch_one("SELECT * FROM infra_metrics ORDER BY ts DESC LIMIT 1")
|
||||
|
||||
# 活跃告警
|
||||
active_alerts = fetch_all("SELECT * FROM alerts WHERE ack = 0 ORDER BY ts DESC")
|
||||
|
||||
# HTTP 端点最新状态聚合
|
||||
http_summary = []
|
||||
for row in http_all:
|
||||
http_summary.append({
|
||||
"endpoint": row["endpoint"],
|
||||
"status_code": row["status_code"],
|
||||
"resp_time_ms": row["resp_time_ms"],
|
||||
"healthy": row["healthy"],
|
||||
"ts": ts_to_iso(row["ts"])
|
||||
})
|
||||
|
||||
# DB 层最近长查询
|
||||
long_q = fetch_all("SELECT * FROM db_processlist WHERE time_sec >= 30 ORDER BY time_sec DESC LIMIT 5")
|
||||
|
||||
self._send_json({
|
||||
"db_latest": {
|
||||
**({} if not db else db),
|
||||
"ts_iso": ts_to_iso(db["ts"]) if db else None
|
||||
},
|
||||
"http_summary": http_summary,
|
||||
"biz_latest": {
|
||||
**({} if not biz else biz),
|
||||
"ts_iso": ts_to_iso(biz["ts"]) if biz else None
|
||||
},
|
||||
"infra_latest": {
|
||||
**({} if not infra else infra),
|
||||
"ts_iso": ts_to_iso(infra["ts"]) if infra else None
|
||||
},
|
||||
"active_alerts": [{**a, "ts_iso": ts_to_iso(a["ts"])} for a in active_alerts],
|
||||
"long_queries": [{**q, "ts_iso": ts_to_iso(q["ts"])} for q in long_q]
|
||||
})
|
||||
|
||||
def _db_recent(self, qs):
|
||||
minutes = int(qs.get('minutes', [30])[0])
|
||||
since = int(datetime.now().timestamp()) - minutes * 60
|
||||
rows = fetch_all(
|
||||
"SELECT * FROM db_metrics WHERE ts >= ? ORDER BY ts ASC",
|
||||
(since,)
|
||||
)
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
def _db_long_queries(self, qs):
|
||||
since_min = int(qs.get('since_minutes', [60])[0])
|
||||
since = int(datetime.now().timestamp()) - since_min * 60
|
||||
rows = fetch_all(
|
||||
"SELECT * FROM db_processlist WHERE ts >= ? ORDER BY time_sec DESC LIMIT 100",
|
||||
(since,)
|
||||
)
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
def _http_recent(self, qs):
|
||||
minutes = int(qs.get('minutes', [30])[0])
|
||||
since = int(datetime.now().timestamp()) - minutes * 60
|
||||
rows = fetch_all(
|
||||
"SELECT * FROM http_metrics WHERE ts >= ? ORDER BY ts ASC",
|
||||
(since,)
|
||||
)
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
def _biz_recent(self, qs):
|
||||
hours = int(qs.get('hours', [24])[0])
|
||||
since = int(datetime.now().timestamp()) - hours * 3600
|
||||
rows = fetch_all(
|
||||
"SELECT * FROM business_metrics WHERE ts >= ? ORDER BY ts ASC",
|
||||
(since,)
|
||||
)
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
def _infra_recent(self, qs):
|
||||
minutes = int(qs.get('minutes', [30])[0])
|
||||
since = int(datetime.now().timestamp()) - minutes * 60
|
||||
rows = fetch_all(
|
||||
"SELECT * FROM infra_metrics WHERE ts >= ? ORDER BY ts ASC",
|
||||
(since,)
|
||||
)
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
def _alerts(self, qs):
|
||||
limit = int(qs.get('limit', [100])[0])
|
||||
only_unack = qs.get('unack', ['0'])[0]
|
||||
sql = "SELECT * FROM alerts"
|
||||
params = []
|
||||
if only_unack == '1':
|
||||
sql += " WHERE ack = 0"
|
||||
sql += " ORDER BY ts DESC LIMIT ?"
|
||||
params.append(limit)
|
||||
rows = fetch_all(sql, tuple(params))
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
# ---- 启动 ----
|
||||
if __name__ == '__main__':
|
||||
port = int(sys.argv[1]) if len(sys.argv) > 1 else 8455
|
||||
server = http.server.HTTPServer(('0.0.0.0', port), MonitorHandler)
|
||||
print(f"OA Monitor API listening on :{port} (DB={DB_PATH})", flush=True)
|
||||
server.serve_forever()
|
||||
+46
-25
@@ -4,6 +4,9 @@
|
||||
# v2 修复(2026-10-28):
|
||||
# - 新增 Step 0:数据新鲜度前置检查,防止采集脚本挂了后 check-alert 读旧数据乱告警
|
||||
# - Step 3 业务层检测被 Step 0 结果门禁
|
||||
# v3 修复(2026-11-04):
|
||||
# - 业务层 P0 推送断档假阳性:sqlite_exec 吞错误(2>/dev/null)→ 查询失败返回空串 → :-0 默认为 0 → 误报
|
||||
# - 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验,查询失败跳过检测并 warn
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
@@ -36,7 +39,7 @@ else
|
||||
if [[ ${BIZ_AGE} -gt 1800 ]]; then # > 30 分钟
|
||||
BIZ_FRESH=false
|
||||
HUMAN_AGE=$((BIZ_AGE / 60))
|
||||
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec \"SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')\") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec \"SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')\") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
|
||||
BODY="**🔴 采集脚本挂了!**\n\n| 表 | 最新 ts | 年龄 |\n|---|---|---|\n| business_metrics | $(sqlite_exec "SELECT datetime(${BIZ_TS}, 'unixepoch', 'localtime')") | **${HUMAN_AGE}min** |\n| db_metrics | $(sqlite_exec "SELECT datetime(${DB_TS}, 'unixepoch', 'localtime')") | ${DB_AGE}s |\n\n> ⚠️ collect-business.sh 超过 30min 没写入,先告警采集挂了,防止后续业务层读旧数据乱告警"
|
||||
dingtalk_alert "P1" "采集脚本挂了!business_metrics 过期 ${HUMAN_AGE}min" "${BODY}"
|
||||
log_error "[ALERT] 数据过期 ${HUMAN_AGE}min,禁止跑业务层检测(防假阳性)"
|
||||
else
|
||||
@@ -94,34 +97,52 @@ fi
|
||||
if [[ "${BIZ_FRESH}" != "true" ]]; then
|
||||
log_warn "[ALERT] ⏸️ business_metrics 数据过期,跳过业务层检测(防假阳性)"
|
||||
else
|
||||
# 数据新鲜 → 正常业务层判断
|
||||
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
ONGOING=$(sqlite_exec "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" | head -1)
|
||||
HOUR_MSG=${HOUR_MSG:-0}
|
||||
ONGOING=${ONGOING:-0}
|
||||
# v3 修复:显式调用 sqlite3 + 检查退出码 + 正则校验,区分"查询失败"和"结果为 0"
|
||||
MSG_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null)
|
||||
MSG_RC=$?
|
||||
ONGOING_RAW=$(sqlite3 "${MONITOR_DB}" "SELECT ongoing_requests FROM business_metrics ORDER BY ts DESC LIMIT 1;" 2>/dev/null)
|
||||
ONGOING_RC=$?
|
||||
|
||||
CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal
|
||||
WORK_HOURS_ENABLED=false
|
||||
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
|
||||
WORK_HOURS_ENABLED=true
|
||||
# 双重校验:退出码必须为 0,且返回值非空且为纯数字
|
||||
BIZ_QUERY_OK=true
|
||||
if [[ ${MSG_RC} -ne 0 ]] || [[ -z "${MSG_RAW}" ]] || [[ ! "${MSG_RAW}" =~ ^[0-9]+$ ]]; then
|
||||
log_warn "[ALERT] SQLite msg_log_hourly 查询异常 rc=${MSG_RC} raw='${MSG_RAW}',跳过业务层(防假阳性)"
|
||||
BIZ_QUERY_OK=false
|
||||
fi
|
||||
if [[ ${ONGOING_RC} -ne 0 ]] || [[ -z "${ONGOING_RAW}" ]] || [[ ! "${ONGOING_RAW}" =~ ^[0-9]+$ ]]; then
|
||||
log_warn "[ALERT] SQLite ongoing_requests 查询异常 rc=${ONGOING_RC} raw='${ONGOING_RAW}',跳过业务层(防假阳性)"
|
||||
BIZ_QUERY_OK=false
|
||||
fi
|
||||
|
||||
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
|
||||
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then
|
||||
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
|
||||
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
||||
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n"
|
||||
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n"
|
||||
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n"
|
||||
BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n"
|
||||
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n"
|
||||
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n"
|
||||
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**"
|
||||
dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
|
||||
fi
|
||||
if [[ "${BIZ_QUERY_OK}" != "true" ]]; then
|
||||
log_warn "[ALERT] ⏸️ 业务层数据查询异常,跳过钉钉推送断档 P0 检测"
|
||||
else
|
||||
log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警"
|
||||
HOUR_MSG=${MSG_RAW}
|
||||
ONGOING=${ONGOING_RAW}
|
||||
|
||||
CURR_HOUR=$((10#$(date +%H))) # 算术强制 10 进制,防 "08"/"09" 被当 8/9 或 octal
|
||||
WORK_HOURS_ENABLED=false
|
||||
if [[ ${CURR_HOUR} -ge 8 ]] && [[ ${CURR_HOUR} -le 21 ]]; then
|
||||
WORK_HOURS_ENABLED=true
|
||||
fi
|
||||
|
||||
if [[ "${WORK_HOURS_ENABLED}" == "true" ]]; then
|
||||
if [[ ${HOUR_MSG} -eq 0 ]] && [[ ${ONGOING} -gt 0 ]]; then
|
||||
BODY="**🔴 P0 钉钉推送完全断档!**\n\n"
|
||||
BODY="${BODY}| 指标 | 值 |\n|---|---|\n"
|
||||
BODY="${BODY}| message_log 小时增量 | **${HOUR_MSG} 条** |\n"
|
||||
BODY="${BODY}| 正在流转审批 | **${ONGOING} 条** |\n"
|
||||
BODY="${BODY}| 当前时段 | **工作时间**(${CURR_HOUR}:00)|\n"
|
||||
BODY="${BODY}| 数据新鲜度 | **${BIZ_AGE}s** ✅ |\n\n"
|
||||
BODY="${BODY}> ⚠️ 审批在流转,但钉钉消息完全没发!\n"
|
||||
BODY="${BODY}> 常见原因:Util_Message 死锁、Resin 消息推送线程挂了、Redis 报错\n"
|
||||
BODY="${BODY}> **应急:SSH 到 OA 服务器执行 \`ps aux | grep Resin\` → \`stopresin.sh && startresin.sh\`**"
|
||||
dingtalk_alert "P0" "钉钉推送完全断档(消息=0 但业务在跑)" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'business', '钉钉推送完全断档', 'msg=${HOUR_MSG} ongoing=${ONGOING}');"
|
||||
fi
|
||||
else
|
||||
log_info "[ALERT] ⏸️ 非工作时间(${CURR_HOUR}:00),跳过业务层告警"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
Executable
+161
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""fix-info-read-dummy.py — 补 DUMMY 根治 updateBizState 死循环 (Python 3.6 兼容)"""
|
||||
import sys, os, time, subprocess, json, urllib.request
|
||||
|
||||
DB_HOST, DB_PORT, DB_USER = "39.100.72.102", 3306, "ecology"
|
||||
DB_PASS, DB_NAME = "Z!BYT+jSDDyYHMdX+qFSLzJCYXW", "ecology"
|
||||
OA_HOST, OA_USER, OA_PASS = "39.99.236.138", "root", "KTOFKjVkVOGlNgvs8yUP1+PM1VM"
|
||||
DINGTALK = "https://oapi.dingtalk.com/robot/send?access_token=cbc113a59a6a67cdc1b23bc052bb51bc84582504867284172584f0642455e249"
|
||||
BATCH = 5000
|
||||
LOG = "/opt/oa-monitor/logs/fix-info-read.log"
|
||||
|
||||
# === 正确值(从之前 DUMMY_FIX_ 样例提取)===
|
||||
MESSAGETYPE = 25 # int!不是 'archive'
|
||||
BIZTYPE = 86 # int!上次成功用的
|
||||
STATE = 2 # 已读
|
||||
BIZSTATE = 1 # 已更新(关键!让 updateBizState 不再重试)
|
||||
GROUPTYPE = "archive-fix"
|
||||
BIZTITLE = "dummy fix"
|
||||
|
||||
def log(msg, level="INFO"):
|
||||
line = "[%s] [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S"), level, msg)
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def run_mysql(sql, fetch=True):
|
||||
env = os.environ.copy(); env["MYSQL_PWD"] = DB_PASS
|
||||
r = subprocess.run(["mysql","-h"+DB_HOST,"-P"+str(DB_PORT),"-u"+DB_USER,DB_NAME,"-N","-e",sql],
|
||||
env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=600)
|
||||
if r.returncode != 0:
|
||||
log("MySQL ERROR: " + r.stderr.decode().strip(), "ERROR")
|
||||
return None
|
||||
if not fetch: return True
|
||||
return r.stdout.decode().strip()
|
||||
|
||||
def dingtalk(level, title, body=""):
|
||||
text = "**[%s] %s**\n\n%s\n\n> OA DUMMY @ %s" % (level, title, body, time.strftime("%Y-%m-%d %H:%M:%S"))
|
||||
try:
|
||||
urllib.request.urlopen(urllib.request.Request(DINGTALK, json.dumps({"msgtype":"markdown","markdown":{"title":title,"text":text}}).encode(), {"Content-Type":"application/json"}), timeout=5)
|
||||
except Exception as e: log("钉钉: "+str(e), "WARN")
|
||||
|
||||
def check_http():
|
||||
env = os.environ.copy()
|
||||
cmd = ("python3 -c 'import paramiko; ssh=paramiko.SSHClient(); ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()); "
|
||||
"ssh.connect(\"%s\",username=\"%s\",password=\"%s\",timeout=15); "
|
||||
"_,o,_=ssh.exec_command(\"curl -s -o /dev/null -w %%{http_code} http://127.0.0.1:80/\"); "
|
||||
"print(o.read().decode(\"utf-8\",\"replace\")); ssh.close()'" % (OA_HOST, OA_USER, OA_PASS))
|
||||
r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=25, env=env)
|
||||
return "200" in r.stdout.decode().strip()
|
||||
|
||||
def check_tr():
|
||||
r = run_mysql("SHOW GLOBAL STATUS WHERE Variable_name='Threads_running';")
|
||||
if not r: return 999
|
||||
try: return int(r.split("\t")[-1].strip())
|
||||
except: return 999
|
||||
|
||||
def wait_safe(max_wait=120):
|
||||
for i in range(max_wait // 5):
|
||||
if check_http() and check_tr() < 30: return True
|
||||
log("等待恢复 (%ds) HTTP=%s TR=%d" % (i*5, check_http(), check_tr()), "WARN")
|
||||
time.sleep(5)
|
||||
return False
|
||||
|
||||
# === Phase 0 ===
|
||||
log("===== Phase 0: 前置检查 =====")
|
||||
DRY = "--dry-run" in sys.argv
|
||||
if DRY: log("*** DRY RUN ***")
|
||||
|
||||
ok = check_http(); log("Resin HTTP 80 = %s" % ok)
|
||||
if not ok: log("Resin 不通!中止!", "CRITICAL"); sys.exit(1)
|
||||
|
||||
tr = check_tr(); log("Threads_running = %d" % tr)
|
||||
if tr >= 15:
|
||||
wait_safe(180); tr = check_tr()
|
||||
|
||||
existing = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_FIX_%';")
|
||||
log("现有 DUMMY_FIX = %s" % existing)
|
||||
|
||||
log("Phase 1: 备份表...")
|
||||
run_mysql("CREATE TABLE IF NOT EXISTS ecology_message_info_read_bak_20261104 AS SELECT * FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';", fetch=False)
|
||||
|
||||
dingtalk("INFO", "DUMMY 补数据启动", "HTTP=200 ✅\nTR=%d\nDUMMY_FIX=%s" % (tr, existing))
|
||||
|
||||
# === Phase 2 ===
|
||||
total = [0]
|
||||
|
||||
def insert_user(uid, prefix, col):
|
||||
last_id = 0; batch = 0; user_total = 0
|
||||
while True:
|
||||
sql = ("INSERT INTO ecology_message_info_read "
|
||||
"(ID, MESSAGEID, MESSAGETYPE, USERID, TARGETID, STATE, BIZSTATE, "
|
||||
"CREATEDATE, CREATETIME, MESSAGEGROUPTYPE, BIZTITLE, BIZTYPE) "
|
||||
"SELECT CONCAT('%s%s_', REQUESTID), 0, %d, %d, REQUESTID, %d, %d, "
|
||||
"CURDATE(), CURTIME(), '%s', '%s', %d "
|
||||
"FROM workflow_requestbase "
|
||||
"WHERE %s = %d AND STATUS LIKE '%%归档%%' AND DELETED = 0 AND REQUESTID > %d "
|
||||
"ORDER BY REQUESTID LIMIT %d;" %
|
||||
(prefix, "%d", MESSAGETYPE, uid, STATE, BIZSTATE, GROUPTYPE, BIZTITLE, BIZTYPE,
|
||||
col, uid, last_id, BATCH))
|
||||
if DRY:
|
||||
batch += 1; user_total += BATCH; last_id += BATCH
|
||||
if batch >= 5: break
|
||||
time.sleep(0.2); continue
|
||||
|
||||
run_mysql(sql, fetch=False)
|
||||
rc = run_mysql("SELECT ROW_COUNT();")
|
||||
inserted = int(rc) if rc and rc.isdigit() else 0
|
||||
if inserted == 0: break
|
||||
|
||||
last_sql = "SELECT MAX(REQUESTID) FROM workflow_requestbase WHERE %s = %d AND REQUESTID > %d LIMIT %d;" % (col, uid, last_id, BATCH)
|
||||
lv = run_mysql(last_sql)
|
||||
if lv and lv.isdigit(): last_id = int(lv)
|
||||
|
||||
batch += 1; user_total += inserted; total[0] += inserted
|
||||
log(" User=%d batch=%d +%d rows (total=%d)" % (uid, batch, inserted, total[0]))
|
||||
|
||||
if not check_http() or check_tr() >= 30:
|
||||
log("异常!暂停...", "WARN"); wait_safe(10)
|
||||
time.sleep(1)
|
||||
|
||||
if user_total > 0:
|
||||
log("User=%d 完成: %d 条 %d 批" % (uid, user_total, batch))
|
||||
|
||||
# Phase 2a — 全量业务用户(不限 TOP 100)
|
||||
log("===== Phase 2a: CREATER (全量) =====")
|
||||
creaters_raw = run_mysql("SELECT CREATER, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND CREATER!=1 GROUP BY CREATER ORDER BY COUNT(*) DESC;")
|
||||
creaters = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in creaters_raw.strip().split("\n") if l.strip()]
|
||||
log("CREATER 用户=%d 归档总量=%d" % (len(creaters), sum(c for _,c in creaters)))
|
||||
|
||||
for i, (uid, tot) in enumerate(creaters, 1):
|
||||
log("--- CREATER #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
|
||||
insert_user(uid, "DUMMY_B_CREATER_", "CREATER")
|
||||
if i % 10 == 0:
|
||||
dingtalk("INFO", "CREATER %d/100" % i, "已插入=%d" % total[0])
|
||||
|
||||
# Phase 2b — 全量业务用户(不限 TOP 100)
|
||||
log("===== Phase 2b: LASTOPERATOR (全量) =====")
|
||||
ops_raw = run_mysql("SELECT LASTOPERATOR, COUNT(*) FROM workflow_requestbase WHERE STATUS LIKE '%%归档%%' AND DELETED=0 AND LASTOPERATOR!=1 GROUP BY LASTOPERATOR ORDER BY COUNT(*) DESC;")
|
||||
ops = [(int(l.split("\t")[0]), int(l.split("\t")[1])) for l in ops_raw.strip().split("\n") if l.strip()]
|
||||
log("OPERATOR 用户=%d 归档总量=%d" % (len(ops), sum(c for _,c in ops)))
|
||||
|
||||
for i, (uid, tot) in enumerate(ops, 1):
|
||||
log("--- OPERATOR #%d/100 uid=%d 归档=%d ---" % (i, uid, tot))
|
||||
insert_user(uid, "DUMMY_B_OPERATOR_", "LASTOPERATOR")
|
||||
if i % 10 == 0:
|
||||
dingtalk("INFO", "OPERATOR %d/100" % i, "已插入=%d" % total[0])
|
||||
|
||||
# Phase 3
|
||||
log("===== Phase 3: 验证 =====")
|
||||
fc = run_mysql("SELECT COUNT(*) FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%';")
|
||||
log("DUMMY_B 总数 = %s" % fc)
|
||||
http_ok = check_http(); log("Resin HTTP = %s" % http_ok)
|
||||
tr2 = check_tr(); log("Threads_running = %d" % tr2)
|
||||
lm = run_mysql("SELECT MAX(CREATETIME) FROM ecology_message_log WHERE CREATEDATE=CURDATE();")
|
||||
log("message_log 最新 = %s" % lm)
|
||||
|
||||
dingtalk("P0", "DUMMY 补数据完成 ✅",
|
||||
"**DUMMY_B**: %s 条\n**HTTP**: %s\n**TR**: %d\n**最新消息**: %s\n\n回滚: DELETE FROM ecology_message_info_read WHERE ID LIKE 'DUMMY_B_%%';" %
|
||||
(fc, http_ok, tr2, lm))
|
||||
log("===== 全部完成 =====")
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 告警检测(每 1 分钟)
|
||||
# 遍历所有指标,超过阈值触发钉钉告警
|
||||
# 告警冷却由 common.sh 的 alert_cooldown 表处理(10 分钟同类型不重复)
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
log_info "[ALERT] ========== 告警检测开始 =========="
|
||||
|
||||
# ============================================================================
|
||||
# 1. DB 层检测
|
||||
# ============================================================================
|
||||
# Threads_running 过高
|
||||
TR=$(sqlite_exec "SELECT threads_running FROM db_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
if [[ -n "${TR}" ]] && [[ ${TR} -gt ${ALERT_THREADS_RUNNING_HIGH} ]]; then
|
||||
BODY="**Threads_running: ${TR}**(阈值 ${ALERT_THREADS_RUNNING_HIGH})\n\n活跃连接:$(sqlite_exec 'SELECT threads_connected FROM db_metrics ORDER BY ts DESC LIMIT 1;')\n活跃空闲:$(sqlite_exec 'SELECT threads_idle FROM db_metrics ORDER BY ts DESC LIMIT 1;')"
|
||||
dingtalk_alert "P1" "DB 层 Threads_running 过高" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'db', 'Threads_running 过高', 'threads_running=${TR}');"
|
||||
fi
|
||||
|
||||
# 长查询(processlist 中 Time >= ALERT_SLOW_QUERY_SEC 的)
|
||||
LONG_QUERIES=$(sqlite_exec "SELECT pid, time_sec, user, state, substr(info,1,100) FROM db_processlist WHERE ts > ${NOW}-120 AND time_sec >= ${ALERT_SLOW_QUERY_SEC} ORDER BY time_sec DESC LIMIT 5;")
|
||||
if [[ -n "${LONG_QUERIES}" ]]; then
|
||||
COUNT=$(echo "${LONG_QUERIES}" | wc -l | tr -d ' ')
|
||||
DETAIL=""
|
||||
while IFS='|' read -r pid time user state info; do
|
||||
DETAIL="${DETAIL}\n- PID \`${pid}\` Time **${time}s** User \`${user}\` State \`${state}\`\n \`\`\`sql\n ${info}\n \`\`\`"
|
||||
done <<< "${LONG_QUERIES}"
|
||||
BODY="**检测到 ${COUNT} 条长查询**(阈值 ${ALERT_SLOW_QUERY_SEC}s)\n\n${DETAIL}\n\n> ⚠️ 长查询会阻塞消息推送线程,建议优先处理"
|
||||
dingtalk_alert "P0" "DB 层长查询阻塞(${COUNT} 条)" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询阻塞', '${COUNT} 条 > ${ALERT_SLOW_QUERY_SEC}s');"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 2. HTTP 端点检测
|
||||
# ============================================================================
|
||||
# 最近 3 次的失败次数
|
||||
FAIL_COUNT=0
|
||||
FAIL_DETAIL=""
|
||||
for endpoint in "${OA_ENDPOINTS[@]}"; do
|
||||
FAIL_HERE=$(sqlite_exec "SELECT COUNT(*) FROM http_metrics WHERE endpoint='${endpoint}' AND ts > ${NOW}-120 AND healthy=0;")
|
||||
FAIL_HERE=${FAIL_HERE:-0}
|
||||
if [[ ${FAIL_HERE} -ge ${ALERT_HTTP_FAIL_COUNT} ]]; then
|
||||
FAIL_COUNT=$((FAIL_COUNT + 1))
|
||||
LAST_STATUS=$(sqlite_exec "SELECT status_code FROM http_metrics WHERE endpoint='${endpoint}' ORDER BY ts DESC LIMIT 1;")
|
||||
FAIL_DETAIL="${FAIL_DETAIL}\n- \`${endpoint}\` 最近 ${FAIL_HERE} 次失败,最后状态码 ${LAST_STATUS}"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${FAIL_COUNT} -gt 0 ]]; then
|
||||
BODY="**${FAIL_COUNT} 个 OA 端点连续失败**(阈值 ${ALERT_HTTP_FAIL_COUNT} 次)\n\n${FAIL_DETAIL}\n\n> 请检查 OA 应用服务器 39.99.236.138 + nginx + e-cology 服务状态"
|
||||
dingtalk_alert "P1" "OA HTTP 端点故障" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'http', 'HTTP 端点故障', '${FAIL_COUNT} 个端点失败');"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 3. 业务层检测
|
||||
# ============================================================================
|
||||
# message_log 小时增量过低(钉钉推送可能断了)
|
||||
HOUR_MSG=$(sqlite_exec "SELECT msg_log_hourly FROM business_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
if [[ -n "${HOUR_MSG}" ]] && [[ ${HOUR_MSG} -lt ${ALERT_MSG_LOG_HOUR_MIN} ]] && [[ ${HOUR_MSG} -gt 0 ]]; then
|
||||
BODY="**ecology_message_log 小时增量仅 ${HOUR_MSG} 条**(阈值 ${ALERT_MSG_LOG_HOUR_MIN})\n\n> 钉钉推送可能阻塞。泛微云桥 e-Bridge 为黑盒线程,需登录 OA 后台检查「消息推送」日志或用 SHOW PROCESSLIST 看 active 线程"
|
||||
dingtalk_alert "P1" "钉钉推送疑似停滞" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'business', '钉钉推送疑似停滞', 'msg_hourly=${HOUR_MSG}');"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# 4. 基础设施层检测
|
||||
# ============================================================================
|
||||
# Docker daemon 挂了
|
||||
DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
PREV_DOCKER_OK=$(sqlite_exec "SELECT docker_daemon_ok FROM infra_metrics ORDER BY ts DESC LIMIT 2 OFFSET 1;")
|
||||
if [[ "${DOCKER_OK}" == "0" ]] && [[ "${PREV_DOCKER_OK}" == "0" ]]; then
|
||||
BODY="**Docker daemon 连续 2 次检测为不可用**\n\n> ⚠️ 今天的 rm -rf /ipaas 事故中 Docker daemon 重启过。请立即登录 DSH 检查:\n\`\`\`\nsystemctl status docker\ndocker info | grep Root\ndf -h\n\`\`\`"
|
||||
dingtalk_alert "P0" "Docker daemon 挂了!" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker daemon 不可用', '连续 2 次失败');"
|
||||
fi
|
||||
|
||||
# 容器数突变(对比最近 30 分钟的平均数)
|
||||
CURR_COUNT=$(sqlite_exec "SELECT docker_container_count FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
AVG_COUNT=$(sqlite_exec "SELECT AVG(docker_container_count) FROM infra_metrics WHERE ts > ${NOW}-1800;")
|
||||
if [[ -n "${CURR_COUNT}" ]] && [[ -n "${AVG_COUNT}" ]]; then
|
||||
DELTA=$(( CURR_COUNT - ${AVG_COUNT%%.*} ))
|
||||
ABS_DELTA=${DELTA#-}
|
||||
if [[ ${ABS_DELTA} -ge ${ALERT_DOCKER_CONTAINER_DELTA} ]]; then
|
||||
BODY="**Docker 容器数突变**:当前 ${CURR_COUNT},30 分钟均值 ${AVG_COUNT}\n\n> ⚠️ 容器异常消失!请立刻检查 docker ps -a"
|
||||
dingtalk_alert "P0" "Docker 容器数突变!" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'infra', 'Docker 容器数突变', 'curr=${CURR_COUNT} avg=${AVG_COUNT}');"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 磁盘
|
||||
DISK_PCT=$(sqlite_exec "SELECT disk_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
if [[ -n "${DISK_PCT}" ]] && (( ${DISK_PCT%%.*} >= ALERT_DISK_USAGE_PCT )); then
|
||||
BODY="**DSH 磁盘使用率 ${DISK_PCT}%**(阈值 ${ALERT_DISK_USAGE_PCT}%)\n\n> 请检查:df -h → du -sh /var/lib/docker/* → pm2 logs 清理 → 旧 SQLite 数据归档"
|
||||
dingtalk_alert "P1" "DSH 磁盘告警" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', '磁盘使用率过高', 'disk=${DISK_PCT}%');"
|
||||
fi
|
||||
|
||||
# inode
|
||||
INODE_PCT=$(sqlite_exec "SELECT inode_usage_pct FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
if [[ -n "${INODE_PCT}" ]] && (( ${INODE_PCT%%.*} >= ALERT_INODE_USAGE_PCT )); then
|
||||
BODY="**DSH inode 使用率 ${INODE_PCT}%**(阈值 ${ALERT_INODE_USAGE_PCT}%)\n\n> 可能有大量小文件,检查 find / -type f | wc -l"
|
||||
dingtalk_alert "P1" "DSH inode 告警" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P1', 'infra', 'inode 使用率过高', 'inode=${INODE_PCT}%');"
|
||||
fi
|
||||
|
||||
# Docker root 写入突增
|
||||
ROOT_WRITE=$(sqlite_exec "SELECT docker_root_write_mb FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
if [[ -n "${ROOT_WRITE}" ]] && (( ${ROOT_WRITE%%.*} >= ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN )); then
|
||||
BODY="**Docker data-root 1 分钟内写入 ${ROOT_WRITE}MB**(阈值 ${ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN}MB)\n\n> 可能有容器在疯狂写数据,检查:docker stats → docker logs"
|
||||
dingtalk_alert "P2" "Docker data-root 写入突增" "${BODY}"
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P2', 'infra', 'Docker root 写入突增', 'write_mb=${ROOT_WRITE}');"
|
||||
fi
|
||||
|
||||
log_info "[ALERT] ========== 告警检测完成 =========="
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 数据清理(每天 03:00)
|
||||
# 保留最近 7 天指标,alerts 和 alert_cooldown 保留 30 天
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
RETENTION_7D=$(( NOW - 7 * 86400 ))
|
||||
RETENTION_30D=$(( NOW - 30 * 86400 ))
|
||||
|
||||
log_info "[CLEANUP] 开始清理,阈值 ts < ${RETENTION_7D}(7天),alerts < ${RETENTION_30D}(30天)"
|
||||
|
||||
# 各层指标表
|
||||
for table in db_metrics http_metrics business_metrics infra_metrics db_processlist; do
|
||||
BEFORE=$(sqlite_exec "SELECT COUNT(*) FROM ${table};")
|
||||
sqlite_exec "DELETE FROM ${table} WHERE ts < ${RETENTION_7D};"
|
||||
AFTER=$(sqlite_exec "SELECT COUNT(*) FROM ${table};")
|
||||
log_info "[CLEANUP] ${table}: ${BEFORE} → ${AFTER}(删了 $((BEFORE - AFTER)) 条)"
|
||||
done
|
||||
|
||||
# alerts 保留 30 天
|
||||
BEFORE=$(sqlite_exec "SELECT COUNT(*) FROM alerts;")
|
||||
sqlite_exec "DELETE FROM alerts WHERE ts < ${RETENTION_30D};"
|
||||
AFTER=$(sqlite_exec "SELECT COUNT(*) FROM alerts;")
|
||||
log_info "[CLEANUP] alerts: ${BEFORE} → ${AFTER}(删了 $((BEFORE - AFTER)) 条)"
|
||||
|
||||
# VACUUM 压缩(可选,不频繁)
|
||||
if (( NOW % (86400 * 7) < 3600 )); then
|
||||
log_info "[CLEANUP] 每周 VACUUM SQLite..."
|
||||
sqlite_exec "VACUUM;"
|
||||
fi
|
||||
|
||||
log_info "[CLEANUP] 完成"
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 业务指标采集(每 1 小时)
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
|
||||
db_exec() {
|
||||
local sql="$1"
|
||||
MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -e "${sql}" 2>/dev/null
|
||||
}
|
||||
|
||||
MSG_TOTAL=$(db_exec "SELECT COUNT(*) FROM ecology_message_log;")
|
||||
MSG_TOTAL=${MSG_TOTAL:-0}
|
||||
MSG_FAIL_HOUR=$(db_exec "SELECT COUNT(*) FROM ecology_message_log WHERE retcode != 0 AND operTime >= DATE_SUB(NOW(), INTERVAL 1 HOUR);")
|
||||
MSG_FAIL_HOUR=${MSG_FAIL_HOUR:-0}
|
||||
ONGOING=$(db_exec "SELECT COUNT(*) FROM workflow_requestbase WHERE status = 2 OR (status IS NULL AND currentnodeid > 0);")
|
||||
ONGOING=${ONGOING:-0}
|
||||
OVERDUE=$(db_exec "SELECT COUNT(*) FROM workflow_requestlog WHERE timeout = 1 AND operatedate >= DATE_SUB(NOW(), INTERVAL 1 DAY);")
|
||||
OVERDUE=${OVERDUE:-0}
|
||||
|
||||
if [[ ${MSG_TOTAL} -gt 0 ]]; then
|
||||
FAIL_RATE=$(echo "scale=4; ${MSG_FAIL_HOUR} / (${MSG_TOTAL} + 0.001);" | bc 2>/dev/null || echo "0.0")
|
||||
else
|
||||
FAIL_RATE="0.0"
|
||||
fi
|
||||
|
||||
sqlite_exec "INSERT INTO business_metrics VALUES (
|
||||
${NOW}, ${MSG_TOTAL}, ${FAIL_RATE}, ${ONGOING}, ${OVERDUE}
|
||||
);"
|
||||
|
||||
log_info "[BIZ] ✅ msg_total=${MSG_TOTAL} fail_hour=${MSG_FAIL_HOUR} fail_rate=${FAIL_RATE} ongoing=${ONGOING} overdue=${OVERDUE}"
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · DB 层采集(每 30 秒)
|
||||
# 在 DSH 宿主机上直接 mysql 连 OA DB(DSH 本身就是跳板,不需要再 ssh)
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
|
||||
# ---- MySQL 直连(DSH 上直接连 OA DB 公网映射)----
|
||||
DB_MYSQL() {
|
||||
MYSQL_PWD="${DB_PASS}" timeout 12 mysql -h39.100.72.102 -P3306 -uecology ecology 2>/dev/null
|
||||
}
|
||||
|
||||
db_exec() {
|
||||
DB_MYSQL -e "$1"
|
||||
}
|
||||
|
||||
# ---- 1. SHOW STATUS ----
|
||||
log_info "[DB] SHOW STATUS"
|
||||
STATUS=$(DB_MYSQL -e "SHOW GLOBAL STATUS" | grep -E '^(Threads_running|Threads_connected|Threads_idle|Uptime|Innodb_rows_read|Innodb_rows_inserted|Innodb_rows_updated|Innodb_rows_deleted|Slow_queries)\t')
|
||||
|
||||
THREADS_RUNNING=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_running"{print $2}')
|
||||
THREADS_CONNECTED=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_connected"{print $2}')
|
||||
THREADS_IDLE=$(echo "${STATUS}" | awk -F'\t' '$1=="Threads_idle"{print $2}')
|
||||
UPTIME=$(echo "${STATUS}" | awk -F'\t' '$1=="Uptime"{print $2}')
|
||||
INNODB_READ=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_read"{print $2}')
|
||||
INNODB_INS=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_inserted"{print $2}')
|
||||
INNODB_UPD=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_updated"{print $2}')
|
||||
INNODB_DEL=$(echo "${STATUS}" | awk -F'\t' '$1=="Innodb_rows_deleted"{print $2}')
|
||||
SLOW_QUERIES=$(echo "${STATUS}" | awk -F'\t' '$1=="Slow_queries"{print $2}')
|
||||
|
||||
INNODB_READ=${INNODB_READ:-0}
|
||||
INNODB_INS=${INNODB_INS:-0}
|
||||
INNODB_UPD=${INNODB_UPD:-0}
|
||||
INNODB_DEL=${INNODB_DEL:-0}
|
||||
INNODB_WRITE=$(( INNODB_INS + INNODB_UPD + INNODB_DEL ))
|
||||
THREADS_RUNNING=${THREADS_RUNNING:-0}
|
||||
THREADS_CONNECTED=${THREADS_CONNECTED:-0}
|
||||
THREADS_IDLE=${THREADS_IDLE:-0}
|
||||
UPTIME=${UPTIME:-0}
|
||||
SLOW_QUERIES=${SLOW_QUERIES:-0}
|
||||
|
||||
# ---- 2. 上一次快照 → 差值 ----
|
||||
PREV=$(sqlite_exec "SELECT * FROM db_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
QPS_READ=0; QPS_WRITE=0; SLOW_DELTA=0
|
||||
|
||||
if [[ -n "${PREV}" ]]; then
|
||||
PREV_UPTIME=$(echo "${PREV}" | cut -d'|' -f5)
|
||||
PREV_READ=$(echo "${PREV}" | cut -d'|' -f6)
|
||||
PREV_WRITE=$(echo "${PREV}" | cut -d'|' -f7)
|
||||
PREV_SLOW=$(echo "${PREV}" | cut -d'|' -f9)
|
||||
PREV_UPTIME=${PREV_UPTIME:-0}; PREV_READ=${PREV_READ:-0}
|
||||
PREV_WRITE=${PREV_WRITE:-0}; PREV_SLOW=${PREV_SLOW:-0}
|
||||
DIFF_SEC=$(( NOW - PREV_UPTIME ))
|
||||
if [[ ${DIFF_SEC} -gt 0 ]]; then
|
||||
QPS_READ=$(( (INNODB_READ - PREV_READ) / DIFF_SEC ))
|
||||
QPS_WRITE=$(( (INNODB_WRITE - PREV_WRITE) / DIFF_SEC ))
|
||||
fi
|
||||
SLOW_DELTA=$(( SLOW_QUERIES - PREV_SLOW ))
|
||||
fi
|
||||
|
||||
# ---- 3. 写 SQLite ----
|
||||
sqlite_exec "INSERT INTO db_metrics VALUES (
|
||||
${NOW}, ${THREADS_RUNNING}, ${THREADS_CONNECTED}, ${THREADS_IDLE},
|
||||
${UPTIME}, ${QPS_READ}, ${QPS_WRITE}, ${SLOW_DELTA}
|
||||
);"
|
||||
|
||||
log_info "[DB] ✅ 写入:Threads_running=${THREADS_RUNNING} QPS_read=${QPS_READ} QPS_write=${QPS_WRITE}"
|
||||
|
||||
# ---- 4. 长查询 processlist ----
|
||||
log_info "[DB] processlist"
|
||||
PLIST=$(DB_MYSQL -e 'SHOW PROCESSLIST;')
|
||||
|
||||
echo "${PLIST}" | awk -F'\t' '
|
||||
NR==1 { next }
|
||||
NF < 8 { next }
|
||||
{
|
||||
pid=$1; user=$2; state=$6; time=$7; info=$8
|
||||
if (time+0 >= 5 && pid+0 > 0) {
|
||||
gsub(/\t/, " ", info)
|
||||
printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300)
|
||||
}
|
||||
}
|
||||
' | while IFS=$'\t' read -r pid time user state info; do
|
||||
[[ -z "${pid}" ]] && continue
|
||||
info_safe=$(echo "${info}" | sed "s/'/\"/g")
|
||||
sqlite_exec "INSERT INTO db_processlist (ts, pid, user, state, time_sec, info)
|
||||
VALUES (${NOW}, ${pid}, '${user}', '${state}', ${time}, '${info_safe}');"
|
||||
done
|
||||
|
||||
log_info "[DB] ✅ 完成"
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · HTTP 端点健康检查(每 30 秒)
|
||||
# 在 DSH 上直接 curl
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
|
||||
for endpoint in "${OA_ENDPOINTS[@]}"; do
|
||||
result=$(curl -sk -o /dev/null -w '%{http_code}|%{time_total}' --max-time 10 "${endpoint}" 2>/dev/null)
|
||||
status=$(echo "${result}" | cut -d'|' -f1)
|
||||
time_ms=$(echo "${result}" | cut -d'|' -f2 | awk '{printf "%d", $1*1000}')
|
||||
status=${status:-0}
|
||||
time_ms=${time_ms:-0}
|
||||
|
||||
case "${status}" in
|
||||
000) healthy=0 ;;
|
||||
2*) healthy=1 ;;
|
||||
3*) healthy=1 ;;
|
||||
*) healthy=0 ;;
|
||||
esac
|
||||
|
||||
sqlite_exec "INSERT INTO http_metrics (ts, endpoint, status_code, resp_time_ms, healthy)
|
||||
VALUES (${NOW}, '${endpoint}', ${status}, ${time_ms}, ${healthy});"
|
||||
|
||||
log_info "[HTTP] ${endpoint} status=${status} time=${time_ms}ms healthy=${healthy}"
|
||||
done
|
||||
|
||||
log_info "[HTTP] 全部完成"
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 基础设施层采集(每 30 秒)
|
||||
# 在 DSH 上直接执行 docker / df / du
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
NOW=$(date +%s)
|
||||
DOCKER_ROOT="/var/lib/docker"
|
||||
|
||||
# ---- Docker daemon + 容器数 ----
|
||||
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ')
|
||||
if [[ "${CONTAINER_COUNT}" =~ ^[0-9]+$ ]] && [[ ${CONTAINER_COUNT} -ge 0 ]]; then
|
||||
DOCKER_OK=1
|
||||
else
|
||||
DOCKER_OK=0
|
||||
CONTAINER_COUNT=0
|
||||
fi
|
||||
|
||||
# ---- Docker root 写入量(du overlay2 增量)----
|
||||
PREV_ROOT_WRITE=$(sqlite_exec "SELECT COALESCE(docker_root_write_mb, 0) FROM infra_metrics ORDER BY ts DESC LIMIT 1;")
|
||||
CURRENT_ROOT_MB=$(du -sm "${DOCKER_ROOT}/overlay2" 2>/dev/null | cut -f1 | tr -d ' ')
|
||||
CURRENT_ROOT_MB=${CURRENT_ROOT_MB:-0}
|
||||
PREV_ROOT_WRITE=${PREV_ROOT_WRITE:-0}
|
||||
|
||||
ROOT_WRITE_DELTA_MB=0
|
||||
if [[ -n "${PREV_ROOT_WRITE}" ]] && [[ "${PREV_ROOT_WRITE}" =~ ^[0-9.]+$ ]]; then
|
||||
DELTA=$(( CURRENT_ROOT_MB - ${PREV_ROOT_WRITE%%.*} ))
|
||||
if [[ ${DELTA} -gt 0 ]]; then
|
||||
ROOT_WRITE_DELTA_MB="${DELTA}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---- 磁盘 / inode / load ----
|
||||
DISK_PCT=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
|
||||
INODE_PCT=$(df -i / | tail -1 | awk '{print $5}' | tr -d '%')
|
||||
LOADAVG=$(cat /proc/loadavg | awk '{print $1}')
|
||||
|
||||
DISK_PCT=${DISK_PCT:-0}
|
||||
INODE_PCT=${INODE_PCT:-0}
|
||||
LOADAVG=${LOADAVG:-0}
|
||||
|
||||
sqlite_exec "INSERT INTO infra_metrics VALUES (
|
||||
${NOW}, ${DOCKER_OK}, ${CONTAINER_COUNT}, ${ROOT_WRITE_DELTA_MB},
|
||||
${DISK_PCT}, ${INODE_PCT}, ${LOADAVG}
|
||||
);"
|
||||
|
||||
log_info "[INFRA] ✅ docker_ok=${DOCKER_OK} containers=${CONTAINER_COUNT} root_write_mb=${ROOT_WRITE_DELTA_MB} disk=${DISK_PCT}% inode=${INODE_PCT}% loadavg=${LOADAVG}"
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 全局配置与公共函数
|
||||
# 所有采集/告警脚本 source 此文件,只修改这里的变量即可调整行为
|
||||
# ============================================================================
|
||||
set -u # 未定义变量报错
|
||||
# 注意:不写 set -e,采集脚本单条命令失败不应整体中断
|
||||
|
||||
# ---- 路径 ----
|
||||
export MONITOR_ROOT="/opt/oa-monitor"
|
||||
export MONITOR_DATA="${MONITOR_ROOT}/data"
|
||||
export MONITOR_LOGS="${MONITOR_ROOT}/logs"
|
||||
export MONITOR_DB="${MONITOR_DATA}/oa-monitor.db"
|
||||
|
||||
mkdir -p "${MONITOR_DATA}" "${MONITOR_LOGS}"
|
||||
|
||||
# ---- 工具 ----
|
||||
export MYSQL_CMD="mysql -h39.100.72.102 -P3306 -uecology"
|
||||
export DB_NAME="ecology"
|
||||
export DB_PASS='Z!BYT+jSDDyYHMdX+qFSLzJCYXW'
|
||||
|
||||
export SSH_DSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i ~/.ssh/id_ed25519 root@47.108.21.167"
|
||||
|
||||
# ---- OA 端点(DSH 上访问 OA 应用服务器内网 IP,更稳)----
|
||||
OA_HTTP="http://39.99.236.138"
|
||||
OA_ENDPOINTS=(
|
||||
"${OA_HTTP}:80/wui/main/Login.jsp"
|
||||
"${OA_HTTP}:8444/api/portal/portalinfo"
|
||||
"${OA_HTTP}:8444/api/portal/sso/getLoginInfo.jsp"
|
||||
"${OA_HTTP}/api/workflow/getRequestList"
|
||||
)
|
||||
|
||||
# ---- 钉钉 Webhook(群 723387268,走 DSH 本地转发)----
|
||||
export DINGTALK_WEBHOOK="https://dsh.gs-robot.com/dingtalk/webhook"
|
||||
export DINGTALK_SECRET=""
|
||||
|
||||
# ---- 告警阈值 ----
|
||||
export ALERT_THREADS_RUNNING_HIGH=50 # 当前活跃线程 > 50 告警
|
||||
export ALERT_SLOW_QUERY_SEC=30 # 慢查询 > 30s 告警
|
||||
export ALERT_SLOW_QUERY_COUNT=5 # 同进程号出现 5 次以上告警
|
||||
export ALERT_HTTP_FAIL_COUNT=3 # HTTP 端点连续失败 3 次告警
|
||||
export ALERT_MSG_LOG_HOUR_MIN=100 # 1 小时消息增量 < 100 条告警(可能钉钉推送断了)
|
||||
export ALERT_DOCKER_CONTAINER_DELTA=3 # 容器数突变 > ±3 告警
|
||||
export ALERT_DOCKER_ROOT_WRITE_MB_PER_MIN=1024 # Docker data-root 1 分钟写 > 1GB 告警
|
||||
export ALERT_DISK_USAGE_PCT=85 # 磁盘使用率 > 85% 告警
|
||||
export ALERT_INODE_USAGE_PCT=80 # inode 使用率 > 80% 告警
|
||||
export ALERT_DOCKER_DAEMON_FAIL_COUNT=2 # Docker daemon 连续失败 2 次告警
|
||||
export ALERT_LONG_QUERY_SEC=300 # 长查询 > 300s 触发 KILL(dry-run 默认)
|
||||
|
||||
# ---- 告警抑制 ----
|
||||
export ALERT_COOLDOWN_SEC=600 # 同一告警类型 10 分钟内不重复发
|
||||
|
||||
# ---- SQLite 工具函数 ----
|
||||
sqlite_exec() {
|
||||
sqlite3 "${MONITOR_DB}" "$@" 2>/dev/null
|
||||
}
|
||||
|
||||
# ---- 日志函数 ----
|
||||
log() {
|
||||
local level="$1"; shift
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MONITOR_LOGS}/oa-monitor.log"
|
||||
}
|
||||
|
||||
log_info() { log "INFO" "$@"; }
|
||||
log_warn() { log "WARN" "$@"; }
|
||||
log_error() { log "ERROR" "$@"; }
|
||||
|
||||
# ---- 钉钉告警(markdown 格式)----
|
||||
# 用法:dingtalk_alert <level> <title> <markdown_body>
|
||||
# level: P0 / P1 / P2
|
||||
dingtalk_alert() {
|
||||
local level="$1" title="$2" body="$3"
|
||||
local timestamp
|
||||
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# 告警冷却检查
|
||||
local key="${level}:${title}"
|
||||
local last_ts
|
||||
last_ts=$(sqlite_exec "SELECT COALESCE(last_alert_ts, 0) FROM alert_cooldown WHERE alert_key='${key}';")
|
||||
local now_epoch
|
||||
now_epoch=$(date +%s)
|
||||
if [[ "${last_ts}" =~ ^[0-9]+$ ]] && (( now_epoch - last_ts < ALERT_COOLDOWN_SEC )); then
|
||||
log_info "告警抑制:${key}(距上次 ${now_epoch}s < ${ALERT_COOLDOWN_SEC}s)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 构造 JSON
|
||||
local json
|
||||
json=$(cat <<JSON
|
||||
{"msgtype":"markdown","markdown":{"title":"[OA监控] ${level} ${title}","text":"### [OA监控] **${level}** ${title}\n\n**时间**:${timestamp}\n\n**主机**:DSH (47.108.21.167)\n\n---\n\n${body}\n\n---\n\n> OA 运维监控平台 · $(basename "$0")"}}
|
||||
JSON
|
||||
)
|
||||
|
||||
# 发送
|
||||
local http_code
|
||||
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "${json}" "${DINGTALK_WEBHOOK}" 2>/dev/null)
|
||||
|
||||
if [[ "${http_code}" == "200" ]]; then
|
||||
log_info "钉钉告警发送成功:[${level}] ${title}"
|
||||
sqlite_exec "INSERT OR REPLACE INTO alert_cooldown VALUES ('${key}', ${now_epoch});"
|
||||
else
|
||||
log_error "钉钉告警发送失败(HTTP ${http_code}):[${level}] ${title}"
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>OA 运维监控 · 告警历史</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body { font-family: -apple-system, 'PingFang SC', sans-serif; background: #0d1117; color: #c9d1d9; padding: 16px; }
|
||||
h1 { font-size: 20px; color: #58a6ff; margin-bottom: 16px; }
|
||||
.filter-bar { margin-bottom: 16px; display: flex; gap: 12px; align-items: center; flex-wrap: wrap; }
|
||||
.filter-bar button { background: #21262d; color: #c9d1d9; border: 1px solid #30363d; padding: 6px 14px; border-radius: 6px; cursor: pointer; font-size: 13px; }
|
||||
.filter-bar button.active { background: #1f6feb; border-color: #1f6feb; }
|
||||
.filter-bar button:hover { border-color: #58a6ff; }
|
||||
.stats { display: flex; gap: 16px; margin-bottom: 16px; }
|
||||
.stat { background: #161b22; border: 1px solid #30363d; padding: 12px 16px; border-radius: 8px; }
|
||||
.stat .num { font-size: 24px; font-weight: 700; }
|
||||
.stat .label { color: #8b949e; font-size: 12px; margin-top: 2px; }
|
||||
.tag { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 11px; font-weight: 600; }
|
||||
.tag-p0 { background: rgba(248,81,73,0.25); color: #ff7b72; }
|
||||
.tag-p1 { background: rgba(210,153,34,0.25); color: #d29922; }
|
||||
.tag-p2 { background: rgba(56,139,253,0.25); color: #58a6ff; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 13px; background: #161b22; border-radius: 8px; overflow: hidden; }
|
||||
th { background: #21262d; color: #8b949e; padding: 10px 12px; text-align: left; font-weight: 600; position: sticky; top: 0; }
|
||||
td { padding: 9px 12px; border-top: 1px solid #21262d; }
|
||||
tr:hover td { background: #1f2428; }
|
||||
.btn-ack { background: #1f6feb; color: white; border: none; padding: 4px 10px; border-radius: 4px; cursor: pointer; font-size: 12px; }
|
||||
.btn-resolve { background: #238636; color: white; border: none; padding: 4px 10px; border-radius: 4px; cursor: pointer; font-size: 12px; }
|
||||
.ack-badge { color: #3fb950; font-weight: 600; font-size: 11px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>🚨 告警历史</h1>
|
||||
<div class="filter-bar">
|
||||
<button data-filter="all" class="active">全部</button>
|
||||
<button data-filter="P0">P0</button>
|
||||
<button data-filter="P1">P1</button>
|
||||
<button data-filter="P2">P2</button>
|
||||
<button data-filter="unack">未确认</button>
|
||||
</div>
|
||||
|
||||
<div class="stats">
|
||||
<div class="stat"><div class="num" id="statTotal">0</div><div class="label">告警总数</div></div>
|
||||
<div class="stat"><div class="num" id="statP0">0</div><div class="label">P0 告警</div></div>
|
||||
<div class="stat"><div class="num" id="statUnack">0</div><div class="label">未确认</div></div>
|
||||
</div>
|
||||
|
||||
<table id="alertsTable">
|
||||
<thead><tr><th>级别</th><th>时间</th><th>类别</th><th>标题</th><th>详情</th><th>状态</th></tr></thead>
|
||||
<tbody id="alertsBody"></tbody>
|
||||
</table>
|
||||
|
||||
<script>
|
||||
const API = window.location.origin + '/monitor';
|
||||
let currentFilter = 'all';
|
||||
let allAlerts = [];
|
||||
|
||||
document.querySelectorAll('.filter-bar button').forEach(b => {
|
||||
b.addEventListener('click', () => {
|
||||
document.querySelectorAll('.filter-bar button').forEach(x => x.classList.remove('active'));
|
||||
b.classList.add('active');
|
||||
currentFilter = b.dataset.filter;
|
||||
render();
|
||||
});
|
||||
});
|
||||
|
||||
async function refresh() {
|
||||
const res = await fetch(API + '/api/alerts?limit=200');
|
||||
allAlerts = await res.json();
|
||||
render();
|
||||
}
|
||||
|
||||
function render() {
|
||||
let filtered = allAlerts;
|
||||
if (currentFilter === 'unack') filtered = filtered.filter(a => !a.ack);
|
||||
else if (['P0','P1','P2'].includes(currentFilter)) filtered = filtered.filter(a => a.level === currentFilter);
|
||||
|
||||
// stats
|
||||
document.getElementById('statTotal').textContent = allAlerts.length;
|
||||
document.getElementById('statP0').textContent = allAlerts.filter(a => a.level === 'P0').length;
|
||||
document.getElementById('statUnack').textContent = allAlerts.filter(a => !a.ack).length;
|
||||
|
||||
const tbody = document.getElementById('alertsBody');
|
||||
tbody.innerHTML = filtered.map(a => {
|
||||
const cls = a.level === 'P0' ? 'tag-p0' : (a.level === 'P1' ? 'tag-p1' : 'tag-p2');
|
||||
const ack = a.ack ? '<span class="ack-badge">已确认</span>' : '<span style="color:#d29922;">未确认</span>';
|
||||
const reso = a.resolved ? '✅ 已恢复' : (a.ack ? '' : '');
|
||||
return `<tr>
|
||||
<td><span class="tag ${cls}">${a.level}</span></td>
|
||||
<td>${a.ts_iso}</td>
|
||||
<td>${a.category}</td>
|
||||
<td><b>${a.title}</b></td>
|
||||
<td style="max-width:300px;">${a.detail||''}</td>
|
||||
<td>${ack} ${reso}</td>
|
||||
</tr>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
refresh();
|
||||
setInterval(refresh, 30000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,283 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>OA 运维监控平台 · 总览</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/echarts@5.5.0/dist/echarts.min.js"></script>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'PingFang SC', 'Microsoft YaHei', sans-serif; background: #0d1117; color: #c9d1d9; padding: 16px; }
|
||||
h1 { font-size: 20px; color: #58a6ff; margin-bottom: 16px; }
|
||||
.status-bar { display: flex; gap: 12px; margin-bottom: 16px; flex-wrap: wrap; }
|
||||
.status-chip { background: #161b22; border-radius: 6px; padding: 10px 14px; border: 1px solid #30363d; display: flex; align-items: center; gap: 8px; font-size: 13px; }
|
||||
.status-chip.ok { border-color: #238636; background: rgba(35,134,54,0.1); }
|
||||
.status-chip.warn { border-color: #d29922; background: rgba(210,153,34,0.1); }
|
||||
.status-chip.err { border-color: #f85149; background: rgba(248,81,73,0.1); animation: pulse 2s infinite; }
|
||||
.status-chip .dot { width: 10px; height: 10px; border-radius: 50%; }
|
||||
.status-chip.ok .dot { background: #238636; }
|
||||
.status-chip.warn .dot { background: #d29922; }
|
||||
.status-chip.err .dot { background: #f85149; }
|
||||
@keyframes pulse { 0%,100% { opacity: 1; } 50% { opacity: 0.5; } }
|
||||
|
||||
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(480px, 1fr)); gap: 16px; }
|
||||
.card { background: #161b22; border-radius: 8px; border: 1px solid #30363d; padding: 14px; }
|
||||
.card h2 { font-size: 14px; color: #8b949e; margin-bottom: 10px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; }
|
||||
.chart { width: 100%; height: 220px; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 12px; }
|
||||
th { background: #21262d; color: #8b949e; padding: 6px 8px; text-align: left; font-weight: 600; }
|
||||
td { padding: 5px 8px; border-top: 1px solid #21262d; word-break: break-all; }
|
||||
.tag { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 11px; font-weight: 600; }
|
||||
.tag-ok { background: rgba(35,134,54,0.2); color: #3fb950; }
|
||||
.tag-fail { background: rgba(248,81,73,0.2); color: #f85149; }
|
||||
.tag-p0 { background: rgba(248,81,73,0.25); color: #ff7b72; }
|
||||
.tag-p1 { background: rgba(210,153,34,0.25); color: #d29922; }
|
||||
.tag-p2 { background: rgba(56,139,253,0.25); color: #58a6ff; }
|
||||
.empty { text-align: center; color: #484f58; padding: 20px; font-size: 13px; }
|
||||
.endpoints-table td:first-child { max-width: 220px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.endpoints-table td.status-ok { color: #3fb950; font-weight: 600; }
|
||||
.endpoints-table td.status-fail { color: #f85149; font-weight: 600; }
|
||||
.ts { color: #6e7681; font-size: 11px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>🛡️ OA 运维监控平台 · 总览</h1>
|
||||
|
||||
<div class="status-bar" id="statusBar">
|
||||
<div class="status-chip" id="chip-db"><span class="dot"></span><span>DB 层</span><span class="ts" id="ts-db"></span></div>
|
||||
<div class="status-chip" id="chip-http"><span class="dot"></span><span>HTTP 端点</span><span class="ts" id="ts-http"></span></div>
|
||||
<div class="status-chip" id="chip-business"><span class="dot"></span><span>业务/钉钉推送</span><span class="ts" id="ts-biz"></span></div>
|
||||
<div class="status-chip" id="chip-infra"><span class="dot"></span><span>基础设施</span><span class="ts" id="ts-infra"></span></div>
|
||||
<div class="status-chip err" id="chip-active" style="display:none;"><span class="dot"></span><span id="activeCount"></span> 活跃告警</div>
|
||||
</div>
|
||||
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
<h2>DB · Threads Running / 30s</h2>
|
||||
<div id="chart-threads" class="chart"></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>DB · QPS Read / Write</h2>
|
||||
<div id="chart-qps" class="chart"></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>DB · 当前活跃长查询(>30s)</h2>
|
||||
<div id="longQueriesTable"></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>HTTP · 端点状态(最近 2 分钟)</h2>
|
||||
<div id="httpTable"></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>业务 · 钉钉推送小时增量</h2>
|
||||
<div id="chart-msg" class="chart"></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>基础设施 · 容器数 / 磁盘 / Docker</h2>
|
||||
<div id="chart-infra" class="chart"></div>
|
||||
</div>
|
||||
<div class="card" style="grid-column: 1/-1;">
|
||||
<h2>🔴 活跃告警</h2>
|
||||
<div id="alertsTable"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const API = window.location.origin + '/monitor';
|
||||
const REFRESH_MS = 15000;
|
||||
|
||||
// ---- Charts ----
|
||||
const chartThreads = echarts.init(document.getElementById('chart-threads'), 'dark');
|
||||
const chartQps = echarts.init(document.getElementById('chart-qps'), 'dark');
|
||||
const chartMsg = echarts.init(document.getElementById('chart-msg'), 'dark');
|
||||
const chartInfra = echarts.init(document.getElementById('chart-infra'), 'dark');
|
||||
|
||||
function formatTime(ts) {
|
||||
const d = new Date(ts * 1000);
|
||||
return d.toLocaleTimeString('zh-CN', { hour12: false });
|
||||
}
|
||||
|
||||
async function fetchJSON(url) {
|
||||
const res = await fetch(API + url, { cache: 'no-cache' });
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
try {
|
||||
const overview = await fetchJSON('/api/overview');
|
||||
const db30 = await fetchJSON('/api/db/recent?minutes=30');
|
||||
const biz24 = await fetchJSON('/api/business/recent?hours=24');
|
||||
const infra30 = await fetchJSON('/api/infra/recent?minutes=30');
|
||||
|
||||
renderStatus(overview);
|
||||
renderCharts(db30, biz24, infra30);
|
||||
renderLongQueries(overview.long_queries);
|
||||
renderHttpTable(overview.http_summary);
|
||||
renderAlerts(overview.active_alerts);
|
||||
} catch (e) {
|
||||
console.error('refresh fail:', e);
|
||||
}
|
||||
}
|
||||
|
||||
function renderStatus(o) {
|
||||
// DB
|
||||
const dbOk = o.db_latest && o.db_latest.threads_running >= 0;
|
||||
updateChip('chip-db', 'ts-db', dbOk ? 'ok' : 'err', o.db_latest?.ts_iso);
|
||||
|
||||
// HTTP
|
||||
const httpAll = o.http_summary || [];
|
||||
const httpOk = httpAll.length > 0 && httpAll.every(e => e.healthy === 1);
|
||||
updateChip('chip-http', 'ts-http', httpOk ? 'ok' : (httpAll.some(e => !e.healthy) ? 'warn' : 'err'), httpAll[0]?.ts);
|
||||
|
||||
// Business
|
||||
const bizOk = o.biz_latest && o.biz_latest.msg_log_hourly >= 0;
|
||||
updateChip('chip-business', 'ts-biz', bizOk ? 'ok' : 'warn', o.biz_latest?.ts_iso);
|
||||
|
||||
// Infra
|
||||
const infraOk = o.infra_latest && o.infra_latest.docker_daemon_ok === 1;
|
||||
const infraClass = infraOk ? 'ok' : 'err';
|
||||
updateChip('chip-infra', 'ts-infra', infraClass, o.infra_latest?.ts_iso);
|
||||
|
||||
// Active alerts
|
||||
const aa = o.active_alerts || [];
|
||||
const chipActive = document.getElementById('chip-active');
|
||||
if (aa.length > 0) {
|
||||
chipActive.style.display = 'flex';
|
||||
document.getElementById('activeCount').textContent = aa.length + ' 活跃告警';
|
||||
} else {
|
||||
chipActive.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
function updateChip(id, tsId, cls, ts) {
|
||||
const el = document.getElementById(id);
|
||||
el.className = 'status-chip ' + cls;
|
||||
if (ts) document.getElementById(tsId).textContent = ts;
|
||||
}
|
||||
|
||||
function renderCharts(db, biz, infra) {
|
||||
const dbX = db.map(r => formatTime(r.ts));
|
||||
const dbThreads = db.map(r => r.threads_running);
|
||||
const dbQpsR = db.map(r => r.qps_read);
|
||||
const dbQpsW = db.map(r => r.qps_write);
|
||||
|
||||
chartThreads.setOption({
|
||||
tooltip: { trigger: 'axis' },
|
||||
grid: { left: 40, right: 20, top: 20, bottom: 30 },
|
||||
xAxis: { type: 'category', data: dbX, axisLine: { lineStyle: { color: '#30363d' } } },
|
||||
yAxis: { type: 'value', splitLine: { lineStyle: { color: '#21262d' } } },
|
||||
series: [{ name: 'Threads_running', type: 'line', data: dbThreads, smooth: true,
|
||||
lineStyle: { color: '#58a6ff' }, areaStyle: { color: 'rgba(88,166,255,0.15)' },
|
||||
markLine: { data: [{ yAxis: 50, label: '阈值50', lineStyle: { color: '#f85149', type: 'dashed' } }] }
|
||||
}]
|
||||
});
|
||||
|
||||
chartQps.setOption({
|
||||
tooltip: { trigger: 'axis' },
|
||||
legend: { data: ['Read QPS', 'Write QPS'], textStyle: { color: '#c9d1d9' } },
|
||||
grid: { left: 40, right: 20, top: 30, bottom: 30 },
|
||||
xAxis: { type: 'category', data: dbX, axisLine: { lineStyle: { color: '#30363d' } } },
|
||||
yAxis: { type: 'value', splitLine: { lineStyle: { color: '#21262d' } } },
|
||||
series: [
|
||||
{ name: 'Read QPS', type: 'line', data: dbQpsR, smooth: true, lineStyle: { color: '#3fb950' } },
|
||||
{ name: 'Write QPS', type: 'line', data: dbQpsW, smooth: true, lineStyle: { color: '#d29922' } }
|
||||
]
|
||||
});
|
||||
|
||||
// Biz chart
|
||||
const bizX = biz.map(r => r.ts_iso?.substring(11,16) || r.ts);
|
||||
const bizMsg = biz.map(r => r.msg_log_hourly);
|
||||
chartMsg.setOption({
|
||||
tooltip: { trigger: 'axis' },
|
||||
grid: { left: 60, right: 20, top: 20, bottom: 30 },
|
||||
xAxis: { type: 'category', data: bizX, axisLine: { lineStyle: { color: '#30363d' } } },
|
||||
yAxis: { type: 'value', splitLine: { lineStyle: { color: '#21262d' } } },
|
||||
series: [{ name: '小时增量', type: 'bar', data: bizMsg, itemStyle: { color: '#a371f7' },
|
||||
markLine: { data: [{ yAxis: 100, label: '阈值100', lineStyle: { color: '#f85149', type: 'dashed' } }] }
|
||||
}]
|
||||
});
|
||||
|
||||
// Infra chart
|
||||
const infraX = infra.map(r => formatTime(r.ts));
|
||||
const infraC = infra.map(r => r.docker_container_count);
|
||||
const infraD = infra.map(r => r.disk_usage_pct);
|
||||
chartInfra.setOption({
|
||||
tooltip: { trigger: 'axis' },
|
||||
legend: { data: ['容器数', '磁盘%'], textStyle: { color: '#c9d1d9' } },
|
||||
grid: { left: 40, right: 50, top: 30, bottom: 30 },
|
||||
xAxis: { type: 'category', data: infraX, axisLine: { lineStyle: { color: '#30363d' } } },
|
||||
yAxis: [
|
||||
{ type: 'value', name: '容器数', splitLine: { lineStyle: { color: '#21262d' } } },
|
||||
{ type: 'value', name: '磁盘%', min: 0, max: 100, splitLine: { show: false } }
|
||||
],
|
||||
series: [
|
||||
{ name: '容器数', type: 'line', data: infraC, smooth: true, lineStyle: { color: '#3fb950' }, areaStyle: { color: 'rgba(63,185,80,0.1)' } },
|
||||
{ name: '磁盘%', type: 'line', yAxisIndex: 1, data: infraD, smooth: true, lineStyle: { color: '#d29922' },
|
||||
markLine: { data: [{ yAxis: 85, label: '85%', lineStyle: { color: '#f85149', type: 'dashed' } }] }
|
||||
}
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
function renderLongQueries(queries) {
|
||||
const el = document.getElementById('longQueriesTable');
|
||||
if (!queries || queries.length === 0) {
|
||||
el.innerHTML = '<div class="empty">✅ 当前无活跃长查询(>30s)</div>';
|
||||
return;
|
||||
}
|
||||
let html = '<table><tr><th>PID</th><th>Time(s)</th><th>User</th><th>State</th><th>SQL</th></tr>';
|
||||
queries.forEach(q => {
|
||||
html += `<tr><td>${q.pid}</td><td style="color:#f85149;font-weight:600;">${q.time_sec}</td><td>${q.user}</td><td>${q.state||'-'}</td><td style="max-width:260px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="${q.info||''}">${q.info||''}</td></tr>`;
|
||||
});
|
||||
html += '</table>';
|
||||
el.innerHTML = html;
|
||||
}
|
||||
|
||||
function renderHttpTable(list) {
|
||||
const el = document.getElementById('httpTable');
|
||||
if (!list || list.length === 0) {
|
||||
el.innerHTML = '<div class="empty">等待数据...</div>';
|
||||
return;
|
||||
}
|
||||
let html = '<table class="endpoints-table"><tr><th>端点</th><th>状态</th><th>耗时(ms)</th><th>时间</th></tr>';
|
||||
const seen = new Set();
|
||||
list.sort((a, b) => (b.healthy - a.healthy) || (a.endpoint.localeCompare(b.endpoint)));
|
||||
list.forEach(e => {
|
||||
if (seen.has(e.endpoint)) return;
|
||||
seen.add(e.endpoint);
|
||||
const ok = e.healthy === 1;
|
||||
html += `<tr><td title="${e.endpoint}">${e.endpoint}</td>
|
||||
<td class="${ok?'status-ok':'status-fail'}">${ok?'OK ('+e.status_code+')':'FAIL ('+e.status_code+')'}</td>
|
||||
<td>${e.resp_time_ms}</td><td class="ts">${e.ts}</td></tr>`;
|
||||
});
|
||||
html += '</table>';
|
||||
el.innerHTML = html;
|
||||
}
|
||||
|
||||
function renderAlerts(alerts) {
|
||||
const el = document.getElementById('alertsTable');
|
||||
if (!alerts || alerts.length === 0) {
|
||||
el.innerHTML = '<div class="empty">🎉 当前无活跃告警</div>';
|
||||
return;
|
||||
}
|
||||
let html = '<table><tr><th>级别</th><th>时间</th><th>类别</th><th>标题</th><th>详情</th></tr>';
|
||||
alerts.forEach(a => {
|
||||
const cls = a.level === 'P0' ? 'tag-p0' : (a.level === 'P1' ? 'tag-p1' : 'tag-p2');
|
||||
html += `<tr><td><span class="tag ${cls}">${a.level}</span></td>
|
||||
<td class="ts">${a.ts_iso}</td>
|
||||
<td>${a.category}</td>
|
||||
<td><b>${a.title}</b></td>
|
||||
<td>${a.detail||''}</td></tr>`;
|
||||
});
|
||||
html += '</table>';
|
||||
el.innerHTML = html;
|
||||
}
|
||||
|
||||
refresh();
|
||||
setInterval(refresh, REFRESH_MS);
|
||||
window.addEventListener('resize', () => {
|
||||
chartThreads.resize(); chartQps.resize(); chartMsg.resize(); chartInfra.resize();
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · SQLite 数据库初始化
|
||||
# 只建表,不删数据;幂等可重复执行
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
log_info "初始化 SQLite 数据库:${MONITOR_DB}"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS db_metrics (
|
||||
ts INTEGER PRIMARY KEY,
|
||||
threads_running INTEGER,
|
||||
threads_connected INTEGER,
|
||||
threads_idle INTEGER,
|
||||
uptime_sec INTEGER,
|
||||
qps_read INTEGER,
|
||||
qps_write INTEGER,
|
||||
slow_queries INTEGER -- 近 30s 慢查询数(>30s)
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS db_processlist (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER,
|
||||
pid INTEGER,
|
||||
user TEXT,
|
||||
state TEXT,
|
||||
time_sec INTEGER,
|
||||
info TEXT
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_db_proc_ts ON db_processlist(ts);"
|
||||
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_db_proc_time ON db_processlist(time_sec);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS http_metrics (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER,
|
||||
endpoint TEXT,
|
||||
status_code INTEGER,
|
||||
resp_time_ms INTEGER,
|
||||
healthy INTEGER -- 0/1
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_http_ts_ep ON http_metrics(ts, endpoint);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS business_metrics (
|
||||
ts INTEGER PRIMARY KEY,
|
||||
msg_log_hourly INTEGER, -- 本小时 ecology_message_log 增量
|
||||
msg_log_fail_rate REAL, -- 失败率
|
||||
ongoing_requests INTEGER, -- 进行中流程数
|
||||
overdue_tasks INTEGER -- 超时待办数
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS infra_metrics (
|
||||
ts INTEGER PRIMARY KEY,
|
||||
docker_daemon_ok INTEGER, -- 0/1
|
||||
docker_container_count INTEGER,
|
||||
docker_root_write_mb REAL, -- 1 分钟 Docker data-root 写入量 MB
|
||||
disk_usage_pct REAL,
|
||||
inode_usage_pct REAL,
|
||||
load_avg REAL
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS alerts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER,
|
||||
level TEXT, -- P0/P1/P2
|
||||
category TEXT, -- db/http/business/infra
|
||||
title TEXT,
|
||||
detail TEXT,
|
||||
ack INTEGER DEFAULT 0, -- 0=未确认 1=已确认
|
||||
resolved INTEGER DEFAULT 0 -- 0=未恢复 1=已恢复
|
||||
);"
|
||||
|
||||
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_alerts_ts ON alerts(ts);"
|
||||
sqlite_exec "CREATE INDEX IF NOT EXISTS idx_alerts_ack ON alerts(ack, resolved);"
|
||||
|
||||
sqlite_exec "CREATE TABLE IF NOT EXISTS alert_cooldown (
|
||||
alert_key TEXT PRIMARY KEY,
|
||||
last_alert_ts INTEGER
|
||||
);"
|
||||
|
||||
log_info "SQLite schema 初始化完成"
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# OA Monitor · 长查询检测 dry-run KILL(每 10 分钟)
|
||||
# 默认 DRY_RUN=1(只报告,不真 KILL)。--execute 才真的 KILL
|
||||
# ============================================================================
|
||||
set -u
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../lib/common.sh"
|
||||
|
||||
DRY_RUN=1
|
||||
for arg in "$@"; do
|
||||
[[ "${arg}" == "--execute" ]] && DRY_RUN=0
|
||||
[[ "${arg}" == "--dry-run" ]] && DRY_RUN=1
|
||||
done
|
||||
|
||||
NOW=$(date +%s)
|
||||
|
||||
db_exec() {
|
||||
local sql="$1"
|
||||
MYSQL_PWD="${DB_PASS}" timeout 30 mysql -h39.100.72.102 -P3306 -uecology ecology -e "${sql}" 2>/dev/null
|
||||
}
|
||||
|
||||
log_info "[LONG-CHECK] 开始(DRY_RUN=${DRY_RUN},阈值 ${ALERT_LONG_QUERY_SEC}s)"
|
||||
|
||||
RESULT=$(db_exec "SHOW FULL PROCESSLIST;" | awk -F'\t' -v t="${ALERT_LONG_QUERY_SEC}" '
|
||||
NR==1 { next }
|
||||
NF < 10 { next }
|
||||
{
|
||||
pid=$1; user=$2; state=$6; time=$7; info=$9
|
||||
if (time+0 >= t+0 && time+0 > 0) {
|
||||
gsub(/\t/, " ", info)
|
||||
printf "%d\t%d\t%s\t%s\t%s\n", pid, time, user, state, substr(info,1,300)
|
||||
}
|
||||
}
|
||||
')
|
||||
|
||||
if [[ -z "${RESULT}" ]]; then
|
||||
log_info "[LONG-CHECK] ✅ 无长查询"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
COUNT=$(echo "${RESULT}" | wc -l | tr -d ' ')
|
||||
log_warn "[LONG-CHECK] ⚠️ ${COUNT} 条长查询"
|
||||
|
||||
DETAIL=""
|
||||
while IFS=$'\t' read -r pid time user state info; do
|
||||
[[ -z "${pid}" ]] && continue
|
||||
DETAIL="${DETAIL}\n- PID \`${pid}\` **${time}s** \`${user}\` ${state}"
|
||||
DETAIL="${DETAIL}\n \`\`\`\n ${info}\n \`\`\`"
|
||||
done <<< "${RESULT}"
|
||||
|
||||
MODE_LABEL=$([[ ${DRY_RUN} -eq 1 ]] && echo "🔒 DRY-RUN" || echo "🚨 执行 KILL")
|
||||
BODY="**${COUNT} 条长查询**(${ALERT_LONG_QUERY_SEC}s 阈值)· ${MODE_LABEL}\n\n${DETAIL}"
|
||||
dingtalk_alert "P0" "长查询阻塞需处理" "${BODY}"
|
||||
|
||||
sqlite_exec "INSERT INTO alerts (ts, level, category, title, detail) VALUES (${NOW}, 'P0', 'db', '长查询', '${COUNT} 条, dry_run=${DRY_RUN}');"
|
||||
|
||||
if [[ ${DRY_RUN} -eq 0 ]]; then
|
||||
log_warn "[LONG-CHECK] 执行 KILL!"
|
||||
while IFS=$'\t' read -r pid time user state info; do
|
||||
[[ -z "${pid}" ]] && continue
|
||||
db_exec "KILL ${pid};"
|
||||
log_warn "[LONG-CHECK] KILL ${pid} (${time}s, ${user})"
|
||||
done <<< "${RESULT}"
|
||||
else
|
||||
log_info "[LONG-CHECK] DRY-RUN,不执行 KILL"
|
||||
fi
|
||||
@@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env python3
|
||||
# ============================================================================
|
||||
# OA Monitor · Python HTTP API(纯标准库,无第三方依赖)
|
||||
# 唯一职责:读 SQLite → 吐 JSON
|
||||
# 只读,绝不写 SQLite;所有 POST/PUT/DELETE 返回 405
|
||||
# ============================================================================
|
||||
import http.server, json, sqlite3, os, sys, urllib.parse
|
||||
from datetime import datetime, timezone
|
||||
|
||||
DB_PATH = "/opt/oa-monitor/data/oa-monitor.db"
|
||||
DASHBOARD_DIR = "/opt/oa-monitor/dashboard"
|
||||
|
||||
def get_db():
|
||||
return sqlite3.connect(DB_PATH, timeout=5)
|
||||
|
||||
def ts_to_iso(ts):
|
||||
if not ts: return None
|
||||
return datetime.fromtimestamp(int(ts)).strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
def fetch_all(sql, params=()):
|
||||
db = get_db()
|
||||
db.row_factory = sqlite3.Row
|
||||
try:
|
||||
rows = db.execute(sql, params).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def fetch_one(sql, params=()):
|
||||
rows = fetch_all(sql, params)
|
||||
return rows[0] if rows else None
|
||||
|
||||
class MonitorHandler(http.server.BaseHTTPRequestHandler):
|
||||
# ---- CORS + JSON 头 ----
|
||||
def _send_json(self, data, code=200):
|
||||
body = json.dumps(data, ensure_ascii=False, default=str).encode()
|
||||
self.send_response(code)
|
||||
self.send_header('Content-Type', 'application/json; charset=utf-8')
|
||||
self.send_header('Content-Length', str(len(body)))
|
||||
self.send_header('Access-Control-Allow-Origin', '*')
|
||||
self.send_header('Access-Control-Allow-Methods', 'GET, OPTIONS')
|
||||
self.send_header('Access-Control-Allow-Headers', 'Content-Type')
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_OPTIONS(self):
|
||||
self._send_json({})
|
||||
|
||||
# ---- 路由 ----
|
||||
def do_GET(self):
|
||||
parsed = urllib.parse.urlparse(self.path)
|
||||
path = parsed.path.rstrip('/')
|
||||
qs = urllib.parse.parse_qs(parsed.query)
|
||||
|
||||
try:
|
||||
if path.startswith('/api/'):
|
||||
self._route_api(path, qs)
|
||||
else:
|
||||
self._route_static(path)
|
||||
except Exception as e:
|
||||
self._send_json({"error": str(e)}, 500)
|
||||
|
||||
def _route_api(self, path, qs):
|
||||
if path == '/api/health':
|
||||
self._health()
|
||||
elif path == '/api/overview':
|
||||
self._overview()
|
||||
elif path == '/api/db/recent':
|
||||
self._db_recent(qs)
|
||||
elif path == '/api/db/long_queries':
|
||||
self._db_long_queries(qs)
|
||||
elif path == '/api/http/recent':
|
||||
self._http_recent(qs)
|
||||
elif path == '/api/business/recent':
|
||||
self._biz_recent(qs)
|
||||
elif path == '/api/infra/recent':
|
||||
self._infra_recent(qs)
|
||||
elif path == '/api/alerts':
|
||||
self._alerts(qs)
|
||||
else:
|
||||
self._send_json({"error": "not found", "path": path}, 404)
|
||||
|
||||
# ---- 静态文件服务 ----
|
||||
def _route_static(self, path):
|
||||
# 安全:禁止 .. 和 /etc 等
|
||||
if '..' in path or path.startswith('/etc'):
|
||||
self._send_json({"error": "forbidden"}, 403); return
|
||||
|
||||
# 根路径 → index.html
|
||||
if path == '' or path == '/':
|
||||
path = '/index.html'
|
||||
|
||||
filepath = os.path.join(DASHBOARD_DIR, path.lstrip('/'))
|
||||
# 双重安全
|
||||
if not filepath.startswith(DASHBOARD_DIR):
|
||||
self._send_json({"error": "forbidden"}, 403); return
|
||||
|
||||
if not os.path.isfile(filepath):
|
||||
self._send_json({"error": "not found", "path": path}, 404); return
|
||||
|
||||
with open(filepath, 'rb') as f:
|
||||
content = f.read()
|
||||
|
||||
ext = os.path.splitext(filepath)[1].lower()
|
||||
mime = {
|
||||
'.html': 'text/html; charset=utf-8',
|
||||
'.css': 'text/css; charset=utf-8',
|
||||
'.js': 'application/javascript; charset=utf-8',
|
||||
'.json': 'application/json; charset=utf-8',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.png': 'image/png',
|
||||
'.ico': 'image/x-icon',
|
||||
}.get(ext, 'application/octet-stream')
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', mime)
|
||||
self.send_header('Content-Length', str(len(content)))
|
||||
self.send_header('Access-Control-Allow-Origin', '*')
|
||||
self.send_header('Cache-Control', 'no-cache')
|
||||
self.end_headers()
|
||||
self.wfile.write(content)
|
||||
|
||||
def do_POST(self): self._send_json({"error": "read-only"}, 405)
|
||||
def do_PUT(self): self._send_json({"error": "read-only"}, 405)
|
||||
def do_DELETE(self): self._send_json({"error": "read-only"}, 405)
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
pass # 静默日志
|
||||
|
||||
# ---- Endpoint 实现 ----
|
||||
def _health(self):
|
||||
db_ok = os.path.exists(DB_PATH)
|
||||
db_size = os.path.getsize(DB_PATH) if db_ok else 0
|
||||
# 最新写入时间
|
||||
last = fetch_one("SELECT MAX(ts) AS last_ts FROM db_metrics")
|
||||
self._send_json({
|
||||
"service": "oa-monitor-api",
|
||||
"db_exists": db_ok,
|
||||
"db_size_bytes": db_size,
|
||||
"latest_data_ts": ts_to_iso(last['last_ts'] if last else None),
|
||||
"now": ts_to_iso(int(datetime.now().timestamp()))
|
||||
})
|
||||
|
||||
def _overview(self):
|
||||
# 各层最新一条
|
||||
db = fetch_one("SELECT * FROM db_metrics ORDER BY ts DESC LIMIT 1")
|
||||
http_all = fetch_all("SELECT endpoint, status_code, resp_time_ms, healthy, ts FROM http_metrics WHERE ts > ? ORDER BY ts DESC", ((datetime.now().timestamp() - 120),))
|
||||
biz = fetch_one("SELECT * FROM business_metrics ORDER BY ts DESC LIMIT 1")
|
||||
infra = fetch_one("SELECT * FROM infra_metrics ORDER BY ts DESC LIMIT 1")
|
||||
|
||||
# 活跃告警
|
||||
active_alerts = fetch_all("SELECT * FROM alerts WHERE ack = 0 ORDER BY ts DESC")
|
||||
|
||||
# HTTP 端点最新状态聚合
|
||||
http_summary = []
|
||||
for row in http_all:
|
||||
http_summary.append({
|
||||
"endpoint": row["endpoint"],
|
||||
"status_code": row["status_code"],
|
||||
"resp_time_ms": row["resp_time_ms"],
|
||||
"healthy": row["healthy"],
|
||||
"ts": ts_to_iso(row["ts"])
|
||||
})
|
||||
|
||||
# DB 层最近长查询
|
||||
long_q = fetch_all("SELECT * FROM db_processlist WHERE time_sec >= 30 ORDER BY time_sec DESC LIMIT 5")
|
||||
|
||||
self._send_json({
|
||||
"db_latest": {
|
||||
**({} if not db else db),
|
||||
"ts_iso": ts_to_iso(db["ts"]) if db else None
|
||||
},
|
||||
"http_summary": http_summary,
|
||||
"biz_latest": {
|
||||
**({} if not biz else biz),
|
||||
"ts_iso": ts_to_iso(biz["ts"]) if biz else None
|
||||
},
|
||||
"infra_latest": {
|
||||
**({} if not infra else infra),
|
||||
"ts_iso": ts_to_iso(infra["ts"]) if infra else None
|
||||
},
|
||||
"active_alerts": [{**a, "ts_iso": ts_to_iso(a["ts"])} for a in active_alerts],
|
||||
"long_queries": [{**q, "ts_iso": ts_to_iso(q["ts"])} for q in long_q]
|
||||
})
|
||||
|
||||
def _db_recent(self, qs):
|
||||
minutes = int(qs.get('minutes', [30])[0])
|
||||
since = int(datetime.now().timestamp()) - minutes * 60
|
||||
rows = fetch_all(
|
||||
"SELECT * FROM db_metrics WHERE ts >= ? ORDER BY ts ASC",
|
||||
(since,)
|
||||
)
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
def _db_long_queries(self, qs):
|
||||
since_min = int(qs.get('since_minutes', [60])[0])
|
||||
since = int(datetime.now().timestamp()) - since_min * 60
|
||||
rows = fetch_all(
|
||||
"SELECT * FROM db_processlist WHERE ts >= ? ORDER BY time_sec DESC LIMIT 100",
|
||||
(since,)
|
||||
)
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
def _http_recent(self, qs):
|
||||
minutes = int(qs.get('minutes', [30])[0])
|
||||
since = int(datetime.now().timestamp()) - minutes * 60
|
||||
rows = fetch_all(
|
||||
"SELECT * FROM http_metrics WHERE ts >= ? ORDER BY ts ASC",
|
||||
(since,)
|
||||
)
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
def _biz_recent(self, qs):
|
||||
hours = int(qs.get('hours', [24])[0])
|
||||
since = int(datetime.now().timestamp()) - hours * 3600
|
||||
rows = fetch_all(
|
||||
"SELECT * FROM business_metrics WHERE ts >= ? ORDER BY ts ASC",
|
||||
(since,)
|
||||
)
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
def _infra_recent(self, qs):
|
||||
minutes = int(qs.get('minutes', [30])[0])
|
||||
since = int(datetime.now().timestamp()) - minutes * 60
|
||||
rows = fetch_all(
|
||||
"SELECT * FROM infra_metrics WHERE ts >= ? ORDER BY ts ASC",
|
||||
(since,)
|
||||
)
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
def _alerts(self, qs):
|
||||
limit = int(qs.get('limit', [100])[0])
|
||||
only_unack = qs.get('unack', ['0'])[0]
|
||||
sql = "SELECT * FROM alerts"
|
||||
params = []
|
||||
if only_unack == '1':
|
||||
sql += " WHERE ack = 0"
|
||||
sql += " ORDER BY ts DESC LIMIT ?"
|
||||
params.append(limit)
|
||||
rows = fetch_all(sql, tuple(params))
|
||||
self._send_json([{**r, "ts_iso": ts_to_iso(r["ts"])} for r in rows])
|
||||
|
||||
# ---- 启动 ----
|
||||
if __name__ == '__main__':
|
||||
port = int(sys.argv[1]) if len(sys.argv) > 1 else 8455
|
||||
server = http.server.HTTPServer(('0.0.0.0', port), MonitorHandler)
|
||||
print(f"OA Monitor API listening on :{port} (DB={DB_PATH})", flush=True)
|
||||
server.serve_forever()
|
||||
@@ -0,0 +1 @@
|
||||
1231
|
||||
@@ -1 +1 @@
|
||||
13286 9890 1793773202
|
||||
30952 16490 1793792402
|
||||
Reference in new issue
Block a user