fix: check-alert v3 修复钉钉推送断档 P0 假阳性
根因:sqlite_exec 吞掉 stderr(2>/dev/null),查询失败返回空串, 0 默认为 0 导致误报 修复:业务层取数显式调用 sqlite3 + 检查退出码 + 正则校验, 查询失败跳过检测并 warn 日志
This commit is contained in:
1 parent
7ab75364b1
commit
20fa48fb34
17 files changed
+1747
-26
No files matched your search
@@ -0,0 +1,102 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>OA 运维监控 · 告警历史</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body { font-family: -apple-system, 'PingFang SC', sans-serif; background: #0d1117; color: #c9d1d9; padding: 16px; }
|
||||
h1 { font-size: 20px; color: #58a6ff; margin-bottom: 16px; }
|
||||
.filter-bar { margin-bottom: 16px; display: flex; gap: 12px; align-items: center; flex-wrap: wrap; }
|
||||
.filter-bar button { background: #21262d; color: #c9d1d9; border: 1px solid #30363d; padding: 6px 14px; border-radius: 6px; cursor: pointer; font-size: 13px; }
|
||||
.filter-bar button.active { background: #1f6feb; border-color: #1f6feb; }
|
||||
.filter-bar button:hover { border-color: #58a6ff; }
|
||||
.stats { display: flex; gap: 16px; margin-bottom: 16px; }
|
||||
.stat { background: #161b22; border: 1px solid #30363d; padding: 12px 16px; border-radius: 8px; }
|
||||
.stat .num { font-size: 24px; font-weight: 700; }
|
||||
.stat .label { color: #8b949e; font-size: 12px; margin-top: 2px; }
|
||||
.tag { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 11px; font-weight: 600; }
|
||||
.tag-p0 { background: rgba(248,81,73,0.25); color: #ff7b72; }
|
||||
.tag-p1 { background: rgba(210,153,34,0.25); color: #d29922; }
|
||||
.tag-p2 { background: rgba(56,139,253,0.25); color: #58a6ff; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 13px; background: #161b22; border-radius: 8px; overflow: hidden; }
|
||||
th { background: #21262d; color: #8b949e; padding: 10px 12px; text-align: left; font-weight: 600; position: sticky; top: 0; }
|
||||
td { padding: 9px 12px; border-top: 1px solid #21262d; }
|
||||
tr:hover td { background: #1f2428; }
|
||||
.btn-ack { background: #1f6feb; color: white; border: none; padding: 4px 10px; border-radius: 4px; cursor: pointer; font-size: 12px; }
|
||||
.btn-resolve { background: #238636; color: white; border: none; padding: 4px 10px; border-radius: 4px; cursor: pointer; font-size: 12px; }
|
||||
.ack-badge { color: #3fb950; font-weight: 600; font-size: 11px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>🚨 告警历史</h1>
|
||||
<div class="filter-bar">
|
||||
<button data-filter="all" class="active">全部</button>
|
||||
<button data-filter="P0">P0</button>
|
||||
<button data-filter="P1">P1</button>
|
||||
<button data-filter="P2">P2</button>
|
||||
<button data-filter="unack">未确认</button>
|
||||
</div>
|
||||
|
||||
<div class="stats">
|
||||
<div class="stat"><div class="num" id="statTotal">0</div><div class="label">告警总数</div></div>
|
||||
<div class="stat"><div class="num" id="statP0">0</div><div class="label">P0 告警</div></div>
|
||||
<div class="stat"><div class="num" id="statUnack">0</div><div class="label">未确认</div></div>
|
||||
</div>
|
||||
|
||||
<table id="alertsTable">
|
||||
<thead><tr><th>级别</th><th>时间</th><th>类别</th><th>标题</th><th>详情</th><th>状态</th></tr></thead>
|
||||
<tbody id="alertsBody"></tbody>
|
||||
</table>
|
||||
|
||||
<script>
|
||||
const API = window.location.origin + '/monitor';
|
||||
let currentFilter = 'all';
|
||||
let allAlerts = [];
|
||||
|
||||
document.querySelectorAll('.filter-bar button').forEach(b => {
|
||||
b.addEventListener('click', () => {
|
||||
document.querySelectorAll('.filter-bar button').forEach(x => x.classList.remove('active'));
|
||||
b.classList.add('active');
|
||||
currentFilter = b.dataset.filter;
|
||||
render();
|
||||
});
|
||||
});
|
||||
|
||||
async function refresh() {
|
||||
const res = await fetch(API + '/api/alerts?limit=200');
|
||||
allAlerts = await res.json();
|
||||
render();
|
||||
}
|
||||
|
||||
function render() {
|
||||
let filtered = allAlerts;
|
||||
if (currentFilter === 'unack') filtered = filtered.filter(a => !a.ack);
|
||||
else if (['P0','P1','P2'].includes(currentFilter)) filtered = filtered.filter(a => a.level === currentFilter);
|
||||
|
||||
// stats
|
||||
document.getElementById('statTotal').textContent = allAlerts.length;
|
||||
document.getElementById('statP0').textContent = allAlerts.filter(a => a.level === 'P0').length;
|
||||
document.getElementById('statUnack').textContent = allAlerts.filter(a => !a.ack).length;
|
||||
|
||||
const tbody = document.getElementById('alertsBody');
|
||||
tbody.innerHTML = filtered.map(a => {
|
||||
const cls = a.level === 'P0' ? 'tag-p0' : (a.level === 'P1' ? 'tag-p1' : 'tag-p2');
|
||||
const ack = a.ack ? '<span class="ack-badge">已确认</span>' : '<span style="color:#d29922;">未确认</span>';
|
||||
const reso = a.resolved ? '✅ 已恢复' : (a.ack ? '' : '');
|
||||
return `<tr>
|
||||
<td><span class="tag ${cls}">${a.level}</span></td>
|
||||
<td>${a.ts_iso}</td>
|
||||
<td>${a.category}</td>
|
||||
<td><b>${a.title}</b></td>
|
||||
<td style="max-width:300px;">${a.detail||''}</td>
|
||||
<td>${ack} ${reso}</td>
|
||||
</tr>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
refresh();
|
||||
setInterval(refresh, 30000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,283 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>OA 运维监控平台 · 总览</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/echarts@5.5.0/dist/echarts.min.js"></script>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'PingFang SC', 'Microsoft YaHei', sans-serif; background: #0d1117; color: #c9d1d9; padding: 16px; }
|
||||
h1 { font-size: 20px; color: #58a6ff; margin-bottom: 16px; }
|
||||
.status-bar { display: flex; gap: 12px; margin-bottom: 16px; flex-wrap: wrap; }
|
||||
.status-chip { background: #161b22; border-radius: 6px; padding: 10px 14px; border: 1px solid #30363d; display: flex; align-items: center; gap: 8px; font-size: 13px; }
|
||||
.status-chip.ok { border-color: #238636; background: rgba(35,134,54,0.1); }
|
||||
.status-chip.warn { border-color: #d29922; background: rgba(210,153,34,0.1); }
|
||||
.status-chip.err { border-color: #f85149; background: rgba(248,81,73,0.1); animation: pulse 2s infinite; }
|
||||
.status-chip .dot { width: 10px; height: 10px; border-radius: 50%; }
|
||||
.status-chip.ok .dot { background: #238636; }
|
||||
.status-chip.warn .dot { background: #d29922; }
|
||||
.status-chip.err .dot { background: #f85149; }
|
||||
@keyframes pulse { 0%,100% { opacity: 1; } 50% { opacity: 0.5; } }
|
||||
|
||||
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(480px, 1fr)); gap: 16px; }
|
||||
.card { background: #161b22; border-radius: 8px; border: 1px solid #30363d; padding: 14px; }
|
||||
.card h2 { font-size: 14px; color: #8b949e; margin-bottom: 10px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; }
|
||||
.chart { width: 100%; height: 220px; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 12px; }
|
||||
th { background: #21262d; color: #8b949e; padding: 6px 8px; text-align: left; font-weight: 600; }
|
||||
td { padding: 5px 8px; border-top: 1px solid #21262d; word-break: break-all; }
|
||||
.tag { display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 11px; font-weight: 600; }
|
||||
.tag-ok { background: rgba(35,134,54,0.2); color: #3fb950; }
|
||||
.tag-fail { background: rgba(248,81,73,0.2); color: #f85149; }
|
||||
.tag-p0 { background: rgba(248,81,73,0.25); color: #ff7b72; }
|
||||
.tag-p1 { background: rgba(210,153,34,0.25); color: #d29922; }
|
||||
.tag-p2 { background: rgba(56,139,253,0.25); color: #58a6ff; }
|
||||
.empty { text-align: center; color: #484f58; padding: 20px; font-size: 13px; }
|
||||
.endpoints-table td:first-child { max-width: 220px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.endpoints-table td.status-ok { color: #3fb950; font-weight: 600; }
|
||||
.endpoints-table td.status-fail { color: #f85149; font-weight: 600; }
|
||||
.ts { color: #6e7681; font-size: 11px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>🛡️ OA 运维监控平台 · 总览</h1>
|
||||
|
||||
<div class="status-bar" id="statusBar">
|
||||
<div class="status-chip" id="chip-db"><span class="dot"></span><span>DB 层</span><span class="ts" id="ts-db"></span></div>
|
||||
<div class="status-chip" id="chip-http"><span class="dot"></span><span>HTTP 端点</span><span class="ts" id="ts-http"></span></div>
|
||||
<div class="status-chip" id="chip-business"><span class="dot"></span><span>业务/钉钉推送</span><span class="ts" id="ts-biz"></span></div>
|
||||
<div class="status-chip" id="chip-infra"><span class="dot"></span><span>基础设施</span><span class="ts" id="ts-infra"></span></div>
|
||||
<div class="status-chip err" id="chip-active" style="display:none;"><span class="dot"></span><span id="activeCount"></span> 活跃告警</div>
|
||||
</div>
|
||||
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
<h2>DB · Threads Running / 30s</h2>
|
||||
<div id="chart-threads" class="chart"></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>DB · QPS Read / Write</h2>
|
||||
<div id="chart-qps" class="chart"></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>DB · 当前活跃长查询(>30s)</h2>
|
||||
<div id="longQueriesTable"></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>HTTP · 端点状态(最近 2 分钟)</h2>
|
||||
<div id="httpTable"></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>业务 · 钉钉推送小时增量</h2>
|
||||
<div id="chart-msg" class="chart"></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>基础设施 · 容器数 / 磁盘 / Docker</h2>
|
||||
<div id="chart-infra" class="chart"></div>
|
||||
</div>
|
||||
<div class="card" style="grid-column: 1/-1;">
|
||||
<h2>🔴 活跃告警</h2>
|
||||
<div id="alertsTable"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const API = window.location.origin + '/monitor';
|
||||
const REFRESH_MS = 15000;
|
||||
|
||||
// ---- Charts ----
|
||||
const chartThreads = echarts.init(document.getElementById('chart-threads'), 'dark');
|
||||
const chartQps = echarts.init(document.getElementById('chart-qps'), 'dark');
|
||||
const chartMsg = echarts.init(document.getElementById('chart-msg'), 'dark');
|
||||
const chartInfra = echarts.init(document.getElementById('chart-infra'), 'dark');
|
||||
|
||||
function formatTime(ts) {
|
||||
const d = new Date(ts * 1000);
|
||||
return d.toLocaleTimeString('zh-CN', { hour12: false });
|
||||
}
|
||||
|
||||
async function fetchJSON(url) {
|
||||
const res = await fetch(API + url, { cache: 'no-cache' });
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
try {
|
||||
const overview = await fetchJSON('/api/overview');
|
||||
const db30 = await fetchJSON('/api/db/recent?minutes=30');
|
||||
const biz24 = await fetchJSON('/api/business/recent?hours=24');
|
||||
const infra30 = await fetchJSON('/api/infra/recent?minutes=30');
|
||||
|
||||
renderStatus(overview);
|
||||
renderCharts(db30, biz24, infra30);
|
||||
renderLongQueries(overview.long_queries);
|
||||
renderHttpTable(overview.http_summary);
|
||||
renderAlerts(overview.active_alerts);
|
||||
} catch (e) {
|
||||
console.error('refresh fail:', e);
|
||||
}
|
||||
}
|
||||
|
||||
function renderStatus(o) {
|
||||
// DB
|
||||
const dbOk = o.db_latest && o.db_latest.threads_running >= 0;
|
||||
updateChip('chip-db', 'ts-db', dbOk ? 'ok' : 'err', o.db_latest?.ts_iso);
|
||||
|
||||
// HTTP
|
||||
const httpAll = o.http_summary || [];
|
||||
const httpOk = httpAll.length > 0 && httpAll.every(e => e.healthy === 1);
|
||||
updateChip('chip-http', 'ts-http', httpOk ? 'ok' : (httpAll.some(e => !e.healthy) ? 'warn' : 'err'), httpAll[0]?.ts);
|
||||
|
||||
// Business
|
||||
const bizOk = o.biz_latest && o.biz_latest.msg_log_hourly >= 0;
|
||||
updateChip('chip-business', 'ts-biz', bizOk ? 'ok' : 'warn', o.biz_latest?.ts_iso);
|
||||
|
||||
// Infra
|
||||
const infraOk = o.infra_latest && o.infra_latest.docker_daemon_ok === 1;
|
||||
const infraClass = infraOk ? 'ok' : 'err';
|
||||
updateChip('chip-infra', 'ts-infra', infraClass, o.infra_latest?.ts_iso);
|
||||
|
||||
// Active alerts
|
||||
const aa = o.active_alerts || [];
|
||||
const chipActive = document.getElementById('chip-active');
|
||||
if (aa.length > 0) {
|
||||
chipActive.style.display = 'flex';
|
||||
document.getElementById('activeCount').textContent = aa.length + ' 活跃告警';
|
||||
} else {
|
||||
chipActive.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
function updateChip(id, tsId, cls, ts) {
|
||||
const el = document.getElementById(id);
|
||||
el.className = 'status-chip ' + cls;
|
||||
if (ts) document.getElementById(tsId).textContent = ts;
|
||||
}
|
||||
|
||||
function renderCharts(db, biz, infra) {
|
||||
const dbX = db.map(r => formatTime(r.ts));
|
||||
const dbThreads = db.map(r => r.threads_running);
|
||||
const dbQpsR = db.map(r => r.qps_read);
|
||||
const dbQpsW = db.map(r => r.qps_write);
|
||||
|
||||
chartThreads.setOption({
|
||||
tooltip: { trigger: 'axis' },
|
||||
grid: { left: 40, right: 20, top: 20, bottom: 30 },
|
||||
xAxis: { type: 'category', data: dbX, axisLine: { lineStyle: { color: '#30363d' } } },
|
||||
yAxis: { type: 'value', splitLine: { lineStyle: { color: '#21262d' } } },
|
||||
series: [{ name: 'Threads_running', type: 'line', data: dbThreads, smooth: true,
|
||||
lineStyle: { color: '#58a6ff' }, areaStyle: { color: 'rgba(88,166,255,0.15)' },
|
||||
markLine: { data: [{ yAxis: 50, label: '阈值50', lineStyle: { color: '#f85149', type: 'dashed' } }] }
|
||||
}]
|
||||
});
|
||||
|
||||
chartQps.setOption({
|
||||
tooltip: { trigger: 'axis' },
|
||||
legend: { data: ['Read QPS', 'Write QPS'], textStyle: { color: '#c9d1d9' } },
|
||||
grid: { left: 40, right: 20, top: 30, bottom: 30 },
|
||||
xAxis: { type: 'category', data: dbX, axisLine: { lineStyle: { color: '#30363d' } } },
|
||||
yAxis: { type: 'value', splitLine: { lineStyle: { color: '#21262d' } } },
|
||||
series: [
|
||||
{ name: 'Read QPS', type: 'line', data: dbQpsR, smooth: true, lineStyle: { color: '#3fb950' } },
|
||||
{ name: 'Write QPS', type: 'line', data: dbQpsW, smooth: true, lineStyle: { color: '#d29922' } }
|
||||
]
|
||||
});
|
||||
|
||||
// Biz chart
|
||||
const bizX = biz.map(r => r.ts_iso?.substring(11,16) || r.ts);
|
||||
const bizMsg = biz.map(r => r.msg_log_hourly);
|
||||
chartMsg.setOption({
|
||||
tooltip: { trigger: 'axis' },
|
||||
grid: { left: 60, right: 20, top: 20, bottom: 30 },
|
||||
xAxis: { type: 'category', data: bizX, axisLine: { lineStyle: { color: '#30363d' } } },
|
||||
yAxis: { type: 'value', splitLine: { lineStyle: { color: '#21262d' } } },
|
||||
series: [{ name: '小时增量', type: 'bar', data: bizMsg, itemStyle: { color: '#a371f7' },
|
||||
markLine: { data: [{ yAxis: 100, label: '阈值100', lineStyle: { color: '#f85149', type: 'dashed' } }] }
|
||||
}]
|
||||
});
|
||||
|
||||
// Infra chart
|
||||
const infraX = infra.map(r => formatTime(r.ts));
|
||||
const infraC = infra.map(r => r.docker_container_count);
|
||||
const infraD = infra.map(r => r.disk_usage_pct);
|
||||
chartInfra.setOption({
|
||||
tooltip: { trigger: 'axis' },
|
||||
legend: { data: ['容器数', '磁盘%'], textStyle: { color: '#c9d1d9' } },
|
||||
grid: { left: 40, right: 50, top: 30, bottom: 30 },
|
||||
xAxis: { type: 'category', data: infraX, axisLine: { lineStyle: { color: '#30363d' } } },
|
||||
yAxis: [
|
||||
{ type: 'value', name: '容器数', splitLine: { lineStyle: { color: '#21262d' } } },
|
||||
{ type: 'value', name: '磁盘%', min: 0, max: 100, splitLine: { show: false } }
|
||||
],
|
||||
series: [
|
||||
{ name: '容器数', type: 'line', data: infraC, smooth: true, lineStyle: { color: '#3fb950' }, areaStyle: { color: 'rgba(63,185,80,0.1)' } },
|
||||
{ name: '磁盘%', type: 'line', yAxisIndex: 1, data: infraD, smooth: true, lineStyle: { color: '#d29922' },
|
||||
markLine: { data: [{ yAxis: 85, label: '85%', lineStyle: { color: '#f85149', type: 'dashed' } }] }
|
||||
}
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
function renderLongQueries(queries) {
|
||||
const el = document.getElementById('longQueriesTable');
|
||||
if (!queries || queries.length === 0) {
|
||||
el.innerHTML = '<div class="empty">✅ 当前无活跃长查询(>30s)</div>';
|
||||
return;
|
||||
}
|
||||
let html = '<table><tr><th>PID</th><th>Time(s)</th><th>User</th><th>State</th><th>SQL</th></tr>';
|
||||
queries.forEach(q => {
|
||||
html += `<tr><td>${q.pid}</td><td style="color:#f85149;font-weight:600;">${q.time_sec}</td><td>${q.user}</td><td>${q.state||'-'}</td><td style="max-width:260px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="${q.info||''}">${q.info||''}</td></tr>`;
|
||||
});
|
||||
html += '</table>';
|
||||
el.innerHTML = html;
|
||||
}
|
||||
|
||||
function renderHttpTable(list) {
|
||||
const el = document.getElementById('httpTable');
|
||||
if (!list || list.length === 0) {
|
||||
el.innerHTML = '<div class="empty">等待数据...</div>';
|
||||
return;
|
||||
}
|
||||
let html = '<table class="endpoints-table"><tr><th>端点</th><th>状态</th><th>耗时(ms)</th><th>时间</th></tr>';
|
||||
const seen = new Set();
|
||||
list.sort((a, b) => (b.healthy - a.healthy) || (a.endpoint.localeCompare(b.endpoint)));
|
||||
list.forEach(e => {
|
||||
if (seen.has(e.endpoint)) return;
|
||||
seen.add(e.endpoint);
|
||||
const ok = e.healthy === 1;
|
||||
html += `<tr><td title="${e.endpoint}">${e.endpoint}</td>
|
||||
<td class="${ok?'status-ok':'status-fail'}">${ok?'OK ('+e.status_code+')':'FAIL ('+e.status_code+')'}</td>
|
||||
<td>${e.resp_time_ms}</td><td class="ts">${e.ts}</td></tr>`;
|
||||
});
|
||||
html += '</table>';
|
||||
el.innerHTML = html;
|
||||
}
|
||||
|
||||
function renderAlerts(alerts) {
|
||||
const el = document.getElementById('alertsTable');
|
||||
if (!alerts || alerts.length === 0) {
|
||||
el.innerHTML = '<div class="empty">🎉 当前无活跃告警</div>';
|
||||
return;
|
||||
}
|
||||
let html = '<table><tr><th>级别</th><th>时间</th><th>类别</th><th>标题</th><th>详情</th></tr>';
|
||||
alerts.forEach(a => {
|
||||
const cls = a.level === 'P0' ? 'tag-p0' : (a.level === 'P1' ? 'tag-p1' : 'tag-p2');
|
||||
html += `<tr><td><span class="tag ${cls}">${a.level}</span></td>
|
||||
<td class="ts">${a.ts_iso}</td>
|
||||
<td>${a.category}</td>
|
||||
<td><b>${a.title}</b></td>
|
||||
<td>${a.detail||''}</td></tr>`;
|
||||
});
|
||||
html += '</table>';
|
||||
el.innerHTML = html;
|
||||
}
|
||||
|
||||
refresh();
|
||||
setInterval(refresh, REFRESH_MS);
|
||||
window.addEventListener('resize', () => {
|
||||
chartThreads.resize(); chartQps.resize(); chartMsg.resize(); chartInfra.resize();
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in new issue
Block a user