前端日志系统的架构设计之采集、传输、存储与查询的性能平衡详解
作者：大山哥AGI
前端日志框架,是一套用于规范日志收集、分级、格式化、输出、存储(或远程上报)的工具集合,本质是对原生console的增强与工程化封装,这篇文章主要介绍了前端日志系统的架构设计之采集、传输、存储与查询的性能平衡的相关资料,需要的朋友可以参考下
一、前端日志的定位与边界
前端日志不是后端日志的延伸。后端日志关注请求链路与服务状态，前端日志关注用户行为、渲染性能与异常捕获。两者的采集量、传输频率与存储策略完全不同。
一个出行平台的前端日志系统，日均采集量在 800 万条左右。如果全量上报，带宽成本与查询延迟会超出可接受范围。核心矛盾：采集粒度越细，信息越完整，但传输和存储成本越高。架构设计的本质是在这个矛盾中找到平衡点。
二、采集层设计
2.1 日志分类与优先级
日志按优先级分为三级，不同级别的采集策略不同。
// log-classification.ts — 日志分类定义
enum LogLevel {
ERROR = "error", // 异常、崩溃 — 必须全量采集
PERFORMANCE = "perf", // 渲染耗时、接口延迟 — 采样采集
BEHAVIOR = "behavior", // 用户点击、滚动、导航 — 极低比例采样
}
interface LogEntry {
level: LogLevel;
timestamp: number;
sessionId: string;
userId?: string;
page: string;
payload: Record<string, unknown>;
}
// 各级别的采集策略
interface CollectionPolicy {
level: LogLevel;
sampleRate: number; // 采集比例 0-1
maxPerMinute: number; // 单用户每分钟上限
batchInterval: number; // 批量上报间隔(ms)
}
const policies: CollectionPolicy[] = [
{ level: LogLevel.ERROR, sampleRate: 1.0, maxPerMinute: 30, batchInterval: 1000 },
{ level: LogLevel.PERFORMANCE, sampleRate: 0.1, maxPerMinute: 10, batchInterval: 5000 },
{ level: LogLevel.BEHAVIOR, sampleRate: 0.01, maxPerMinute: 3, batchInterval: 10000 },
];
2.2 采集 SDK 核心
采集 SDK 需要轻量、无侵入、不影响页面渲染性能。核心设计：监听而非拦截，异步而非同步。
// logger-sdk.ts — 前端日志采集 SDK
interface LoggerConfig {
endpoint: string; // 上报地址
appId: string; // 应用标识
sampleRates: Record<LogLevel, number>;
maxBatchSize: number; // 单批次最大条数
flushInterval: number; // 定时上报间隔(ms)
maxQueueSize: number; // 本地队列最大容量
}
class FrontendLogger {
private queue: LogEntry[] = [];
private flushTimer: number | null = null;
private sessionId: string;
private config: LoggerConfig;
private droppedCount: number = 0; // 因队列满而丢弃的数量
constructor(config: LoggerConfig) {
this.config = config;
this.sessionId = this.generateSessionId();
// 注册全局错误监听
this.registerErrorListeners();
// 注册性能指标监听
this.registerPerformanceListeners();
// 注册行为监听
this.registerBehaviorListeners();
// 启动定时上报
this.startFlushTimer();
// 页面卸载时强制上报
window.addEventListener("visibilitychange", () => {
if (document.visibilityState === "hidden") {
this.flush(); // 立即上报剩余日志
}
});
}
// 手动记录日志（对外接口）
log(level: LogLevel, payload: Record<string, unknown>): void {
if (!this.shouldSample(level)) return;
const entry: LogEntry = {
level,
timestamp: Date.now(),
sessionId: this.sessionId,
userId: this.tryGetUserId(),
page: window.location.pathname,
payload,
};
this.enqueue(entry);
}
// 采样判断
private shouldSample(level: LogLevel): boolean {
const rate = this.config.sampleRates[level] ?? 1;
return Math.random() < rate;
}
// 入队：超出容量时丢弃低优先级日志
private enqueue(entry: LogEntry): void {
if (this.queue.length >= this.config.maxQueueSize) {
// 丢弃行为日志，保留错误日志
const behaviorIdx = this.queue.findIndex((e) => e.level === LogLevel.BEHAVIOR);
if (behaviorIdx !== -1) {
this.queue.splice(behaviorIdx, 1);
} else {
this.droppedCount++;
return; // 队列满且无可丢弃项，直接跳过
}
}
this.queue.push(entry);
// 达到批次上限时立即上报
if (this.queue.length >= this.config.maxBatchSize) {
this.flush();
}
}
// 批量上报
private async flush(): Promise<void> {
if (this.queue.length === 0) return;
// 取出待上报日志，清空队列
const batch = this.queue.splice(0, this.config.maxBatchSize);
try {
const response = await fetch(this.config.endpoint, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
appId: this.config.appId,
entries: batch,
dropped: this.droppedCount,
}),
keepalive: true, // 页面卸载时也能完成请求
});
if (!response.ok) {
// 上报失败，重新入队（最多重试一次）
this.queue.unshift(...batch);
console.warn(`[Logger] 上报失败 HTTP ${response.status}，${batch.length} 条日志重入队列`);
}
this.droppedCount = 0;
} catch (err) {
// 网络异常时重新入队
this.queue.unshift(...batch);
console.warn(`[Logger] 上报异常: ${(err as Error).message}`);
}
}
// 定时上报
private startFlushTimer(): void {
this.flushTimer = window.setInterval(() => {
this.flush();
}, this.config.flushInterval);
}
// 全局错误监听
private registerErrorListeners(): void {
window.addEventListener("error", (event) => {
this.log(LogLevel.ERROR, {
type: "runtime_error",
message: event.message,
filename: event.filename,
lineno: event.lineno,
stack: event.error?.stack?.substring(0, 500) ?? "", // 截断避免超长
});
});
window.addEventListener("unhandledrejection", (event) => {
this.log(LogLevel.ERROR, {
type: "unhandled_rejection",
reason: String(event.reason).substring(0, 300),
});
});
}
// 性能指标监听
private registerPerformanceListeners(): void {
// 页面加载性能
window.addEventListener("load", () => {
setTimeout(() => {
const perf = performance.getEntriesByType("navigation")[0] as PerformanceNavigationTiming;
if (!perf) return;
this.log(LogLevel.PERFORMANCE, {
type: "page_load",
dns: perf.domainLookupEnd - perf.domainLookupStart,
tcp: perf.connectEnd - perf.connectStart,
ttfb: perf.responseStart - perf.requestStart,
domReady: perf.domContentLoadedEventEnd - perf.fetchStart,
fullLoad: perf.loadEventEnd - perf.fetchStart,
});
}, 100); // 延迟获取确保指标完整
});
// 长任务监听
if ("PerformanceObserver" in window) {
try {
const observer = new PerformanceObserver((list) => {
for (const entry of list.getEntries()) {
if (entry.duration > 50) {
this.log(LogLevel.PERFORMANCE, {
type: "long_task",
duration: entry.duration,
name: entry.name,
startTime: entry.startTime,
});
}
}
});
observer.observe({ entryTypes: ["longtask"] });
} catch {
// 浏览器不支持 longtask，静默忽略
}
}
}
// 行为监听：关键点击事件
private registerBehaviorListeners(): void {
// 仅监听关键操作区域的点击
const actionElements = document.querySelectorAll("[data-track]");
actionElements.forEach((el) => {
el.addEventListener("click", () => {
this.log(LogLevel.BEHAVIOR, {
type: "click",
target: el.getAttribute("data-track") ?? el.tagName,
text: (el.textContent ?? "").substring(0, 50),
});
});
});
}
private generateSessionId(): string {
return `${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;
}
private tryGetUserId(): string | undefined {
// 从本地存储获取用户标识，未登录时为 undefined
try {
return localStorage.getItem("user_id") ?? undefined;
} catch {
return undefined;
}
}
destroy(): void {
if (this.flushTimer) clearInterval(this.flushTimer);
this.flush();
}
}
三、传输层设计
传输层的关键指标：上报成功率、带宽占用、对页面性能的影响。
3.1 批量与压缩策略
// transport.ts — 传输层核心逻辑
interface TransportConfig {
endpoint: string;
maxBatchSize: number;
compressionThreshold: number; // 触发压缩的最小条数
retryMaxAttempts: number;
retryBaseDelay: number; // 重试基础延迟(ms)
}
class LogTransport {
private config: TransportConfig;
private retryAttempts: number = 0;
constructor(config: TransportConfig) {
this.config = config;
}
async sendBatch(entries: LogEntry[]): Promise<boolean> {
const payload = JSON.stringify(entries);
// 大批次时压缩，小批次直接发送
let body: string | Blob;
let headers: Record<string, string>;
if (entries.length >= this.config.compressionThreshold) {
try {
const compressed = await this.compress(payload);
body = compressed;
headers = {
"Content-Type": "application/json",
"Content-Encoding": "gzip",
};
} catch {
// 压缩失败，降级为未压缩发送
body = payload;
headers = { "Content-Type": "application/json" };
}
} else {
body = payload;
headers = { "Content-Type": "application/json" };
}
try {
const response = await fetch(this.config.endpoint, {
method: "POST",
headers,
body,
keepalive: true,
});
if (response.status === 429) {
// 服务端限流，指数退避
this.retryAttempts++;
const delay = this.config.retryBaseDelay * Math.pow(2, this.retryAttempts);
console.warn(`[Transport] 限流，${delay}ms 后重试`);
await this.sleep(delay);
return false;
}
if (response.ok) {
this.retryAttempts = 0;
return true;
}
return false;
} catch (err) {
console.warn(`[Transport] 发送失败: ${(err as Error).message}`);
return false;
}
}
private async compress(data: string): Promise<Blob> {
// 使用 CompressionStream API（现代浏览器支持）
if ("CompressionStream" in window) {
const stream = new Blob([data]).stream();
const compressedStream = stream.pipeThrough(new CompressionStream("gzip"));
return new Response(compressedStream).blob();
}
// 不支持压缩时返回原始数据
return new Blob([data], { type: "application/json" });
}
private sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, Math.min(ms, 30000)));
}
}
3.2 离线缓存与降级上报
用户网络不稳定时，日志需要暂存本地，网络恢复后批量上报。
// offline-buffer.ts — 离线缓存策略
class OfflineLogBuffer {
private storageKey: string;
private maxStorageEntries: number = 500;
constructor(appId: string) {
this.storageKey = `log_buffer_${appId}`;
}
// 暂存到 IndexedDB（容量比 localStorage 大）
async store(entries: LogEntry[]): Promise<void> {
try {
const db = await this.openDB();
const tx = db.transaction("logs", "readwrite");
const store = tx.objectStore("logs");
for (const entry of entries) {
// 超出容量时删除最旧的日志
const count = await store.count();
if (count >= this.maxStorageEntries) {
const oldest = await store.openCursor();
if (oldest) await store.delete(oldest.value.id);
}
await store.add({ ...entry, id: `${entry.timestamp}-${Math.random().toString(36).slice(2)}` });
}
await tx.done;
} catch (err) {
// IndexedDB 不可用时降级到 localStorage
this.storeToLocalStorage(entries);
}
}
// 恢复暂存日志
async retrieve(maxCount: number): Promise<LogEntry[]> {
try {
const db = await this.openDB();
const tx = db.transaction("logs", "readonly");
const store = tx.objectStore("logs");
const all = await store.getAll();
// 按时间排序，取最新 maxCount 条
const sorted = all.sort((a: LogEntry, b: LogEntry) => b.timestamp - a.timestamp);
return sorted.slice(0, maxCount);
} catch {
return this.retrieveFromLocalStorage(maxCount);
}
}
// 清除已上报的日志
async clear(): Promise<void> {
try {
const db = await this.openDB();
const tx = db.transaction("logs", "readwrite");
tx.objectStore("logs").clear();
await tx.done;
} catch {
localStorage.removeItem(this.storageKey);
}
}
// 降级存储：localStorage
private storeToLocalStorage(entries: LogEntry[]): void {
try {
const existing = JSON.parse(localStorage.getItem(this.storageKey) ?? "[]");
const merged = [...existing, ...entries].slice(-this.maxStorageEntries);
localStorage.setItem(this.storageKey, JSON.stringify(merged));
} catch {
// localStorage 不可用时彻底降级，丢弃日志
console.warn("[OfflineBuffer] 降级存储失败，日志丢弃");
}
}
private retrieveFromLocalStorage(maxCount: number): LogEntry[] {
try {
const stored = JSON.parse(localStorage.getItem(this.storageKey) ?? "[]");
return stored.slice(0, maxCount);
} catch {
return [];
}
}
private async openDB(): Promise<IDBDatabase> {
return new Promise((resolve, reject) => {
const request = indexedDB.open("frontend_logs", 1);
request.onupgradeneeded = () => {
const db = request.result;
if (!db.objectStoreNames.contains("logs")) {
db.createObjectStore("logs", { keyPath: "id" });
}
};
request.onsuccess = () => resolve(request.result);
request.onerror = () => reject(new Error("IndexedDB 打开失败"));
});
}
}
四、存储与查询层设计
4.1 时序数据库选型
前端日志的时间属性强，查询模式固定（按时间范围 + 页面 + 日志级别筛选）。时序数据库比关系型数据库更适合。
存储策略：热数据（7天内）保留在 InfluxDB，冷数据归档到对象存储。查询优先走热数据，历史分析走归档。
// storage-policy.ts — 存储策略配置
interface StorageTier {
name: string;
retention: string; // 数据保留时长
backend: string; // 存储引擎
queryLatency: string; // 预估查询延迟
costPerGB: string; // 每GB存储成本
}
const tiers: StorageTier[] = [
{ name: "热数据", retention: "7d", backend: "InfluxDB", queryLatency: "<100ms", costPerGB: "¥2.5/月" },
{ name: "温数据", retention: "30d", backend: "ClickHouse", queryLatency: "<500ms", costPerGB: "¥1.0/月" },
{ name: "冷数据", retention: "365d", backend: "对象存储+Parquet", queryLatency: "1-5s", costPerGB: "¥0.12/月" },
];
// 数据路由：根据查询时间范围选择存储层
function selectStorageTier(queryStart: Date): StorageTier {
const ageInDays = (Date.now() - queryStart.getTime()) / 86400000;
if (ageInDays <= 7) return tiers[0];
if (ageInDays <= 30) return tiers[1];
return tiers[2];
}
4.2 查询性能优化
// query-service.ts — 查询服务
interface LogQuery {
start: Date;
end: Date;
level?: LogLevel;
page?: string;
userId?: string;
keyword?: string;
limit: number;
}
interface QueryResult {
total: number;
entries: LogEntry[];
latencyMs: number;
source: string; // 来自哪个存储层
}
// 查询缓存：LRU 策略
const queryCache = new Map<string, { result: QueryResult; expiresAt: number }>();
const CACHE_TTL = 5 * 60 * 1000; // 5分钟缓存
async function queryLogs(params: LogQuery): Promise<QueryResult> {
const cacheKey = JSON.stringify(params);
const cached = queryCache.get(cacheKey);
if (cached && cached.expiresAt > Date.now()) {
return { ...cached.result, latencyMs: 0, source: "cache" };
}
// 选择存储层
const tier = selectStorageTier(params.start);
const startTime = Date.now();
let entries: LogEntry[];
try {
entries = await fetchFromStorage(tier, params);
} catch (err) {
// 查询失败时返回空结果，不抛异常
console.error(`[QueryService] ${tier.backend} 查询失败: ${(err as Error).message}`);
entries = [];
}
const result: QueryResult = {
total: entries.length,
entries: entries.slice(0, params.limit),
latencyMs: Date.now() - startTime,
source: tier.backend,
};
// 缓存结果
queryCache.set(cacheKey, { result, expiresAt: Date.now() + CACHE_TTL });
// 缓存容量控制
if (queryCache.size > 200) {
const oldest = queryCache.keys().next().value;
if (oldest) queryCache.delete(oldest);
}
return result;
}
async function fetchFromStorage(tier: StorageTier, params: LogQuery): Promise<LogEntry[]> {
// 根据存储层调用不同接口
const url = tier.backend === "InfluxDB"
? "/api/logs/influx"
: tier.backend === "ClickHouse"
? "/api/logs/clickhouse"
: "/api/logs/archive";
const response = await fetch(url, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(params),
});
if (!response.ok) {
throw new Error(`查询接口返回 HTTP ${response.status}`);
}
return response.json();
}
五、总结
前端日志系统的架构设计，核心是在采集粒度与成本之间做取舍。出行平台的实践数据表明，分级采样策略将日均传输量从 800 万条压降至 120 万条，带宽成本降低 85%，而错误覆盖率保持在 100%。
关键设计原则：
- 分级采样：错误全量、性能 10%、行为 1%，信息价值决定采集比例。
- 批量压缩：≥30 条触发 Gzip，小批次直接发送，避免压缩本身成为开销。
- 离线缓存：IndexedDB 优先、localStorage 降级，网络恢复后自动上报。
- 分层存储：热数据 InfluxDB、温数据 ClickHouse、冷数据对象存储，查询延迟与成本分层优化。
- 队列降级：队列满时丢弃行为日志保留错误日志，保证核心信息不丢。
日志系统的有效性不只取决于技术架构，更取决于采集策略是否匹配业务需求。每一条日志都有存储成本，只有对业务决策有实际支撑作用的日志才值得全量采集。其余的，用采样就够了。
到此这篇关于前端日志系统的架构设计之采集、传输、存储与查询的性能平衡的文章就介绍到这了,更多相关前端日志系统架构设计内容请搜索脚本之家以前的文章或继续浏览下面的相关文章希望大家以后多多支持脚本之家！