8장: AI 시스템의 관측 가능성

interface LLMSpan {
  traceId: string;
  spanId: string;
  parentSpanId?: string;
  operation: string;
  startTime: number;
  endTime?: number;
  attributes: {
    model?: string;
    inputTokens?: number;
    outputTokens?: number;
    promptVersion?: string;
    temperature?: number;
    toolCalls?: string[];
    cacheHit?: boolean;
    costUsd?: number;
  };
  status: "ok" | "error";
  errorMessage?: string;
}
 
class LLMTracer {
  private spans: Map<string, LLMSpan> = new Map();
 
  startSpan(
    traceId: string,
    operation: string,
    parentSpanId?: string
  ): LLMSpan {
    const span: LLMSpan = {
      traceId,
      spanId: crypto.randomUUID(),
      parentSpanId,
      operation,
      startTime: Date.now(),
      attributes: {},
      status: "ok",
    };
    this.spans.set(span.spanId, span);
    return span;
  }
 
  endSpan(spanId: string, attributes: Partial<LLMSpan["attributes"]>): void {
    const span = this.spans.get(spanId);
    if (!span) return;
 
    span.endTime = Date.now();
    span.attributes = { ...span.attributes, ...attributes };
 
    this.export(span);
  }
 
  markError(spanId: string, error: string): void {
    const span = this.spans.get(spanId);
    if (!span) return;
 
    span.status = "error";
    span.errorMessage = error;
  }
 
  private export(span: LLMSpan): void {
    // OpenTelemetry Collector 또는 Langfuse로 전송
    const duration = span.endTime
      ? span.endTime - span.startTime
      : 0;
 
    console.info(JSON.stringify({
      trace_id: span.traceId,
      span_id: span.spanId,
      parent_span_id: span.parentSpanId,
      operation: span.operation,
      duration_ms: duration,
      ...span.attributes,
      status: span.status,
    }));
  }
}

interface AIMetrics {
  // 성능
  ttfbMs: number;
  tokensPerSecond: number;
  endToEndLatencyMs: number;
 
  // 토큰
  inputTokens: number;
  outputTokens: number;
  totalTokens: number;
 
  // 품질
  guardrailTriggered: boolean;
  hallucinationDetected: boolean;
  userFeedbackScore?: number;
 
  // 비용
  estimatedCostUsd: number;
  modelUsed: string;
  cacheHit: boolean;
}
 
class AIMetricsCollector {
  private metrics: AIMetrics[] = [];
 
  record(metric: AIMetrics): void {
    this.metrics.push(metric);
    this.emitToPrometheus(metric);
  }
 
  getAggregated(windowMinutes: number): {
    avgLatency: number;
    p95Latency: number;
    guardrailRate: number;
    hallucinationRate: number;
    avgCost: number;
    cacheHitRate: number;
  } {
    const recent = this.metrics.filter(
      (_, i) => i > this.metrics.length - 1000
    );
 
    const latencies = recent.map((m) => m.endToEndLatencyMs).sort((a, b) => a - b);
    const guardrailCount = recent.filter((m) => m.guardrailTriggered).length;
    const hallucinationCount = recent.filter((m) => m.hallucinationDetected).length;
    const cacheHitCount = recent.filter((m) => m.cacheHit).length;
 
    return {
      avgLatency: latencies.reduce((a, b) => a + b, 0) / latencies.length,
      p95Latency: latencies[Math.floor(latencies.length * 0.95)] ?? 0,
      guardrailRate: guardrailCount / recent.length,
      hallucinationRate: hallucinationCount / recent.length,
      avgCost:
        recent.reduce((sum, m) => sum + m.estimatedCostUsd, 0) / recent.length,
      cacheHitRate: cacheHitCount / recent.length,
    };
  }
 
  private emitToPrometheus(metric: AIMetrics): void {
    // Prometheus 메트릭 내보내기
  }
}

interface PromptVersion {
  id: string;
  name: string;
  version: string;
  template: string;
  variables: string[];
  metadata: {
    author: string;
    createdAt: string;
    description: string;
    parentVersion?: string;
  };
  abTest?: {
    enabled: boolean;
    trafficPercent: number;
    controlVersion: string;
  };
}
 
class PromptRegistry {
  private versions: Map<string, PromptVersion[]> = new Map();
 
  register(prompt: PromptVersion): void {
    const existing = this.versions.get(prompt.name) ?? [];
    existing.push(prompt);
    this.versions.set(prompt.name, existing);
  }
 
  resolve(name: string, userId?: string): PromptVersion {
    const versions = this.versions.get(name);
    if (!versions || versions.length === 0) {
      throw new Error(`프롬프트를 찾을 수 없습니다: ${name}`);
    }
 
    // A/B 테스트가 활성화된 버전 확인
    const abVersion = versions.find((v) => v.abTest?.enabled);
    if (abVersion && userId) {
      const hash = this.hashUserId(userId);
      if (hash % 100 < (abVersion.abTest?.trafficPercent ?? 0)) {
        return abVersion;
      }
    }
 
    // 최신 안정 버전 반환
    return versions[versions.length - 1]!;
  }
 
  private hashUserId(userId: string): number {
    let hash = 0;
    for (let i = 0; i < userId.length; i++) {
      hash = (hash << 5) - hash + userId.charCodeAt(i);
      hash |= 0;
    }
    return Math.abs(hash);
  }
}