9장: 확장성과 멀티테넌시 설계

interface ConversationState {
  sessionId: string;
  messages: Array<{ role: string; content: string }>;
  systemPrompt: string;
  modelConfig: {
    model: string;
    temperature: number;
    maxTokens: number;
  };
  metadata: {
    tenantId: string;
    userId: string;
    createdAt: string;
    lastActivityAt: string;
  };
}
 
class InferenceWorker {
  constructor(
    private readonly stateStore: StateStore,
    private readonly llmClient: LLMClient,
    private readonly metricsCollector: MetricsCollector
  ) {}
 
  async processRequest(
    sessionId: string,
    userMessage: string
  ): Promise<string> {
    // 1. 외부 저장소에서 대화 상태 로드
    const state = await this.stateStore.load(sessionId);
    if (!state) {
      throw new Error(`세션을 찾을 수 없습니다: ${sessionId}`);
    }
 
    // 2. 메시지 추가
    state.messages.push({ role: "user", content: userMessage });
 
    // 3. LLM 호출
    const startTime = Date.now();
    const response = await this.llmClient.chat({
      model: state.modelConfig.model,
      messages: [
        { role: "system", content: state.systemPrompt },
        ...state.messages,
      ],
      temperature: state.modelConfig.temperature,
      maxTokens: state.modelConfig.maxTokens,
    });
 
    // 4. 응답 저장 및 상태 갱신
    state.messages.push({ role: "assistant", content: response.content });
    state.metadata.lastActivityAt = new Date().toISOString();
    await this.stateStore.save(sessionId, state);
 
    // 5. 메트릭 기록
    this.metricsCollector.record({
      sessionId,
      tenantId: state.metadata.tenantId,
      latencyMs: Date.now() - startTime,
      inputTokens: response.usage.inputTokens,
      outputTokens: response.usage.outputTokens,
    });
 
    return response.content;
  }
}
 
interface StateStore {
  load(sessionId: string): Promise<ConversationState | null>;
  save(sessionId: string, state: ConversationState): Promise<void>;
  delete(sessionId: string): Promise<void>;
}
 
interface LLMClient {
  chat(params: {
    model: string;
    messages: Array<{ role: string; content: string }>;
    temperature: number;
    maxTokens: number;
  }): Promise<{ content: string; usage: { inputTokens: number; outputTokens: number } }>;
}
 
interface MetricsCollector {
  record(data: {
    sessionId: string;
    tenantId: string;
    latencyMs: number;
    inputTokens: number;
    outputTokens: number;
  }): void;
}

interface TenantConfig {
  tenantId: string;
  tier: "starter" | "professional" | "enterprise";
 
  // 모델 설정
  allowedModels: string[];
  defaultModel: string;
  customSystemPrompt?: string;
 
  // 사용량 제한
  quotas: {
    requestsPerMinute: number;
    requestsPerDay: number;
    tokensPerMonth: number;
    maxInputTokensPerRequest: number;
    maxOutputTokensPerRequest: number;
  };
 
  // 기능 플래그
  features: {
    ragEnabled: boolean;
    toolUseEnabled: boolean;
    streamingEnabled: boolean;
    customModelsEnabled: boolean;
  };
 
  // 데이터 격리
  dataConfig: {
    vectorNamespace: string;
    cachePrefix: string;
    logRetentionDays: number;
  };
}
 
class TenantConfigService {
  private configCache: Map<string, { config: TenantConfig; expiresAt: number }> =
    new Map();
 
  constructor(private readonly configStore: ConfigStore) {}
 
  async getConfig(tenantId: string): Promise<TenantConfig> {
    const cached = this.configCache.get(tenantId);
    if (cached && Date.now() < cached.expiresAt) {
      return cached.config;
    }
 
    const config = await this.configStore.loadTenantConfig(tenantId);
    this.configCache.set(tenantId, {
      config,
      expiresAt: Date.now() + 300000, // 5분 캐시
    });
 
    return config;
  }
 
  async enforceQuota(
    tenantId: string,
    usage: { tokens: number }
  ): Promise<{ allowed: boolean; reason?: string }> {
    const config = await this.getConfig(tenantId);
    const monthlyUsage = await this.configStore.getMonthlyTokenUsage(tenantId);
 
    if (monthlyUsage + usage.tokens > config.quotas.tokensPerMonth) {
      return {
        allowed: false,
        reason: `월간 토큰 한도 초과: ${monthlyUsage}/${config.quotas.tokensPerMonth}`,
      };
    }
 
    return { allowed: true };
  }
}
 
interface ConfigStore {
  loadTenantConfig(tenantId: string): Promise<TenantConfig>;
  getMonthlyTokenUsage(tenantId: string): Promise<number>;
}

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: inference-worker-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: inference-worker
  minReplicas: 2
  maxReplicas: 20
  metrics:
    - type: Pods
      pods:
        metric:
          name: inference_queue_depth
        target:
          type: AverageValue
          averageValue: "5"
    - type: Pods
      pods:
        metric:
          name: inference_p95_latency_ms
        target:
          type: AverageValue
          averageValue: "3000"
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60
      policies:
        - type: Pods
          value: 4
          periodSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
        - type: Pods
          value: 1
          periodSeconds: 120