7장: Collector 아키텍처와 배포

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: "0.0.0.0:4317"
      http:
        endpoint: "0.0.0.0:4318"
  
  prometheus:
    config:
      scrape_configs:
        - job_name: "node-exporters"
          scrape_interval: 15s
          static_configs:
            - targets: ["node-exporter:9100"]
  
  hostmetrics:
    collection_interval: 30s
    scrapers:
      cpu: {}
      memory: {}
      disk: {}
      network: {}

processors:
  batch:
    timeout: 1s              # 최대 대기 시간
    send_batch_size: 1024    # 배치당 최대 항목 수
    send_batch_max_size: 2048  # 배치 최대 크기 제한

processors:
  memory_limiter:
    check_interval: 1s
    limit_mib: 512          # 메모리 상한 (512MB)
    spike_limit_mib: 128    # 급격한 증가 허용량

processors:
  filter/traces:
    traces:
      span:
        # 헬스 체크 스팬 제거
        - 'attributes["http.target"] == "/healthz"'
        - 'attributes["http.target"] == "/readyz"'
        # 정적 리소스 스팬 제거
        - 'IsMatch(attributes["http.target"], "/static/.*")'
  
  filter/metrics:
    metrics:
      metric:
        # 특정 메트릭 제거
        - 'name == "http.server.active_requests" and resource.attributes["service.name"] == "debug-service"'
  
  filter/logs:
    logs:
      log_record:
        # DEBUG 로그 제거
        - 'severity_number < 9'

processors:
  attributes/insert:
    actions:
      # 환경 정보 추가
      - key: deployment.cluster
        value: "production-kr"
        action: insert
      
      # 민감 정보 해싱
      - key: user.email
        action: hash
      
      # 불필요한 속성 삭제
      - key: http.user_agent
        action: delete
      
      # 속성값 변환
      - key: http.url
        pattern: "^(?P<host>[^/]+)/.*$"
        replacement: "\\1"
        action: extract

processors:
  tail_sampling:
    decision_wait: 30s
    num_traces: 50000
    expected_new_traces_per_sec: 1000
    policies:
      # 정책 1: 에러 트레이스 100% 보존
      - name: error-traces
        type: status_code
        status_code:
          status_codes: [ERROR]
      
      # 정책 2: 느린 트레이스 보존 (1초 이상)
      - name: slow-traces
        type: latency
        latency:
          threshold_ms: 1000
      
      # 정책 3: 특정 서비스의 트레이스 100% 보존
      - name: critical-service
        type: string_attribute
        string_attribute:
          key: service.name
          values: [payment-service, auth-service]
      
      # 정책 4: 나머지는 10% 확률 샘플링
      - name: default
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: "0.0.0.0:4317"
      http:
        endpoint: "0.0.0.0:4318"
 
processors:
  memory_limiter:
    check_interval: 1s
    limit_mib: 1024
    spike_limit_mib: 256
 
  filter/healthcheck:
    traces:
      span:
        - 'attributes["http.target"] == "/healthz"'
    logs:
      log_record:
        - 'IsMatch(body, ".*healthcheck.*")'
 
  attributes/common:
    actions:
      - key: deployment.cluster
        value: "production-kr"
        action: insert
 
  batch:
    timeout: 1s
    send_batch_size: 1024
 
exporters:
  otlp/tempo:
    endpoint: "tempo:4317"
    tls:
      insecure: true
 
  prometheusremotewrite:
    endpoint: "http://prometheus:9090/api/v1/write"
 
  otlp/loki:
    endpoint: "loki:3100"
    tls:
      insecure: true
 
  debug:
    verbosity: basic
 
service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, filter/healthcheck, attributes/common, batch]
      exporters: [otlp/tempo]
 
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, attributes/common, batch]
      exporters: [prometheusremotewrite]
 
    logs:
      receivers: [otlp]
      processors: [memory_limiter, filter/healthcheck, attributes/common, batch]
      exporters: [otlp/loki]
 
  telemetry:
    logs:
      level: info
    metrics:
      address: ":8888"

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-collector-agent
  namespace: observability
spec:
  selector:
    matchLabels:
      app: otel-collector-agent
  template:
    metadata:
      labels:
        app: otel-collector-agent
    spec:
      containers:
        - name: collector
          image: otel/opentelemetry-collector-contrib:0.100.0
          args: ["--config=/etc/otelcol/config.yaml"]
          ports:
            - containerPort: 4317  # OTLP gRPC
            - containerPort: 4318  # OTLP HTTP
          resources:
            requests:
              cpu: 100m
              memory: 256Mi
            limits:
              cpu: 500m
              memory: 512Mi
          volumeMounts:
            - name: config
              mountPath: /etc/otelcol
      volumes:
        - name: config
          configMap:
            name: otel-agent-config

apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-collector-gateway
  namespace: observability
spec:
  replicas: 3
  selector:
    matchLabels:
      app: otel-collector-gateway
  template:
    metadata:
      labels:
        app: otel-collector-gateway
    spec:
      containers:
        - name: collector
          image: otel/opentelemetry-collector-contrib:0.100.0
          args: ["--config=/etc/otelcol/config.yaml"]
          ports:
            - containerPort: 4317
            - containerPort: 4318
          resources:
            requests:
              cpu: 500m
              memory: 1Gi
            limits:
              cpu: 2
              memory: 4Gi
          livenessProbe:
            httpGet:
              path: /
              port: 13133
          readinessProbe:
            httpGet:
              path: /
              port: 13133
      volumes:
        - name: config
          configMap:
            name: otel-gateway-config
---
apiVersion: v1
kind: Service
metadata:
  name: otel-collector-gateway
  namespace: observability
spec:
  selector:
    app: otel-collector-gateway
  ports:
    - name: otlp-grpc
      port: 4317
      targetPort: 4317
    - name: otlp-http
      port: 4318
      targetPort: 4318

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
 
# Agent 모드 (DaemonSet)
helm install otel-agent open-telemetry/opentelemetry-collector \
    --namespace observability \
    --set mode=daemonset \
    --set config.receivers.otlp.protocols.grpc.endpoint="0.0.0.0:4317" \
    --values agent-values.yaml
 
# Gateway 모드 (Deployment)
helm install otel-gateway open-telemetry/opentelemetry-collector \
    --namespace observability \
    --set mode=deployment \
    --set replicaCount=3 \
    --values gateway-values.yaml

service:
  telemetry:
    logs:
      level: info
    metrics:
      address: ":8888"  # Prometheus 형식으로 자체 메트릭 노출