8장: 서비스 메시 관측 가능성 - 분산 추적, 메트릭, 로그 통합

apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
spec:
  meshConfig:
    enableTracing: true
    defaultConfig:
      tracing:
        sampling: 100  # 샘플링 비율 (프로덕션에서는 1~10% 권장)
        zipkin:
          address: jaeger-collector.observability:9411

apiVersion: telemetry.istio.io/v1
kind: Telemetry
metadata:
  name: mesh-default
  namespace: istio-system
spec:
  accessLogging:
    - providers:
        - name: envoy

scrape_configs:
  - job_name: 'envoy-stats'
    metrics_path: /stats/prometheus
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true

# 서비스별 성공률 (SLI)
sum(rate(istio_requests_total{
  response_code!~"5.*",
  destination_service="reviews.production.svc.cluster.local"
}[5m])) /
sum(rate(istio_requests_total{
  destination_service="reviews.production.svc.cluster.local"
}[5m]))
 
# P99 지연 시간
histogram_quantile(0.99,
  sum(rate(istio_request_duration_milliseconds_bucket{
    destination_service="reviews.production.svc.cluster.local"
  }[5m])) by (le)
)
 
# 초당 요청 수 (RPS)
sum(rate(istio_requests_total{
  destination_service="reviews.production.svc.cluster.local"
}[5m]))

# 메시 전체 에러율
sum(rate(istio_requests_total{response_code=~"5.*"}[5m])) /
sum(rate(istio_requests_total[5m]))
 
# mTLS 적용 비율
sum(rate(istio_requests_total{
  connection_security_policy="mutual_tls"
}[5m])) /
sum(rate(istio_requests_total[5m]))

# Kiali 설치 (Istio 애드온)
kubectl apply -f https://raw.githubusercontent.com/istio/istio/release-1.25/samples/addons/kiali.yaml
 
# 대시보드 접근
istioctl dashboard kiali

# Cilium Helm values
hubble:
  enabled: true
  metrics:
    enabled:
      - dns
      - drop
      - tcp
      - flow
      - icmp
      - httpV2:exemplars=true;labelsContext=source_ip,destination_ip

# 특정 네임스페이스의 HTTP 트래픽 관찰
hubble observe --namespace production --protocol http
 
# 거부된 트래픽만 필터링
hubble observe --verdict DROPPED
 
# 특정 서비스 간 트래픽
hubble observe --from-label app=frontend --to-label app=reviews
 
# JSON 출력 (로그 수집 시스템 연동)
hubble observe --output json --namespace production

groups:
  - name: service-mesh-slo
    rules:
      # 서비스 에러율이 SLO 위반 (5분간 99.9% 미만)
      - alert: ServiceErrorBudgetBurn
        expr: |
          (
            1 - (
              sum(rate(istio_requests_total{
                response_code!~"5.*",
                destination_service=~".+\\.production\\.svc\\.cluster\\.local"
              }[5m])) by (destination_service)
              /
              sum(rate(istio_requests_total{
                destination_service=~".+\\.production\\.svc\\.cluster\\.local"
              }[5m])) by (destination_service)
            )
          ) > 0.001
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.destination_service }} 에러율 SLO 위반"
 
      # P99 지연 시간 임계치 초과
      - alert: ServiceLatencyHigh
        expr: |
          histogram_quantile(0.99,
            sum(rate(istio_request_duration_milliseconds_bucket{
              destination_service=~".+\\.production\\.svc\\.cluster\\.local"
            }[5m])) by (le, destination_service)
          ) > 500
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.destination_service }} P99 지연 500ms 초과"
 
      # mTLS 비율 저하 감지
      - alert: MtlsCoverageDropped
        expr: |
          sum(rate(istio_requests_total{
            connection_security_policy="mutual_tls"
          }[5m]))
          /
          sum(rate(istio_requests_total[5m]))
          < 0.99
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "mTLS 적용 비율이 99% 미만으로 하락"