Grafana Loki와 LogQL 마스터 가이드: 로그 파이프라인 설계부터 운영까지

[Application] --> [Grafana Alloy] --> [Distributor]
                                           |
                                    [Hash Ring]
                                           |
                                      [Ingester]
                                       /      \
                              [WAL]         [Object Storage (S3/GCS)]
                                                  |
                              [Compactor] <-------+
                                                  |
                              [Query Frontend] ---+---> [Querier]
                                                           |
                                                  [Index Gateway]

# Helm values - Monolithic 모드
loki:
  deploymentMode: SingleBinary
  auth_enabled: false
  commonConfig:
    replication_factor: 1
  storage:
    type: filesystem
  schemaConfig:
    configs:
      - from: '2024-01-01'
        store: tsdb
        object_store: filesystem
        schema: v13
        index:
          prefix: index_
          period: 24h
singleBinary:
  replicas: 1
  persistence:
    size: 50Gi

# Helm values - Simple Scalable 모드
loki:
  deploymentMode: SimpleScalable
  auth_enabled: true
  commonConfig:
    replication_factor: 3
  storage:
    type: s3
    s3:
      endpoint: s3.ap-northeast-2.amazonaws.com
      region: ap-northeast-2
      bucketnames: company-loki-logs
      access_key_id: ${AWS_ACCESS_KEY_ID}
      secret_access_key: ${AWS_SECRET_ACCESS_KEY}
  schemaConfig:
    configs:
      - from: '2024-01-01'
        store: tsdb
        object_store: s3
        schema: v13
        index:
          prefix: index_
          period: 24h
write:
  replicas: 3
  persistence:
    size: 50Gi
  resources:
    requests:
      cpu: '1'
      memory: 2Gi
    limits:
      memory: 4Gi
read:
  replicas: 3
  resources:
    requests:
      cpu: '1'
      memory: 2Gi
    limits:
      memory: 4Gi
backend:
  replicas: 2
  persistence:
    size: 50Gi
gateway:
  replicas: 2

# 기본 스트림 셀렉터
{namespace="production", app="payment-service"}

# 라인 필터 체이닝 - 필터는 순서대로 적용되므로 가장 좁은 필터를 먼저 배치
{namespace="production", app="payment-service"}
  |= "error"
  != "health-check"
  |~ "timeout|connection refused"

# Loki 3.x의 패턴 매치 필터 - regex보다 10배 빠름
{namespace="production"} |> "error <_> timeout"

# JSON 파서 - 구조화된 로그에서 필드 추출
{app="api-gateway"} | json | status >= 500

# logfmt 파서
{app="auth-service"} | logfmt | level="error" | duration > 5s

# pattern 파서 - nginx access log 파싱
{app="nginx"}
  | pattern "<ip> - - [<timestamp>] \"<method> <path> <_>\" <status> <bytes>"
  | status >= 400
  | line_format "{{.ip}} {{.method}} {{.path}} {{.status}}"

# regexp 파서 - 복잡한 비정형 로그
{app="legacy-service"}
  | regexp "(?P<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}) \\[(?P<level>\\w+)\\] (?P<message>.*)"
  | level="ERROR"

# 초당 에러 로그 발생률
rate({namespace="production", app="payment-service"} |= "error" [5m])

# 서비스별 5xx 에러 카운트
sum by (app) (
  count_over_time(
    {namespace="production"} | json | status >= 500 [5m]
  )
)

# unwrap을 활용한 응답 시간 P99 산출
quantile_over_time(0.99,
  {app="api-gateway"}
    | json
    | unwrap response_time_ms
    | __error__=""
  [5m]
) by (endpoint)

# 평균 응답 시간 트렌드
avg_over_time(
  {app="api-gateway"}
    | json
    | unwrap duration_seconds
    | __error__=""
  [5m]
) by (service)

# bytes_over_time - 로그 볼륨 모니터링
sum by (namespace) (bytes_over_time({namespace=~".+"} [1h]))

// Kubernetes Pod 디스커버리
discovery.kubernetes "pods" {
  role = "pod"
}

// 레이블 재매핑 - 필요한 레이블만 선택
discovery.relabel "pods" {
  targets = discovery.kubernetes.pods.targets

  rule {
    source_labels = ["__meta_kubernetes_namespace"]
    target_label  = "namespace"
  }
  rule {
    source_labels = ["__meta_kubernetes_pod_name"]
    target_label  = "pod"
  }
  rule {
    source_labels = ["__meta_kubernetes_pod_container_name"]
    target_label  = "container"
  }
  rule {
    source_labels = ["__meta_kubernetes_pod_label_app"]
    target_label  = "app"
  }
}

// Kubernetes 로그 소스
loki.source.kubernetes "pods" {
  targets    = discovery.relabel.pods.output
  forward_to = [loki.process.pipeline.receiver]
}

// 로그 처리 파이프라인
loki.process "pipeline" {
  // JSON 로그 파싱
  stage.json {
    expressions = {
      level   = "level",
      message = "msg",
    }
  }

  // level 레이블 정규화
  stage.label_drop {
    values = ["filename", "stream"]
  }

  // debug 로그 드롭 - 프로덕션에서 볼륨 절감
  stage.match {
    selector = "{level=\"debug\"}"
    action   = "drop"
  }

  forward_to = [loki.write.default.receiver]
}

// Loki 전송
loki.write "default" {
  endpoint {
    url = "http://loki-gateway.monitoring.svc:3100/loki/api/v1/push"
    tenant_id = "production"
  }
  external_labels = {
    cluster = "prod-kr-01",
    region  = "ap-northeast-2",
  }
}

# Promtail 설정을 Alloy River 문법으로 변환
alloy convert --source-format=promtail --output=alloy-config.river promtail-config.yaml

# 변환 결과 검증 (dry-run)
alloy run --stability.level=generally-available alloy-config.river

# Kubernetes DaemonSet 배포 (Helm)
helm upgrade --install alloy grafana/alloy \
  --namespace monitoring \
  --set alloy.configMap=true \
  -f alloy-values.yaml

# loki.yaml - TSDB + S3 스토리지 설정
schema_config:
  configs:
    - from: '2024-01-01'
      store: tsdb
      object_store: s3
      schema: v13
      index:
        prefix: loki_index_
        period: 24h

storage_config:
  tsdb_shipper:
    active_index_directory: /loki/tsdb-index
    cache_location: /loki/tsdb-cache
  aws:
    s3: s3://ap-northeast-2/company-loki-chunks
    s3forcepathstyle: false

# Loki 3.4+ 통합 스토리지 설정 (Thanos Object Storage Client)
common:
  storage:
    s3:
      endpoint: s3.ap-northeast-2.amazonaws.com
      region: ap-northeast-2
      bucketnames: company-loki-data
      access_key_id: ${AWS_ACCESS_KEY_ID}
      secret_access_key: ${AWS_SECRET_ACCESS_KEY}

# loki.yaml - 리텐션 설정
compactor:
  working_directory: /loki/compactor
  compaction_interval: 10m
  retention_enabled: true
  retention_delete_delay: 2h
  retention_delete_worker_count: 150

limits_config:
  retention_period: 720h # 글로벌: 30일
  retention_stream:
    - selector: '{namespace="production"}'
      priority: 1
      period: 2160h # 프로덕션: 90일
    - selector: '{namespace="staging"}'
      priority: 2
      period: 168h # 스테이징: 7일
    - selector: '{level="debug"}'
      priority: 3
      period: 72h # 디버그 로그: 3일

# loki-alert-rules.yaml
groups:
  - name: application-errors
    rules:
      - alert: HighErrorRate
        expr: |
          sum by (app, namespace) (
            rate({namespace="production"} |= "error" [5m])
          ) > 10
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: 'High error rate in {{ $labels.app }}'
          description: '{{ $labels.app }} in {{ $labels.namespace }} has error rate {{ $value }}/s for 5 minutes.'

      - alert: SlowResponseTime
        expr: |
          quantile_over_time(0.95,
            {app="api-gateway"}
              | json
              | unwrap response_time_ms
              | __error__=""
            [5m]
          ) by (app) > 3000
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: 'P95 response time exceeds 3s for {{ $labels.app }}'

      - alert: LogVolumeSpike
        expr: |
          sum by (namespace) (bytes_over_time({namespace=~".+"} [5m]))
            / sum by (namespace) (bytes_over_time({namespace=~".+"} [5m] offset 1h))
          > 3
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: 'Log volume spike in {{ $labels.namespace }} (3x increase)'

# 활성 스트림 수 확인
curl -s http://loki:3100/metrics | grep loki_ingester_streams_created_total

# 테넌트별 스트림 수 확인
curl -s http://loki:3100/loki/api/v1/index/stats \
  --header "X-Scope-OrgID: production" \
  --data-urlencode 'query={namespace="production"}' \
  --data-urlencode 'start=1h'

# Ingester 메모리 사용량 확인
kubectl top pods -n monitoring -l app.kubernetes.io/component=ingester

# limits_config 튜닝
limits_config:
  max_query_length: 721h # 최대 쿼리 시간 범위
  max_query_parallelism: 32 # 쿼리 병렬도
  query_timeout: 5m # 단일 쿼리 타임아웃
  split_queries_by_interval: 30m # 쿼리 분할 간격
  max_query_series: 500 # 최대 시리즈 수

# Loki 3.4+ out-of-order 허용 설정
limits_config:
  unordered_writes: true
  max_chunk_age: 2h

# 1. 장애 Ingester Pod 확인
kubectl get pods -n monitoring -l app.kubernetes.io/component=ingester

# 2. WAL 상태 확인
kubectl exec -n monitoring ingester-0 -- ls -la /loki/wal/

# 3. 장애 Pod 재시작 (WAL이 있으므로 데이터 복구 가능)
kubectl delete pod -n monitoring ingester-0

# 4. 복구 후 해시 링 상태 확인
curl -s http://loki:3100/ring | jq '.shards[] | {addr, state, tokens}'

# 5. flush 강제 실행 (필요시)
curl -X POST http://loki:3100/ingester/flush