Observability & 모니터링 완전 가이드 2025: 로깅, 메트릭, 트레이싱, 알림 전략

// Node.js OpenTelemetry 설정
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
import { Resource } from '@opentelemetry/resources';
import {
  ATTR_SERVICE_NAME,
  ATTR_SERVICE_VERSION,
} from '@opentelemetry/semantic-conventions';

const sdk = new NodeSDK({
  resource: new Resource({
    [ATTR_SERVICE_NAME]: 'order-service',
    [ATTR_SERVICE_VERSION]: '1.2.0',
    environment: process.env.NODE_ENV || 'production',
  }),
  traceExporter: new OTLPTraceExporter({
    url: 'http://otel-collector:4318/v1/traces',
  }),
  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter({
      url: 'http://otel-collector:4318/v1/metrics',
    }),
    exportIntervalMillis: 30000,
  }),
  instrumentations: [
    getNodeAutoInstrumentations({
      // HTTP, Express, pg, Redis 등 자동 계측
      '@opentelemetry/instrumentation-http': {
        ignoreIncomingRequestHook: (req) => {
          // 헬스체크 경로 제외
          return req.url === '/health';
        },
      },
      '@opentelemetry/instrumentation-express': {
        enabled: true,
      },
    }),
  ],
});

sdk.start();

// 정상 종료 시 텔레메트리 플러시
process.on('SIGTERM', () => {
  sdk.shutdown().then(() => process.exit(0));
});

import { trace, SpanStatusCode, context } from '@opentelemetry/api';

const tracer = trace.getTracer('order-service');

async function processOrder(orderData) {
  // 수동 Span 생성
  return tracer.startActiveSpan('processOrder', async (span) => {
    try {
      // Span에 속성 추가
      span.setAttribute('order.id', orderData.id);
      span.setAttribute('order.total', orderData.total);
      span.setAttribute('order.items_count', orderData.items.length);

      // 하위 Span: 재고 확인
      const inventory = await tracer.startActiveSpan(
        'checkInventory',
        async (childSpan) => {
          childSpan.setAttribute('inventory.warehouse', 'us-east-1');
          const result = await inventoryService.check(orderData.items);
          childSpan.end();
          return result;
        }
      );

      if (!inventory.available) {
        span.setStatus({
          code: SpanStatusCode.ERROR,
          message: 'Insufficient inventory',
        });
        throw new Error('Insufficient inventory');
      }

      // 하위 Span: 결제 처리
      const payment = await tracer.startActiveSpan(
        'processPayment',
        async (childSpan) => {
          childSpan.setAttribute('payment.method', orderData.paymentMethod);
          childSpan.setAttribute('payment.amount', orderData.total);
          const result = await paymentService.charge(orderData);
          childSpan.end();
          return result;
        }
      );

      // Span 이벤트 추가
      span.addEvent('order_completed', {
        'order.id': orderData.id,
        'payment.transaction_id': payment.transactionId,
      });

      span.setStatus({ code: SpanStatusCode.OK });
      return { orderId: orderData.id, transactionId: payment.transactionId };
    } catch (error) {
      span.setStatus({
        code: SpanStatusCode.ERROR,
        message: error.message,
      });
      span.recordException(error);
      throw error;
    } finally {
      span.end();
    }
  });
}

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    send_batch_size: 1024
    timeout: 5s

  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128

  attributes:
    actions:
      - key: environment
        value: production
        action: upsert

  # 테일 샘플링 (에러 트레이스 우선 수집)
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: error-policy
        type: status_code
        status_code:
          status_codes: [ERROR]
      - name: slow-policy
        type: latency
        latency:
          threshold_ms: 1000
      - name: probabilistic-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

exporters:
  otlp/jaeger:
    endpoint: jaeger:4317
    tls:
      insecure: true

  prometheus:
    endpoint: 0.0.0.0:8889

  loki:
    endpoint: http://loki:3100/loki/api/v1/push

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch, tail_sampling]
      exporters: [otlp/jaeger]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [prometheus]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [loki]

// pino를 활용한 구조화된 로깅
import pino from 'pino';

const logger = pino({
  level: process.env.LOG_LEVEL || 'info',
  // JSON 포맷 (기본)
  formatters: {
    level(label) {
      return { level: label };
    },
    bindings(bindings) {
      return {
        service: 'order-service',
        version: '1.2.0',
        host: bindings.hostname,
        pid: bindings.pid,
      };
    },
  },
  // 타임스탬프 포맷
  timestamp: pino.stdTimeFunctions.isoTime,
  // 민감 정보 제거
  redact: ['req.headers.authorization', 'req.headers.cookie', '*.password'],
});

// Correlation ID 미들웨어
function correlationMiddleware(req, res, next) {
  const correlationId = req.headers['x-correlation-id'] || generateId();
  req.correlationId = correlationId;
  res.setHeader('x-correlation-id', correlationId);

  // 요청별 child logger
  req.log = logger.child({
    correlationId,
    requestId: generateId(),
    method: req.method,
    path: req.path,
    userAgent: req.headers['user-agent'],
  });

  next();
}

// 사용 예시
app.post('/api/orders', correlationMiddleware, async (req, res) => {
  req.log.info({ body: req.body }, 'Order creation started');

  try {
    const order = await createOrder(req.body);
    req.log.info(
      { orderId: order.id, duration: Date.now() - req.startTime },
      'Order created successfully'
    );
    res.json(order);
  } catch (error) {
    req.log.error(
      { err: error, body: req.body },
      'Order creation failed'
    );
    res.status(500).json({ error: 'Internal server error' });
  }
});

// 환경별 로그 레벨 설정
const logLevels = {
  production: 'info',
  staging: 'debug',
  development: 'trace',
};

# docker-compose.yml - ELK Stack
version: '3.8'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ports:
      - "9200:9200"
    volumes:
      - es-data:/usr/share/elasticsearch/data

  logstash:
    image: docker.elastic.co/logstash/logstash:8.12.0
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
    depends_on:
      - elasticsearch

  kibana:
    image: docker.elastic.co/kibana/kibana:8.12.0
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    depends_on:
      - elasticsearch

volumes:
  es-data:

# logstash.conf
input {
  tcp {
    port => 5044
    codec => json
  }
}

filter {
  # 타임스탬프 파싱
  date {
    match => [ "timestamp", "ISO8601" ]
    target => "@timestamp"
  }

  # 에러 스택 트레이스 파싱
  if [level] == "error" {
    grok {
      match => {
        "stack" => "%{GREEDYDATA:error_class}: %{GREEDYDATA:error_message}"
      }
    }
  }

  # 지역 정보 추가
  if [client_ip] {
    geoip {
      source => "client_ip"
    }
  }
}

output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    index => "logs-%{+YYYY.MM.dd}"
  }
}

# Loki 설정
auth_enabled: false

server:
  http_listen_port: 3100

common:
  ring:
    kvstore:
      store: inmemory
  replication_factor: 1

schema_config:
  configs:
    - from: 2024-01-01
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h

storage_config:
  filesystem:
    directory: /loki/chunks

limits_config:
  retention_period: 30d

// Loki에 직접 로그 전송 (winston-loki)
import winston from 'winston';
import LokiTransport from 'winston-loki';

const logger = winston.createLogger({
  transports: [
    new LokiTransport({
      host: 'http://loki:3100',
      labels: {
        service: 'order-service',
        environment: 'production',
      },
      json: true,
      batching: true,
      interval: 5,
    }),
  ],
});

// Node.js Prometheus 클라이언트
import { Registry, Counter, Gauge, Histogram, Summary } from 'prom-client';

const register = new Registry();

// 기본 메트릭 수집 (CPU, 메모리, 이벤트 루프 등)
import { collectDefaultMetrics } from 'prom-client';
collectDefaultMetrics({ register });

// Counter: HTTP 요청 수
const httpRequestsTotal = new Counter({
  name: 'http_requests_total',
  help: 'Total number of HTTP requests',
  labelNames: ['method', 'path', 'status'],
  registers: [register],
});

// Gauge: 현재 활성 연결 수
const activeConnections = new Gauge({
  name: 'active_connections',
  help: 'Number of active connections',
  registers: [register],
});

// Histogram: 요청 응답 시간
const httpRequestDuration = new Histogram({
  name: 'http_request_duration_seconds',
  help: 'HTTP request duration in seconds',
  labelNames: ['method', 'path', 'status'],
  buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
  registers: [register],
});

// Summary: DB 쿼리 시간
const dbQueryDuration = new Summary({
  name: 'db_query_duration_seconds',
  help: 'Database query duration',
  labelNames: ['operation', 'table'],
  percentiles: [0.5, 0.9, 0.95, 0.99],
  registers: [register],
});

// Express 미들웨어
function metricsMiddleware(req, res, next) {
  const start = process.hrtime.bigint();
  activeConnections.inc();

  res.on('finish', () => {
    const duration = Number(process.hrtime.bigint() - start) / 1e9;
    const labels = {
      method: req.method,
      path: req.route?.path || req.path,
      status: res.statusCode.toString(),
    };

    httpRequestsTotal.inc(labels);
    httpRequestDuration.observe(labels, duration);
    activeConnections.dec();
  });

  next();
}

// 메트릭 엔드포인트
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', register.contentType);
  res.end(await register.metrics());
});

# 초당 요청 수 (RPS)
rate(http_requests_total[5m])

# 서비스별 에러율
sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
/
sum(rate(http_requests_total[5m])) by (service)

# p99 응답 시간
histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))

# p95 응답 시간 (경로별)
histogram_quantile(0.95,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, path)
)

# 메모리 사용량 증가율
deriv(process_resident_memory_bytes[1h])

# 가동 시간
time() - process_start_time_seconds

# Apdex 스코어 (목표 응답시간 0.5초)
(
  sum(rate(http_request_duration_seconds_bucket{le="0.5"}[5m]))
  +
  sum(rate(http_request_duration_seconds_bucket{le="2.0"}[5m]))
) / 2
/
sum(rate(http_request_duration_seconds_count[5m]))

# prometheus-rules.yml
groups:
  - name: request_rates
    interval: 30s
    rules:
      # 사전 계산된 요청률
      - record: service:http_requests:rate5m
        expr: sum(rate(http_requests_total[5m])) by (service)

      # 사전 계산된 에러율
      - record: service:http_errors:rate5m
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)

      # 사전 계산된 지연시간 분위수
      - record: service:http_latency:p99
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
          )

{
  "panels": [
    {
      "title": "Request Rate (RPS)",
      "type": "timeseries",
      "targets": [
        {
          "expr": "sum(rate(http_requests_total[5m])) by (service)",
          "legendFormat": "service"
        }
      ]
    },
    {
      "title": "Error Rate (%)",
      "type": "timeseries",
      "targets": [
        {
          "expr": "100 * sum(rate(http_requests_total{status=~\"5..\"}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service)",
          "legendFormat": "service"
        }
      ]
    },
    {
      "title": "Response Time (p99)",
      "type": "timeseries",
      "targets": [
        {
          "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))",
          "legendFormat": "service"
        }
      ]
    }
  ]
}

# CPU Utilization
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# Memory Utilization
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# Disk I/O Saturation
rate(node_disk_io_time_weighted_seconds_total[5m])

# Network Errors
rate(node_network_receive_errs_total[5m])
+ rate(node_network_transmit_errs_total[5m])

// W3C Trace Context 전파
// 요청 헤더:
// traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01
// tracestate: congo=t61rcWkgMzE

// Express 미들웨어에서 Context 추출/주입
import { propagation, context, trace } from '@opentelemetry/api';

// HTTP 클라이언트에서 Context 주입
async function makeRequest(url, data) {
  const headers = {};

  // 현재 컨텍스트에서 trace 정보를 헤더에 주입
  propagation.inject(context.active(), headers);

  return fetch(url, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      ...headers, // traceparent, tracestate 포함
    },
    body: JSON.stringify(data),
  });
}

// gRPC에서 Context 전파
import { GrpcInstrumentation } from '@opentelemetry/instrumentation-grpc';

// 자동 계측으로 gRPC 메타데이터에 Context 주입/추출

# docker-compose.yml - Jaeger
services:
  jaeger:
    image: jaegertracing/all-in-one:1.53
    ports:
      - "16686:16686"  # UI
      - "4317:4317"    # OTLP gRPC
      - "4318:4318"    # OTLP HTTP
    environment:
      - COLLECTOR_OTLP_ENABLED=true
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch:9200

// 샘플링 설정
import { ParentBasedSampler, TraceIdRatioBasedSampler } from '@opentelemetry/sdk-trace-base';

// 비율 기반 샘플링 (10%만 수집)
const sampler = new TraceIdRatioBasedSampler(0.1);

// 부모 기반 샘플링 (부모가 샘플링되었으면 자식도 샘플링)
const parentBasedSampler = new ParentBasedSampler({
  root: new TraceIdRatioBasedSampler(0.1),
});

# Prometheus 알림 규칙
groups:
  - name: service_alerts
    rules:
      # Critical: 즉시 대응 필요
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)
          > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate on service"
          description: "Error rate is above 5% for 5 minutes"
          runbook_url: "https://wiki.example.com/runbooks/high-error-rate"

      # Warning: 주의 필요
      - alert: HighLatency
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
          ) > 2
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High latency on service"
          description: "p99 latency is above 2 seconds for 10 minutes"

      # Info: 정보 제공
      - alert: PodRestart
        expr: |
          increase(kube_pod_container_status_restarts_total[1h]) > 3
        labels:
          severity: info
        annotations:
          summary: "Pod restarting frequently"

# alertmanager.yml
global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/xxx'

route:
  group_by: ['alertname', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default-slack'

  routes:
    # Critical -> PagerDuty + Slack
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      repeat_interval: 1h
      continue: true

    - match:
        severity: critical
      receiver: 'slack-critical'

    # Warning -> Slack
    - match:
        severity: warning
      receiver: 'slack-warning'
      repeat_interval: 4h

    # Info -> Slack (업무 시간만)
    - match:
        severity: info
      receiver: 'slack-info'
      active_time_intervals:
        - business-hours

receivers:
  - name: 'default-slack'
    slack_configs:
      - channel: '#alerts-general'

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: 'YOUR_PAGERDUTY_KEY'
        severity: critical

  - name: 'slack-critical'
    slack_configs:
      - channel: '#alerts-critical'
        color: 'danger'

  - name: 'slack-warning'
    slack_configs:
      - channel: '#alerts-warning'
        color: 'warning'

  - name: 'slack-info'
    slack_configs:
      - channel: '#alerts-info'

time_intervals:
  - name: business-hours
    time_intervals:
      - weekdays: ['monday:friday']
        times:
          - start_time: '09:00'
            end_time: '18:00'

inhibit_rules:
  # Critical이 발생하면 같은 서비스의 Warning 억제
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: ['service']

# Runbook: High Error Rate

## 알림 조건
- 에러율(5xx)이 5% 이상인 상태가 5분 이상 지속

## 즉시 확인 사항
1. 영향 범위 확인 (어떤 엔드포인트인지)
2. 최근 배포 이력 확인
3. 의존 서비스 상태 확인

## 대응 절차
1. Grafana 대시보드에서 에러 패턴 확인
2. 로그에서 에러 상세 내용 확인
3. 트레이스에서 실패 지점 식별
4. 최근 배포가 원인이면 롤백
5. 의존 서비스 문제이면 서킷 브레이커 확인

## 에스컬레이션
- 15분 내 해결 불가: 팀 리드에 보고
- 30분 내 해결 불가: 시니어 엔지니어 호출

SLO: 99.9% 가용성
= 한 달(30일)에 허용되는 다운타임: 43.2분
= Error Budget: 0.1%

Error Budget 소진율:
- 전체의 50% 소진: 주의
- 전체의 75% 소진: 기능 릴리스 동결
- 전체의 100% 소진: 안정성 작업에만 집중

# Burn Rate 기반 알림
groups:
  - name: slo_alerts
    rules:
      # 빠른 소진 (1시간 내 2% 소진) - 즉시 대응
      - alert: SLOBurnRateCritical
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[1h]))
            /
            sum(rate(http_requests_total[1h]))
          ) > (14.4 * 0.001)
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "SLO burn rate critical - error budget exhausting fast"

      # 느린 소진 (6시간 내 5% 소진) - 주의
      - alert: SLOBurnRateWarning
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[6h]))
            /
            sum(rate(http_requests_total[6h]))
          ) > (6 * 0.001)
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "SLO burn rate elevated"

+-----------------+     +------------------+
| Application     |---->| OTel Collector   |
| (OTel SDK)      |     | (수집/처리)      |
+-----------------+     +--------+---------+
                                 |
                    +------------+------------+
                    |            |            |
              +-----v-----+ +---v---+ +-----v-----+
              | Prometheus | | Tempo | | Loki      |
              | (Metrics)  | |(Trace)| | (Logs)    |
              +-----+------+ +---+---+ +-----+-----+
                    |            |            |
                    +------+-----+------+-----+
                           |            |
                      +----v----+ +----v----+
                      | Grafana | | Grafana |
                      | (시각화)| | (통합)  |
                      +---------+ +---------+

# 로그 보존 정책
retention_policy:
  hot_tier: 7d        # 최근 7일: 빠른 검색
  warm_tier: 30d       # 최근 30일: 느린 검색
  cold_tier: 90d       # 최근 90일: 아카이브
  delete_after: 365d   # 1년 후 삭제

# 인덱스 수명 관리 (Elasticsearch ILM)
index_lifecycle:
  phases:
    hot:
      actions:
        rollover:
          max_size: 50gb
          max_age: 1d
    warm:
      min_age: 7d
      actions:
        shrink:
          number_of_shards: 1
        forcemerge:
          max_num_segments: 1
    cold:
      min_age: 30d
      actions:
        searchable_snapshot:
          snapshot_repository: s3_repo
    delete:
      min_age: 365d

// 적응형 샘플링
class AdaptiveSampler {
  constructor(targetRate = 100) {
    this.targetRate = targetRate; // 초당 목표 수집량
    this.currentRate = 0;
    this.samplingProbability = 1.0;

    // 10초마다 샘플링 확률 조정
    setInterval(() => this.adjust(), 10000);
  }

  shouldSample() {
    this.currentRate++;
    return Math.random() < this.samplingProbability;
  }

  adjust() {
    if (this.currentRate > this.targetRate * 10) {
      // 트래픽이 목표의 10배 이상이면 샘플링 확률 줄이기
      this.samplingProbability = this.targetRate / this.currentRate;
    } else {
      this.samplingProbability = Math.min(1.0, this.samplingProbability * 1.1);
    }
    this.currentRate = 0;
  }
}

# Prometheus Recording Rules로 원본 데이터 집계
groups:
  - name: aggregation
    interval: 1m
    rules:
      # 서비스별 집계 (인스턴스 레이블 제거로 카디널리티 감소)
      - record: service:requests:rate5m
        expr: sum(rate(http_requests_total[5m])) by (service, method, status)

      # 시간 해상도 줄이기 (5분 -> 1시간)
      - record: service:requests:rate1h
        expr: sum(rate(http_requests_total[1h])) by (service)