Observability & モニタリング完全ガイド 2025: ロギング、メトリクス、トレーシング、アラート戦略

// Node.js OpenTelemetry設定
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
import { Resource } from '@opentelemetry/resources';
import {
  ATTR_SERVICE_NAME,
  ATTR_SERVICE_VERSION,
} from '@opentelemetry/semantic-conventions';

const sdk = new NodeSDK({
  resource: new Resource({
    [ATTR_SERVICE_NAME]: 'order-service',
    [ATTR_SERVICE_VERSION]: '1.2.0',
    environment: process.env.NODE_ENV || 'production',
  }),
  traceExporter: new OTLPTraceExporter({
    url: 'http://otel-collector:4318/v1/traces',
  }),
  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter({
      url: 'http://otel-collector:4318/v1/metrics',
    }),
    exportIntervalMillis: 30000,
  }),
  instrumentations: [
    getNodeAutoInstrumentations({
      // HTTP、Express、pg、Redisなどを自動計装
      '@opentelemetry/instrumentation-http': {
        ignoreIncomingRequestHook: (req) => {
          // ヘルスチェックパスを除外
          return req.url === '/health';
        },
      },
      '@opentelemetry/instrumentation-express': {
        enabled: true,
      },
    }),
  ],
});

sdk.start();

// 正常終了時にテレメトリをフラッシュ
process.on('SIGTERM', () => {
  sdk.shutdown().then(() => process.exit(0));
});

import { trace, SpanStatusCode, context } from '@opentelemetry/api';

const tracer = trace.getTracer('order-service');

async function processOrder(orderData) {
  // 手動Span作成
  return tracer.startActiveSpan('processOrder', async (span) => {
    try {
      // Spanに属性を追加
      span.setAttribute('order.id', orderData.id);
      span.setAttribute('order.total', orderData.total);
      span.setAttribute('order.items_count', orderData.items.length);

      // 子Span：在庫確認
      const inventory = await tracer.startActiveSpan(
        'checkInventory',
        async (childSpan) => {
          childSpan.setAttribute('inventory.warehouse', 'us-east-1');
          const result = await inventoryService.check(orderData.items);
          childSpan.end();
          return result;
        }
      );

      if (!inventory.available) {
        span.setStatus({
          code: SpanStatusCode.ERROR,
          message: 'Insufficient inventory',
        });
        throw new Error('Insufficient inventory');
      }

      // 子Span：決済処理
      const payment = await tracer.startActiveSpan(
        'processPayment',
        async (childSpan) => {
          childSpan.setAttribute('payment.method', orderData.paymentMethod);
          childSpan.setAttribute('payment.amount', orderData.total);
          const result = await paymentService.charge(orderData);
          childSpan.end();
          return result;
        }
      );

      // Spanイベント追加
      span.addEvent('order_completed', {
        'order.id': orderData.id,
        'payment.transaction_id': payment.transactionId,
      });

      span.setStatus({ code: SpanStatusCode.OK });
      return { orderId: orderData.id, transactionId: payment.transactionId };
    } catch (error) {
      span.setStatus({
        code: SpanStatusCode.ERROR,
        message: error.message,
      });
      span.recordException(error);
      throw error;
    } finally {
      span.end();
    }
  });
}

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    send_batch_size: 1024
    timeout: 5s

  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128

  # テールサンプリング（エラートレースを優先収集）
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: error-policy
        type: status_code
        status_code:
          status_codes: [ERROR]
      - name: slow-policy
        type: latency
        latency:
          threshold_ms: 1000
      - name: probabilistic-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

exporters:
  otlp/jaeger:
    endpoint: jaeger:4317
    tls:
      insecure: true

  prometheus:
    endpoint: 0.0.0.0:8889

  loki:
    endpoint: http://loki:3100/loki/api/v1/push

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch, tail_sampling]
      exporters: [otlp/jaeger]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [prometheus]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [loki]

// pinoを活用した構造化ロギング
import pino from 'pino';

const logger = pino({
  level: process.env.LOG_LEVEL || 'info',
  formatters: {
    level(label) {
      return { level: label };
    },
    bindings(bindings) {
      return {
        service: 'order-service',
        version: '1.2.0',
        host: bindings.hostname,
        pid: bindings.pid,
      };
    },
  },
  timestamp: pino.stdTimeFunctions.isoTime,
  // 機密情報の除去
  redact: ['req.headers.authorization', 'req.headers.cookie', '*.password'],
});

// Correlation IDミドルウェア
function correlationMiddleware(req, res, next) {
  const correlationId = req.headers['x-correlation-id'] || generateId();
  req.correlationId = correlationId;
  res.setHeader('x-correlation-id', correlationId);

  // リクエスト毎のchild logger
  req.log = logger.child({
    correlationId,
    requestId: generateId(),
    method: req.method,
    path: req.path,
    userAgent: req.headers['user-agent'],
  });

  next();
}

// 使用例
app.post('/api/orders', correlationMiddleware, async (req, res) => {
  req.log.info({ body: req.body }, 'Order creation started');

  try {
    const order = await createOrder(req.body);
    req.log.info(
      { orderId: order.id, duration: Date.now() - req.startTime },
      'Order created successfully'
    );
    res.json(order);
  } catch (error) {
    req.log.error(
      { err: error, body: req.body },
      'Order creation failed'
    );
    res.status(500).json({ error: 'Internal server error' });
  }
});

# docker-compose.yml - ELK Stack
version: '3.8'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ports:
      - "9200:9200"
    volumes:
      - es-data:/usr/share/elasticsearch/data

  logstash:
    image: docker.elastic.co/logstash/logstash:8.12.0
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
    depends_on:
      - elasticsearch

  kibana:
    image: docker.elastic.co/kibana/kibana:8.12.0
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    depends_on:
      - elasticsearch

volumes:
  es-data:

# logstash.conf
input {
  tcp {
    port => 5044
    codec => json
  }
}

filter {
  # タイムスタンプパース
  date {
    match => [ "timestamp", "ISO8601" ]
    target => "@timestamp"
  }

  # エラースタックトレースのパース
  if [level] == "error" {
    grok {
      match => {
        "stack" => "%{GREEDYDATA:error_class}: %{GREEDYDATA:error_message}"
      }
    }
  }
}

output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    index => "logs-%{+YYYY.MM.dd}"
  }
}

// Lokiに直接ログ送信 (winston-loki)
import winston from 'winston';
import LokiTransport from 'winston-loki';

const logger = winston.createLogger({
  transports: [
    new LokiTransport({
      host: 'http://loki:3100',
      labels: {
        service: 'order-service',
        environment: 'production',
      },
      json: true,
      batching: true,
      interval: 5,
    }),
  ],
});

// Node.js Prometheusクライアント
import { Registry, Counter, Gauge, Histogram, Summary } from 'prom-client';

const register = new Registry();

// デフォルトメトリクス収集（CPU、メモリ、イベントループなど）
import { collectDefaultMetrics } from 'prom-client';
collectDefaultMetrics({ register });

// Counter: HTTPリクエスト数
const httpRequestsTotal = new Counter({
  name: 'http_requests_total',
  help: 'Total number of HTTP requests',
  labelNames: ['method', 'path', 'status'],
  registers: [register],
});

// Gauge: 現在のアクティブ接続数
const activeConnections = new Gauge({
  name: 'active_connections',
  help: 'Number of active connections',
  registers: [register],
});

// Histogram: リクエスト応答時間
const httpRequestDuration = new Histogram({
  name: 'http_request_duration_seconds',
  help: 'HTTP request duration in seconds',
  labelNames: ['method', 'path', 'status'],
  buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
  registers: [register],
});

// Expressミドルウェア
function metricsMiddleware(req, res, next) {
  const start = process.hrtime.bigint();
  activeConnections.inc();

  res.on('finish', () => {
    const duration = Number(process.hrtime.bigint() - start) / 1e9;
    const labels = {
      method: req.method,
      path: req.route?.path || req.path,
      status: res.statusCode.toString(),
    };

    httpRequestsTotal.inc(labels);
    httpRequestDuration.observe(labels, duration);
    activeConnections.dec();
  });

  next();
}

// メトリクスエンドポイント
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', register.contentType);
  res.end(await register.metrics());
});

# 秒間リクエスト数 (RPS)
rate(http_requests_total[5m])

# サービス別エラー率
sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
/
sum(rate(http_requests_total[5m])) by (service)

# p99応答時間
histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))

# p95応答時間（パス別）
histogram_quantile(0.95,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, path)
)

# メモリ使用量の増加率
deriv(process_resident_memory_bytes[1h])

# Apdexスコア（目標応答時間0.5秒）
(
  sum(rate(http_request_duration_seconds_bucket{le="0.5"}[5m]))
  +
  sum(rate(http_request_duration_seconds_bucket{le="2.0"}[5m]))
) / 2
/
sum(rate(http_request_duration_seconds_count[5m]))

# prometheus-rules.yml
groups:
  - name: request_rates
    interval: 30s
    rules:
      # 事前計算されたリクエスト率
      - record: service:http_requests:rate5m
        expr: sum(rate(http_requests_total[5m])) by (service)

      # 事前計算されたエラー率
      - record: service:http_errors:rate5m
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)

      # 事前計算されたレイテンシパーセンタイル
      - record: service:http_latency:p99
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
          )

{
  "panels": [
    {
      "title": "Request Rate (RPS)",
      "type": "timeseries",
      "targets": [
        {
          "expr": "sum(rate(http_requests_total[5m])) by (service)",
          "legendFormat": "service"
        }
      ]
    },
    {
      "title": "Error Rate (%)",
      "type": "timeseries",
      "targets": [
        {
          "expr": "100 * sum(rate(http_requests_total{status=~\"5..\"}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service)",
          "legendFormat": "service"
        }
      ]
    },
    {
      "title": "Response Time (p99)",
      "type": "timeseries",
      "targets": [
        {
          "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))",
          "legendFormat": "service"
        }
      ]
    }
  ]
}

# CPU Utilization
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# Memory Utilization
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# Disk I/O Saturation
rate(node_disk_io_time_weighted_seconds_total[5m])

# Network Errors
rate(node_network_receive_errs_total[5m])
+ rate(node_network_transmit_errs_total[5m])

// W3C Trace Context伝搬
import { propagation, context, trace } from '@opentelemetry/api';

// HTTPクライアントでContext注入
async function makeRequest(url, data) {
  const headers = {};

  // 現在のコンテキストからトレース情報をヘッダーに注入
  propagation.inject(context.active(), headers);

  return fetch(url, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      ...headers, // traceparent, tracestate を含む
    },
    body: JSON.stringify(data),
  });
}

# docker-compose.yml - Jaeger
services:
  jaeger:
    image: jaegertracing/all-in-one:1.53
    ports:
      - "16686:16686"  # UI
      - "4317:4317"    # OTLP gRPC
      - "4318:4318"    # OTLP HTTP
    environment:
      - COLLECTOR_OTLP_ENABLED=true
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch:9200

// サンプリング設定
import { ParentBasedSampler, TraceIdRatioBasedSampler } from '@opentelemetry/sdk-trace-base';

// 比率ベースサンプリング（10%のみ収集）
const sampler = new TraceIdRatioBasedSampler(0.1);

// 親ベースサンプリング（親がサンプリングされたら子もサンプリング）
const parentBasedSampler = new ParentBasedSampler({
  root: new TraceIdRatioBasedSampler(0.1),
});

# Prometheusアラートルール
groups:
  - name: service_alerts
    rules:
      # Critical：即座に対応が必要
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)
          > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate on service"
          description: "Error rate is above 5% for 5 minutes"
          runbook_url: "https://wiki.example.com/runbooks/high-error-rate"

      # Warning：注意が必要
      - alert: HighLatency
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
          ) > 2
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High latency on service"

      # Info：情報提供
      - alert: PodRestart
        expr: |
          increase(kube_pod_container_status_restarts_total[1h]) > 3
        labels:
          severity: info
        annotations:
          summary: "Pod restarting frequently"

# alertmanager.yml
global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/xxx'

route:
  group_by: ['alertname', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default-slack'

  routes:
    # Critical -> PagerDuty + Slack
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      repeat_interval: 1h
      continue: true

    - match:
        severity: critical
      receiver: 'slack-critical'

    # Warning -> Slack
    - match:
        severity: warning
      receiver: 'slack-warning'
      repeat_interval: 4h

receivers:
  - name: 'default-slack'
    slack_configs:
      - channel: '#alerts-general'

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: 'YOUR_PAGERDUTY_KEY'
        severity: critical

  - name: 'slack-critical'
    slack_configs:
      - channel: '#alerts-critical'
        color: 'danger'

  - name: 'slack-warning'
    slack_configs:
      - channel: '#alerts-warning'
        color: 'warning'

inhibit_rules:
  # Criticalが発生したら同じサービスのWarningを抑制
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: ['service']

SLO: 99.9%可用性
= 1ヶ月（30日）に許容されるダウンタイム：43.2分
= Error Budget: 0.1%

Error Budget消費率：
- 全体の50%消費：注意
- 全体の75%消費：機能リリース凍結
- 全体の100%消費：安定性作業にのみ集中

# Burn Rateベースのアラート
groups:
  - name: slo_alerts
    rules:
      # 高速消費（1時間で2%消費）- 即座に対応
      - alert: SLOBurnRateCritical
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[1h]))
            /
            sum(rate(http_requests_total[1h]))
          ) > (14.4 * 0.001)
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "SLO burn rate critical - error budget exhausting fast"

      # 低速消費（6時間で5%消費）- 注意
      - alert: SLOBurnRateWarning
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[6h]))
            /
            sum(rate(http_requests_total[6h]))
          ) > (6 * 0.001)
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "SLO burn rate elevated"

+-----------------+     +------------------+
| Application     |---->| OTel Collector   |
| (OTel SDK)      |     | (収集/処理)      |
+-----------------+     +--------+---------+
                                 |
                    +------------+------------+
                    |            |            |
              +-----v-----+ +---v---+ +-----v-----+
              | Prometheus | | Tempo | | Loki      |
              | (Metrics)  | |(Trace)| | (Logs)    |
              +-----+------+ +---+---+ +-----+-----+
                    |            |            |
                    +------+-----+------+-----+
                           |
                      +----v----+
                      | Grafana |
                      | (統合可視化)|
                      +---------+

# ログ保存ポリシー
retention_policy:
  hot_tier: 7d        # 直近7日：高速検索
  warm_tier: 30d       # 直近30日：低速検索
  cold_tier: 90d       # 直近90日：アーカイブ
  delete_after: 365d   # 1年後に削除

// 適応型サンプリング
class AdaptiveSampler {
  constructor(targetRate = 100) {
    this.targetRate = targetRate; // 秒間目標収集量
    this.currentRate = 0;
    this.samplingProbability = 1.0;

    // 10秒ごとにサンプリング確率を調整
    setInterval(() => this.adjust(), 10000);
  }

  shouldSample() {
    this.currentRate++;
    return Math.random() < this.samplingProbability;
  }

  adjust() {
    if (this.currentRate > this.targetRate * 10) {
      this.samplingProbability = this.targetRate / this.currentRate;
    } else {
      this.samplingProbability = Math.min(1.0, this.samplingProbability * 1.1);
    }
    this.currentRate = 0;
  }
}

# Prometheus Recording Rulesで元データを集約
groups:
  - name: aggregation
    interval: 1m
    rules:
      # サービス別集約（インスタンスラベル削除でカーディナリティ削減）
      - record: service:requests:rate5m
        expr: sum(rate(http_requests_total[5m])) by (service, method, status)

      # 時間解像度を下げる（5分 -> 1時間）
      - record: service:requests:rate1h
        expr: sum(rate(http_requests_total[1h])) by (service)