オブザーバビリティ(Observability)完全ガイド2025：Prometheus、Grafana、OpenTelemetryでシステムを透明に

# Counter例
http_requests_total{method="GET", path="/api/users", status="200"} 15234

# Gauge例
node_memory_usage_bytes{instance="web-01"} 1073741824

# Histogram例
http_request_duration_seconds_bucket{le="0.1"} 24054
http_request_duration_seconds_bucket{le="0.5"} 33444
http_request_duration_seconds_bucket{le="1.0"} 34055

{
  "timestamp": "2025-03-15T10:30:45.123Z",
  "level": "ERROR",
  "service": "payment-service",
  "traceId": "abc123def456",
  "spanId": "span789",
  "message": "Payment processing failed",
  "userId": "user-42",
  "orderId": "order-1234",
  "error": "Timeout connecting to payment gateway",
  "duration_ms": 5000
}

[Trace: abc123def456]
|-- [Span: API Gateway] 2ms
|   |-- [Span: Auth Service] 5ms
|   |   +-- [Span: Redis Cache Lookup] 1ms
|   |-- [Span: User Service] 15ms
|   |   +-- [Span: PostgreSQL Query] 8ms
|   +-- [Span: Payment Service] 5003ms  <-- ボトルネック！
|       +-- [Span: External Payment API] 5000ms (TIMEOUT)
+-- Total: 5025ms

+-------------+     +--------------+     +-----------+
|  Targets    |---->|  Prometheus  |---->|  Grafana  |
|  (exporters)|pull |  Server      |query|           |
+-------------+     |  - TSDB      |     +-----------+
                    |  - Rules     |
                    |  - AlertMgr  |
                    +--------------+
                          |
                    +-----v-----+
                    | AlertMgr  |
                    | - Routing |
                    | - Silence |
                    +-----------+

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "alert_rules.yml"
  - "recording_rules.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']

  - job_name: 'app-service'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)

# 1. 現在の秒間リクエスト数 (rate)
rate(http_requests_total[5m])

# 2. サービス別エラー率
sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
/
sum(rate(http_requests_total[5m])) by (service)

# 3. 95パーセンタイル応答時間
histogram_quantile(0.95,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
)

# 4. メモリ使用率 (%)
(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes)
/ node_memory_MemTotal_bytes * 100

# 5. CPU使用率
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 6. ディスク空き容量が10%未満のノード
node_filesystem_avail_bytes / node_filesystem_size_bytes * 100 < 10

# 7. Pod再起動回数（過去1時間）
increase(kube_pod_container_status_restarts_total[1h]) > 3

# 8. サービス可用性（過去30日間）
1 - (
  sum(increase(http_requests_total{status=~"5.."}[30d]))
  /
  sum(increase(http_requests_total[30d]))
)

# recording_rules.yml
groups:
  - name: service_metrics
    interval: 30s
    rules:
      - record: service:http_requests:rate5m
        expr: sum(rate(http_requests_total[5m])) by (service)

      - record: service:http_errors:rate5m
        expr: sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)

      - record: service:http_error_rate:ratio
        expr: service:http_errors:rate5m / service:http_requests:rate5m

      - record: service:http_latency:p95
        expr: histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))

# alert_rules.yml
groups:
  - name: service_alerts
    rules:
      - alert: HighErrorRate
        expr: service:http_error_rate:ratio > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate on service {{ $labels.service }}"
          description: "Error rate is {{ $value | humanizePercentage }} for 5+ minutes"

      - alert: HighLatency
        expr: service:http_latency:p95 > 2.0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High p95 latency on {{ $labels.service }}"
          description: "P95 latency is {{ $value }}s (threshold: 2s)"

      - alert: PodCrashLooping
        expr: increase(kube_pod_container_status_restarts_total[1h]) > 5
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Pod {{ $labels.pod }} is crash looping"

{
  "dashboard": {
    "title": "Service Overview",
    "panels": [
      {
        "title": "Request Rate",
        "type": "timeseries",
        "datasource": "Prometheus",
        "targets": [
          {
            "expr": "sum(rate(http_requests_total[5m])) by (service)",
            "legendFormat": "{{ service }}"
          }
        ]
      },
      {
        "title": "Error Rate",
        "type": "stat",
        "datasource": "Prometheus",
        "targets": [
          {
            "expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m])) * 100"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "thresholds": {
              "steps": [
                { "value": 0, "color": "green" },
                { "value": 1, "color": "yellow" },
                { "value": 5, "color": "red" }
              ]
            },
            "unit": "percent"
          }
        }
      }
    ],
    "templating": {
      "list": [
        {
          "name": "service",
          "type": "query",
          "query": "label_values(http_requests_total, service)",
          "refresh": 2
        },
        {
          "name": "environment",
          "type": "custom",
          "options": ["production", "staging", "development"]
        }
      ]
    }
  }
}

# Grafana Alert Rule (provisioning)
apiVersion: 1
groups:
  - orgId: 1
    name: service_alerts
    folder: Production
    interval: 1m
    rules:
      - uid: high-error-rate
        title: High Error Rate
        condition: C
        data:
          - refId: A
            datasourceUid: prometheus
            model:
              expr: sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          - refId: B
            datasourceUid: prometheus
            model:
              expr: sum(rate(http_requests_total[5m])) by (service)
          - refId: C
            datasourceUid: __expr__
            model:
              type: math
              expression: "$A / $B > 0.05"
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Error rate exceeds 5%"

+--------------+     +----------------+     +-------------+
| Application  |---->|  OTel          |---->|  Backend    |
| + OTel SDK   |     |  Collector     |     |  - Jaeger   |
|              |     |  - Receivers   |     |  - Tempo    |
|              |     |  - Processors  |     |  - Prometheus|
|              |     |  - Exporters   |     |  - Loki     |
+--------------+     +----------------+     +-------------+

// tracing.ts - アプリケーション起動時に最初にimport
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc';
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
import { Resource } from '@opentelemetry/resources';
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';

const sdk = new NodeSDK({
  resource: new Resource({
    [ATTR_SERVICE_NAME]: 'payment-service',
    [ATTR_SERVICE_VERSION]: '1.2.0',
    environment: 'production',
  }),
  traceExporter: new OTLPTraceExporter({
    url: 'http://otel-collector:4317',
  }),
  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter({
      url: 'http://otel-collector:4317',
    }),
    exportIntervalMillis: 30000,
  }),
  instrumentations: [
    getNodeAutoInstrumentations({
      '@opentelemetry/instrumentation-http': { enabled: true },
      '@opentelemetry/instrumentation-express': { enabled: true },
      '@opentelemetry/instrumentation-pg': { enabled: true },
      '@opentelemetry/instrumentation-redis': { enabled: true },
    }),
  ],
});

sdk.start();

import { trace, SpanStatusCode, context } from '@opentelemetry/api';

const tracer = trace.getTracer('payment-service');

async function processPayment(orderId: string, amount: number) {
  return tracer.startActiveSpan('processPayment', async (span) => {
    try {
      span.setAttribute('order.id', orderId);
      span.setAttribute('payment.amount', amount);
      span.setAttribute('payment.currency', 'USD');

      // 子Spanを生成
      const validationResult = await tracer.startActiveSpan(
        'validatePayment',
        async (validationSpan) => {
          const result = await validatePaymentDetails(orderId);
          validationSpan.setAttribute('validation.result', result.valid);
          validationSpan.end();
          return result;
        }
      );

      if (!validationResult.valid) {
        span.setStatus({
          code: SpanStatusCode.ERROR,
          message: 'Payment validation failed',
        });
        throw new Error('Invalid payment');
      }

      const result = await chargePayment(orderId, amount);
      span.setAttribute('payment.transactionId', result.transactionId);
      span.setStatus({ code: SpanStatusCode.OK });

      return result;
    } catch (error) {
      span.recordException(error);
      span.setStatus({
        code: SpanStatusCode.ERROR,
        message: error.message,
      });
      throw error;
    } finally {
      span.end();
    }
  });
}

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

  prometheus:
    config:
      scrape_configs:
        - job_name: 'otel-collector'
          scrape_interval: 10s
          static_configs:
            - targets: ['0.0.0.0:8888']

processors:
  batch:
    timeout: 5s
    send_batch_size: 1000

  memory_limiter:
    check_interval: 1s
    limit_mib: 512

  attributes:
    actions:
      - key: environment
        value: production
        action: upsert

exporters:
  otlp/jaeger:
    endpoint: jaeger:4317
    tls:
      insecure: true

  prometheusremotewrite:
    endpoint: http://prometheus:9090/api/v1/write

  loki:
    endpoint: http://loki:3100/loki/api/v1/push

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch, attributes]
      exporters: [otlp/jaeger]

    metrics:
      receivers: [otlp, prometheus]
      processors: [memory_limiter, batch]
      exporters: [prometheusremotewrite]

    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [loki]

# docker-compose.yaml
version: '3.8'
services:
  jaeger:
    image: jaegertracing/all-in-one:latest
    ports:
      - "16686:16686"  # Jaeger UI
      - "4317:4317"    # OTLP gRPC
      - "4318:4318"    # OTLP HTTP
    environment:
      - COLLECTOR_OTLP_ENABLED=true

  otel-collector:
    image: otel/opentelemetry-collector-contrib:latest
    command: ["--config=/etc/otel-collector-config.yaml"]
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
    ports:
      - "4317:4317"
      - "4318:4318"
    depends_on:
      - jaeger

# サービス別エラーログ
{service="payment-service"} |= "ERROR"

# JSONパース後フィルタ
{service="api-gateway"} | json | status >= 500

# エラー発生頻度（1分あたり）
count_over_time({service="payment-service"} |= "ERROR" [1m])

# 遅いリクエストフィルタ（1秒以上）
{service="api-gateway"} | json | duration > 1000

# 特定トレースIDで全ログ検索
{service=~".+"} |= "trace_id=abc123def456"

import pino from 'pino';

const logger = pino({
  level: process.env.LOG_LEVEL || 'info',
  formatters: {
    level(label) {
      return { level: label };
    },
  },
  timestamp: pino.stdTimeFunctions.isoTime,
  base: {
    service: 'payment-service',
    version: '1.2.0',
    environment: process.env.NODE_ENV,
  },
});

// リクエストごとのコンテキスト付与
function createRequestLogger(req) {
  return logger.child({
    requestId: req.id,
    traceId: req.headers['x-trace-id'],
    userId: req.user?.id,
    method: req.method,
    path: req.url,
  });
}

// 使用例
app.use((req, res, next) => {
  req.log = createRequestLogger(req);
  req.log.info('Request received');

  res.on('finish', () => {
    req.log.info({
      statusCode: res.statusCode,
      duration: Date.now() - req.startTime,
    }, 'Request completed');
  });

  next();
});

# 可用性SLI：成功リクエスト比率
sum(rate(http_requests_total{status!~"5.."}[30d]))
/
sum(rate(http_requests_total[30d]))

# レイテンシSLI：P99 < 300msのリクエスト比率
sum(rate(http_request_duration_seconds_bucket{le="0.3"}[30d]))
/
sum(rate(http_request_duration_seconds_count[30d]))

# SLO定義 (Sloth形式)
version: "prometheus/v1"
service: "payment-service"
labels:
  team: "platform"
slos:
  - name: "availability"
    objective: 99.9
    sli:
      events:
        error_query: sum(rate(http_requests_total{status=~"5..",service="payment"}[{{.window}}]))
        total_query: sum(rate(http_requests_total{service="payment"}[{{.window}}]))
    alerting:
      page_alert:
        labels:
          severity: critical
      ticket_alert:
        labels:
          severity: warning

# 残りエラーバジェット (%)
1 - (
  (1 - service:availability:ratio30d)
  /
  (1 - 0.999)
)

SLA > SLO > SLI (測定)

例：
- SLA: 99.9% (契約、違反時に返金)
- SLO: 99.95% (内部目標、SLAより厳格)
- SLI: 99.97% (実測値)

          /  P1: Page  \          -> 即時対応 (PagerDuty)
         / (重大、顧客影響) \
        /------------------\
       /   P2: Ticket       \    -> 業務時間内処理 (Jira)
      / (性能低下、潜在リスク) \
     /----------------------\
    /    P3: Notification    \   -> 認知のみ (Slack)
   /  (警告、トレンド変化)      \
  /--------------------------\
 /     P4: Dashboard only     \  -> ダッシュボード確認
/ (参考指標、自動復旧可能)        \

# alertmanager.yml
global:
  resolve_timeout: 5m

route:
  receiver: 'default-slack'
  group_by: ['alertname', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h

  routes:
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      group_wait: 10s
      repeat_interval: 1h

    - match:
        severity: warning
      receiver: 'slack-warnings'
      repeat_interval: 4h

    - match:
        severity: info
      receiver: 'slack-info'
      repeat_interval: 12h

receivers:
  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: 'your-pagerduty-key'
        severity: critical

  - name: 'slack-warnings'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/xxx'
        channel: '#alerts-warning'
        title: '[WARNING] {{ .GroupLabels.alertname }}'

  - name: 'slack-info'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/xxx'
        channel: '#alerts-info'

  - name: 'default-slack'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/xxx'
        channel: '#alerts'

inhibit_rules:
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: ['alertname', 'service']

週次ローテーション例：
- Primary: 第一対応者（5分以内に応答）
- Secondary: バックアップ対応者（Primary未応答時、10分後にエスカレーション）
- マネージャー: 30分以上未解決時にエスカレーション

交代周期: 1週間
ハンドオフ: 毎週月曜日午前10時
報酬: オンコール手当、代替休暇

1. 検知(Detect)
   +-- アラート受信、影響範囲の初期把握

2. 対応(Respond)
   +-- インシデントチャンネル作成、役割割り当て
       - IC (Incident Commander): 調整
       - Tech Lead: 技術調査
       - Comms: 顧客/ステークホルダーへの連絡

3. 緩和(Mitigate)
   +-- 即時対応（ロールバック、スケールアウト等）

4. 解決(Resolve)
   +-- 根本原因の修正、サービス復旧確認

5. 事後分析(Postmortem)
   +-- 非難なき振り返り、再発防止アクションアイテムの導出

+---------------------------------------------+
|                  Grafana                     |
|  (ダッシュボード、アラート、探索)               |
+-------+-------------+------ --------+------+
        |             |               |
   +----v----+  +-----v-----+  +------v---+
   |Prometheus|  |   Loki    |  | Tempo    |
   |(Metrics) |  |  (Logs)   |  |(Traces)  |
   +----^----+  +-----^-----+  +-----^----+
        |             |              |
   +----+-------------+--------------+----+
   |        OpenTelemetry Collector        |
   |  (収集、処理、ルーティング)              |
   +----^-------------^--------------^----+
        |             |              |
   +----+----+  +-----+-----+  +----+----+
   |Service A|  |Service B  |  |Service C|
   |+OTel SDK|  |+OTel SDK  |  |+OTel SDK|
   +---------+  +-----------+  +---------+

# kube-prometheus-stack values.yaml (Helm)
prometheus:
  prometheusSpec:
    retention: 15d
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: gp3
          resources:
            requests:
              storage: 100Gi

grafana:
  dashboardProviders:
    dashboardproviders.yaml:
      apiVersion: 1
      providers:
        - name: 'default'
          folder: ''
          type: file
          options:
            path: /var/lib/grafana/dashboards

alertmanager:
  config:
    route:
      receiver: 'slack'
      group_by: ['alertname', 'namespace']
    receivers:
      - name: 'slack'
        slack_configs:
          - api_url: 'https://hooks.slack.com/services/xxx'
            channel: '#k8s-alerts'