Grafana Tempo 分散トレーシングと TraceQL 運用ガイド 2026

[Application] --> [OTel Collector] --> [Distributor]
                                           |
                                    [Hash Ring]
                                           |
                                      [Ingester]
                                       /      \
                              [WAL]         [Object Storage]
                                                  |
                              [Compactor] <-------+
                                                  |
                              [Query Frontend] ---+---> [Querier]

# tempo-config.yaml (Monolithic)
server:
  http_listen_port: 3200

distributor:
  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: '0.0.0.0:4317'
        http:
          endpoint: '0.0.0.0:4318'
    jaeger:
      protocols:
        thrift_http:
          endpoint: '0.0.0.0:14268'
    zipkin:
      endpoint: '0.0.0.0:9411'

ingester:
  max_block_duration: 5m
  max_block_bytes: 1073741824 # 1GB

storage:
  trace:
    backend: local
    wal:
      path: /var/tempo/wal
    local:
      path: /var/tempo/blocks
    pool:
      max_workers: 100
      queue_depth: 10000

compactor:
  compaction:
    block_retention: 72h

metrics_generator:
  registry:
    external_labels:
      source: tempo
      cluster: local
  storage:
    path: /var/tempo/generator/wal
    remote_write:
      - url: http://prometheus:9090/api/v1/write
        send_exemplars: true
  traces_storage:
    path: /var/tempo/generator/traces
  processor:
    service_graphs:
      dimensions:
        - service.namespace
        - deployment.environment
    span_metrics:
      dimensions:
        - http.method
        - http.status_code
        - http.route

overrides:
  defaults:
    metrics_generator:
      processors:
        - service-graphs
        - span-metrics

# docker-compose.yaml
version: '3.9'

services:
  tempo:
    image: grafana/tempo:2.7.1
    command: ['-config.file=/etc/tempo/tempo.yaml']
    volumes:
      - ./tempo.yaml:/etc/tempo/tempo.yaml
      - tempo-data:/var/tempo
    ports:
      - '3200:3200' # Tempo HTTP API
      - '4317:4317' # OTLP gRPC
      - '4318:4318' # OTLP HTTP
      - '9411:9411' # Zipkin
      - '14268:14268' # Jaeger HTTP
    networks:
      - observability

  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.118.0
    command: ['--config=/etc/otel-collector/config.yaml']
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector/config.yaml
    ports:
      - '4327:4317' # OTLP gRPC（アプリからのアクセス用）
      - '4328:4318' # OTLP HTTP
    depends_on:
      - tempo
    networks:
      - observability

  prometheus:
    image: prom/prometheus:v3.2.1
    volumes:
      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
    ports:
      - '9090:9090'
    networks:
      - observability

  grafana:
    image: grafana/grafana:11.5.2
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
    volumes:
      - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
    ports:
      - '3000:3000'
    depends_on:
      - tempo
      - prometheus
    networks:
      - observability

volumes:
  tempo-data:

networks:
  observability:
    driver: bridge

// 1. 特定サービスのエラースパンを検索
{ resource.service.name = "payment-service" && status = error }

// 2. 500ms 以上かかった HTTP GET リクエストスパン
{ span.http.method = "GET" && duration > 500ms }

// 3. 特定ルートで 5xx レスポンスを返したスパン
{ span.http.route = "/api/v1/orders" && span.http.status_code >= 500 }

// 4. 2つのサービス間の呼び出し関係を追跡（構造オペレーター）
{ resource.service.name = "api-gateway" } >> { resource.service.name = "order-service" }

// 5. 親子直接関係のスパンフィルター
{ resource.service.name = "frontend" } > { span.http.status_code = 503 }

// 6. 兄弟スパン関係の探索
{ span.db.system = "postgresql" } ~ { span.db.system = "redis" }

// 7. 正規表現を使ったスパン名マッチング
{ name =~ "HTTP.*POST" && resource.deployment.environment = "production" }

// 8. トレース全体の持続時間基準でフィルタリング
{ traceDuration > 3s }

// 9. ルートサービス基準でフィルタリング
{ rootServiceName = "ingress-nginx" && duration > 1s }

// 10. 集計関数を使った分析
{ resource.service.name = "checkout-service" } | rate()

// 11. ヒストグラムでレイテンシー分布を確認
{ resource.service.name = "search-service" } | histogram_over_time(duration)

// 12. カウントベースの異常検知
{ status = error } | count() > 100

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: '0.0.0.0:4317'
      http:
        endpoint: '0.0.0.0:4318'

processors:
  batch:
    timeout: 5s
    send_batch_size: 10000
    send_batch_max_size: 11000

  memory_limiter:
    check_interval: 1s
    limit_mib: 4096
    spike_limit_mib: 512

  attributes:
    actions:
      - key: deployment.environment
        value: production
        action: upsert

  tail_sampling:
    decision_wait: 10s
    num_traces: 100000
    expected_new_traces_per_sec: 1000
    policies:
      - name: errors-policy
        type: status_code
        status_code:
          status_codes:
            - ERROR
      - name: slow-traces-policy
        type: latency
        latency:
          threshold_ms: 1000
      - name: probabilistic-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

exporters:
  otlp/tempo:
    endpoint: 'tempo:4317'
    tls:
      insecure: true
    retry_on_failure:
      enabled: true
      initial_interval: 5s
      max_interval: 30s
      max_elapsed_time: 300s
    sending_queue:
      enabled: true
      num_consumers: 10
      queue_size: 5000

  debug:
    verbosity: basic

service:
  telemetry:
    logs:
      level: info
    metrics:
      address: '0.0.0.0:8888'

  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, tail_sampling, attributes, batch]
      exporters: [otlp/tempo, debug]

storage:
  trace:
    backend: s3
    s3:
      bucket: tempo-traces-prod
      endpoint: s3.ap-northeast-2.amazonaws.com
      region: ap-northeast-2
      access_key: ${S3_ACCESS_KEY}
      secret_key: ${S3_SECRET_KEY}
      # または IAM Role 使用時は access_key/secret_key を省略
    wal:
      path: /var/tempo/wal
    block:
      bloom_filter_false_positive: 0.01
      v2_index_downsample_bytes: 1048576
      v2_encoding: zstd
    blocklist_poll: 5m
    pool:
      max_workers: 200
      queue_depth: 20000

compactor:
  compaction:
    block_retention: 336h # 14日間保持
    compacted_block_retention: 1h
    compaction_window: 4h
    max_block_bytes: 107374182400 # 100GB
    max_compaction_objects: 6000000
    retention_concurrency: 10
  ring:
    kvstore:
      store: memberlist

# grafana-datasources.yaml
apiVersion: 1

datasources:
  - name: Tempo
    type: tempo
    access: proxy
    url: http://tempo:3200
    uid: tempo
    jsonData:
      httpMethod: GET
      tracesToLogsV2:
        datasourceUid: loki
        spanStartTimeShift: '-1h'
        spanEndTimeShift: '1h'
        filterByTraceID: true
        filterBySpanID: true
      tracesToMetrics:
        datasourceUid: prometheus
        spanStartTimeShift: '-1h'
        spanEndTimeShift: '1h'
        tags:
          - key: service.name
            value: service
          - key: http.method
            value: method
      tracesToProfiles:
        datasourceUid: pyroscope
        profileTypeId: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds'
        tags:
          - key: service.name
            value: service_name
      serviceMap:
        datasourceUid: prometheus
      nodeGraph:
        enabled: true
      search:
        hide: false
      traceQuery:
        timeShiftEnabled: true
        spanStartTimeShift: '-30m'
        spanEndTimeShift: '30m'

{
  "panels": [
    {
      "title": "Service Request Rate",
      "type": "timeseries",
      "datasource": { "uid": "prometheus", "type": "prometheus" },
      "targets": [
        {
          "expr": "sum(rate(traces_spanmetrics_calls_total{status_code!=\"STATUS_CODE_ERROR\"}[5m])) by (service)",
          "legendFormat": "{{ service }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "custom": { "drawStyle": "line", "lineWidth": 2 }
        }
      }
    },
    {
      "title": "Service Error Rate",
      "type": "timeseries",
      "datasource": { "uid": "prometheus", "type": "prometheus" },
      "targets": [
        {
          "expr": "sum(rate(traces_spanmetrics_calls_total{status_code=\"STATUS_CODE_ERROR\"}[5m])) by (service) / sum(rate(traces_spanmetrics_calls_total[5m])) by (service) * 100",
          "legendFormat": "{{ service }}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "thresholds": {
            "steps": [
              { "color": "green", "value": null },
              { "color": "yellow", "value": 1 },
              { "color": "red", "value": 5 }
            ]
          }
        }
      }
    },
    {
      "title": "P99 Latency by Service",
      "type": "timeseries",
      "datasource": { "uid": "prometheus", "type": "prometheus" },
      "targets": [
        {
          "expr": "histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket[5m])) by (le, service))",
          "legendFormat": "{{ service }}"
        }
      ],
      "fieldConfig": {
        "defaults": { "unit": "s" }
      }
    }
  ]
}