필사 모드: 让日志可搜索，同时别把公司搜穷 — 结构化、映射爆炸、保留期与真实成本

# 差 —— 要搜索就得写正则，格式一变那个正则就废了
log.info(f"order {order_id} for tenant {tenant} failed after {ms}ms: {err}")

# 好 —— 按字段搜索和聚合
log.info(
    "order.failed",
    extra={
        "order.id": order_id,
        "tenant.id": tenant,
        "duration_ms": ms,
        "error.type": type(err).__name__,
        "error.message": str(err)[:500],
        "http.route": route,
    },
)

{
  "@timestamp": "2026-08-02T04:11:52.418Z",
  "severity_text": "ERROR",
  "severity_number": 17,
  "body": "order.failed",
  "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736",
  "span_id": "00f067aa0ba902b7",
  "service.name": "checkout-api",
  "service.version": "2.7.1",
  "deployment.environment.name": "prod",
  "http.route": "/v1/orders/:id",
  "http.response.status_code": 502,
  "error.type": "UpstreamTimeout",
  "order.id": "A-99183",
  "tenant.id": "t-8871",
  "duration_ms": 1421
}

# Python 自动埋点会把 trace_id 和 span_id 注入到日志里
export OTEL_PYTHON_LOG_CORRELATION=true
export OTEL_PYTHON_LOG_LEVEL=info

{
  "message": "cache stats",
  "cache": {
    "user_8871_hits": 12,
    "user_8871_misses": 3,
    "user_9902_hits": 7
  }
}

PUT _index_template/logs-app
{
  "index_patterns": ["logs-app-*"],
  "data_stream": {},
  "priority": 200,
  "template": {
    "settings": {
      "index.number_of_shards": 3,
      "index.number_of_replicas": 1,
      "index.refresh_interval": "30s",
      "index.codec": "zstd_no_dict",
      "index.mapping.total_fields.limit": 1500,
      "index.mapping.depth.limit": 8,
      "index.mapping.nested_fields.limit": 20
    },
    "mappings": {
      "dynamic": "strict",
      "properties": {
        "@timestamp":        { "type": "date" },
        "severity_text":     { "type": "keyword" },
        "body":              { "type": "keyword" },
        "trace_id":          { "type": "keyword" },
        "span_id":           { "type": "keyword" },
        "service.name":      { "type": "keyword" },
        "service.version":   { "type": "keyword" },
        "http.route":        { "type": "keyword" },
        "http.response.status_code": { "type": "short" },
        "error.type":        { "type": "keyword" },
        "error.message":     { "type": "text", "index": true, "norms": false },
        "duration_ms":       { "type": "integer" },
        "tenant.id":         { "type": "keyword" },
        "order.id":          { "type": "keyword", "doc_values": true, "index": true },
        "stack_trace":       { "type": "text", "index": false },
        "attributes":        { "type": "flat_object" }
      }
    }
  }
}

# 每个索引的字段数 —— 到了四位数就已经有问题了
curl -s 'https://opensearch:9200/logs-app-000042/_mapping?pretty' \
  | jq '[paths(type=="object" and has("type")) | length] | length'

# 集群状态大小与映射更新队列
curl -s 'https://opensearch:9200/_cluster/stats?pretty' \
  | jq '.indices.mappings'

curl -s 'https://opensearch:9200/_cluster/health?pretty' \
  | jq '{status, number_of_pending_tasks, task_max_waiting_in_queue_millis}'

# 哪些索引在占用磁盘
curl -s 'https://opensearch:9200/_cat/indices/logs-*?v&s=store.size:desc&h=index,docs.count,store.size,pri,rep' \
  | head -20

PUT _plugins/_ism/policies/logs-app-lifecycle
{
  "policy": {
    "description": "应用日志保留 30 天",
    "default_state": "hot",
    "ism_template": [
      {
        "index_patterns": ["logs-app-*"],
        "priority": 200
      }
    ],
    "states": [
      {
        "name": "hot",
        "actions": [
          {
            "rollover": {
              "min_primary_shard_size": "30gb",
              "min_index_age": "1d"
            }
          }
        ],
        "transitions": [
          { "state_name": "warm", "conditions": { "min_index_age": "3d" } }
        ]
      },
      {
        "name": "warm",
        "actions": [
          { "replica_count": { "number_of_replicas": 0 } },
          { "force_merge": { "max_num_segments": 1 } }
        ],
        "transitions": [
          { "state_name": "cold", "conditions": { "min_index_age": "10d" } }
        ]
      },
      {
        "name": "cold",
        "actions": [
          { "read_only": {} }
        ],
        "transitions": [
          { "state_name": "delete", "conditions": { "min_index_age": "30d" } }
        ]
      },
      {
        "name": "delete",
        "actions": [{ "delete": {} }]
      }
    ]
  }
}

# 已挂载策略的索引及当前状态
curl -s 'https://opensearch:9200/_plugins/_ism/explain/logs-app-*?pretty' \
  | jq 'to_entries[] | select(.value.index != null)
        | {index: .value.index, state: .value."policy_id", step: .value.step.name, failed: .value.failed}'

# 只看失败的受管索引
curl -s 'https://opensearch:9200/_plugins/_ism/explain/logs-app-*?pretty' \
  | jq '[to_entries[] | select(.value.failed == true) | .key]'

# otel-collector.yaml —— 日志流水线
processors:
  memory_limiter:
    check_interval: 1s
    limit_percentage: 80
    spike_limit_percentage: 20

  # 1) 健康检查整个丢弃
  filter/drop_health:
    error_mode: ignore
    logs:
      log_record:
        - 'attributes["http.route"] == "/healthz"'
        - 'attributes["http.route"] == "/readyz"'
        - 'attributes["http.route"] == "/metrics"'

  # 2) 正常路径上的 DEBUG 只留 10%，错误不动
  probabilistic_sampler/debug:
    sampling_percentage: 10
    attribute_source: record
    from_attribute: trace_id

  # 3) 截断过大的字段
  transform/trim:
    error_mode: ignore
    log_statements:
      - context: log
        statements:
          - truncate_all(attributes, 4096)
          - delete_key(attributes, "http.request.body")
          - set(attributes["error.message"], Substring(attributes["error.message"], 0, 500))
            where attributes["error.message"] != nil

  batch:
    timeout: 5s
    send_batch_size: 8192

exporters:
  opensearch:
    http:
      endpoint: https://opensearch.observability.svc:9200
    logs_index: logs-app
    sending_queue:
      enabled: true
      queue_size: 5000
    retry_on_failure:
      enabled: true
      max_elapsed_time: 300s

service:
  pipelines:
    # 错误和警告一律全量保留
    logs/critical:
      receivers: [otlp]
      processors: [memory_limiter, transform/trim, batch]
      exporters: [opensearch]
    # 正常路径做采样
    logs/routine:
      receivers: [otlp]
      processors: [memory_limiter, filter/drop_health, probabilistic_sampler/debug, transform/trim, batch]
      exporters: [opensearch]

# 规模假设
rps            = 5_000          # 每秒请求数
log_bytes      = 800            # 一条日志索引后的大小 (字节)
seconds_of_day = 86_400

daily_gb = rps * log_bytes * seconds_of_day / 1024**3
print(f"每日索引量: {daily_gb:.1f} GB")
# 每日索引量: 321.8 GB

# 同样的信息用指标来表达
routes, status_classes, pods = 120, 5, 40
series = routes * status_classes * pods
samples_per_day = series * (seconds_of_day / 15)      # 15 秒抓取一次
metric_bytes = samples_per_day * 2                    # 压缩后每个样本约 2 字节
print(f"{series:,} 条时间序列，每日 {metric_bytes/1024**3:.3f} GB")
# 24,000 条时间序列，每日 0.257 GB

# 从日志生成计数类指标，发往 Prometheus
connectors:
  count:
    logs:
      log.error.count:
        description: 按严重级别统计的日志发生次数
        conditions:
          - 'severity_number >= 17'
        attributes:
          - key: service.name
          - key: error.type
          - key: http.route

service:
  pipelines:
    logs/in:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [opensearch, count]
    metrics/from_logs:
      receivers: [count]
      processors: [batch]
      exporters: [prometheusremotewrite]

exporters:
  opensearch:
    sending_queue:
      enabled: true
      queue_size: 5000
      # 队列满了就丢弃新数据，不把背压传回应用
      block_on_overflow: false