Elasticsearch完全ガイド2025：検索エンジンからログ分析、ベクトル検索まで

ES 1.x (2014) - 初期安定化
ES 2.x (2015) - Pipeline Aggregation導入
ES 5.x (2016) - Lucene 6、Painlessスクリプティング
ES 6.x (2017) - インデックスごとに単一タイプ
ES 7.x (2019) - タイプ廃止、アダプティブレプリカ選択
ES 8.x (2022) - セキュリティデフォルト有効化、kNNベクトル検索、NLP統合

Doc 1: "The quick brown fox"
Doc 2: "The quick brown dog"
Doc 3: "The lazy brown fox"

Term      | Document IDs
----------|-------------
the       | [1, 2, 3]
quick     | [1, 2]
brown     | [1, 2, 3]
fox       | [1, 3]
dog       | [2]
lazy      | [3]

Index
  └── Shard (Lucene Index)
        ├── Segment 0 (immutable)
        │     ├── Inverted Index
        │     ├── Stored Fields
        │     ├── Doc Values
        │     └── Term Vectors
        ├── Segment 1 (immutable)
        ├── Segment 2 (immutable)
        └── Commit Point (segments_N)

転置インデックス（検索用）: Term → Document IDs
Doc Values（集約/ソート用）: Document ID → Values

PUT /products
{
  "mappings": {
    "properties": {
      "name": {
        "type": "text",
        "analyzer": "standard",
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        }
      },
      "price": {
        "type": "float"
      },
      "category": {
        "type": "keyword"
      },
      "description": {
        "type": "text"
      },
      "created_at": {
        "type": "date",
        "format": "yyyy-MM-dd HH:mm:ss||epoch_millis"
      },
      "location": {
        "type": "geo_point"
      },
      "tags": {
        "type": "keyword"
      },
      "reviews": {
        "type": "nested",
        "properties": {
          "author": { "type": "keyword" },
          "rating": { "type": "integer" },
          "comment": { "type": "text" }
        }
      }
    }
  }
}

テキスト入力
  │
  ▼
Character Filter（文字フィルタ）
  │  - html_strip: HTMLタグ除去
  │  - mapping: 文字置換
  │  - pattern_replace: 正規表現置換
  ▼
Tokenizer（トークナイザ）
  │  - standard: Unicode単語境界基盤
  │  - whitespace: 空白基準分割
  │  - ngram: N-gramトークン生成
  │  - edge_ngram: プレフィックスベーストークン
  │  - kuromoji_tokenizer: 日本語形態素解析
  ▼
Token Filter（トークンフィルタ）
  │  - lowercase: 小文字変換
  │  - stop: ストップワード除去
  │  - synonym: 同義語処理
  │  - stemmer: ステミング
  │  - kuromoji_part_of_speech: 日本語品詞フィルタ
  ▼
トークンストリーム（Term）

PUT /japanese_index
{
  "settings": {
    "analysis": {
      "tokenizer": {
        "kuromoji_custom": {
          "type": "kuromoji_tokenizer",
          "mode": "search",
          "discard_punctuation": true,
          "user_dictionary_rules": [
            "エラスティックサーチ,エラスティックサーチ,エラスティックサーチ,カスタム名詞"
          ]
        }
      },
      "filter": {
        "kuromoji_pos_filter": {
          "type": "kuromoji_part_of_speech",
          "stoptags": [
            "助詞-格助詞",
            "助詞-係助詞",
            "助詞-終助詞"
          ]
        }
      },
      "analyzer": {
        "japanese_analyzer": {
          "type": "custom",
          "tokenizer": "kuromoji_custom",
          "filter": [
            "kuromoji_baseform",
            "kuromoji_pos_filter",
            "lowercase",
            "cjk_width"
          ]
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "title": {
        "type": "text",
        "analyzer": "japanese_analyzer"
      }
    }
  }
}

POST /japanese_index/_analyze
{
  "analyzer": "japanese_analyzer",
  "text": "Elasticsearchは強力な検索エンジンです"
}

// 結果
{
  "tokens": [
    { "token": "elasticsearch", "position": 0 },
    { "token": "強力", "position": 2 },
    { "token": "検索", "position": 4 },
    { "token": "エンジン", "position": 5 }
  ]
}

// 基本match - アナライザでトークン化後に検索
GET /products/_search
{
  "query": {
    "match": {
      "description": "強力な検索エンジン"
    }
  }
}

// match_phrase - 単語の順序まで一致
GET /products/_search
{
  "query": {
    "match_phrase": {
      "description": {
        "query": "検索エンジン",
        "slop": 1
      }
    }
  }
}

// multi_match - 複数フィールドで検索
GET /products/_search
{
  "query": {
    "multi_match": {
      "query": "Elasticsearchガイド",
      "fields": ["title^3", "description", "tags^2"],
      "type": "best_fields"
    }
  }
}

// term - keywordフィールドの完全一致
GET /products/_search
{
  "query": {
    "term": {
      "category": "electronics"
    }
  }
}

// terms - 複数値のいずれかに一致
GET /products/_search
{
  "query": {
    "terms": {
      "status": ["published", "pending"]
    }
  }
}

// range - 範囲検索
GET /products/_search
{
  "query": {
    "range": {
      "price": {
        "gte": 100,
        "lte": 500
      }
    }
  }
}

// exists - フィールド存在確認
GET /products/_search
{
  "query": {
    "exists": {
      "field": "discount"
    }
  }
}

GET /products/_search
{
  "query": {
    "bool": {
      "must": [
        { "match": { "description": "検索エンジン" } }
      ],
      "must_not": [
        { "term": { "status": "deleted" } }
      ],
      "should": [
        { "term": { "featured": true } },
        { "range": { "rating": { "gte": 4.5 } } }
      ],
      "filter": [
        { "term": { "category": "software" } },
        { "range": { "price": { "lte": 1000 } } }
      ],
      "minimum_should_match": 1
    }
  }
}

GET /products/_search
{
  "query": {
    "nested": {
      "path": "reviews",
      "query": {
        "bool": {
          "must": [
            { "term": { "reviews.author": "john" } },
            { "range": { "reviews.rating": { "gte": 4 } } }
          ]
        }
      },
      "inner_hits": {
        "size": 3,
        "highlight": {
          "fields": {
            "reviews.comment": {}
          }
        }
      }
    }
  }
}

GET /products/_search
{
  "query": {
    "function_score": {
      "query": { "match": { "title": "elasticsearch" } },
      "functions": [
        {
          "field_value_factor": {
            "field": "popularity",
            "modifier": "log1p",
            "factor": 2
          }
        },
        {
          "gauss": {
            "created_at": {
              "origin": "now",
              "scale": "30d",
              "decay": 0.5
            }
          }
        },
        {
          "filter": { "term": { "featured": true } },
          "weight": 5
        }
      ],
      "score_mode": "sum",
      "boost_mode": "multiply"
    }
  }
}

// terms集約 - カテゴリ別文書数
GET /products/_search
{
  "size": 0,
  "aggs": {
    "by_category": {
      "terms": {
        "field": "category",
        "size": 20,
        "order": { "_count": "desc" }
      }
    }
  }
}

// date_histogram - 時間帯別集約
GET /logs/_search
{
  "size": 0,
  "aggs": {
    "logs_per_hour": {
      "date_histogram": {
        "field": "timestamp",
        "calendar_interval": "1h",
        "time_zone": "Asia/Tokyo",
        "min_doc_count": 0
      }
    }
  }
}

// ネスト集約 - カテゴリ別平均価格
GET /products/_search
{
  "size": 0,
  "aggs": {
    "by_category": {
      "terms": { "field": "category" },
      "aggs": {
        "avg_price": {
          "avg": { "field": "price" }
        },
        "price_stats": {
          "stats": { "field": "price" }
        }
      }
    }
  }
}

GET /products/_search
{
  "size": 0,
  "aggs": {
    "avg_price": { "avg": { "field": "price" } },
    "max_price": { "max": { "field": "price" } },
    "min_price": { "min": { "field": "price" } },
    "total_sales": { "sum": { "field": "sales_count" } },
    "unique_brands": { "cardinality": { "field": "brand" } },
    "price_percentiles": {
      "percentiles": {
        "field": "price",
        "percents": [25, 50, 75, 90, 99]
      }
    }
  }
}

GET /sales/_search
{
  "size": 0,
  "aggs": {
    "monthly_sales": {
      "date_histogram": {
        "field": "date",
        "calendar_interval": "month"
      },
      "aggs": {
        "total_revenue": {
          "sum": { "field": "revenue" }
        }
      }
    },
    "avg_monthly_revenue": {
      "avg_bucket": {
        "buckets_path": "monthly_sales>total_revenue"
      }
    },
    "max_monthly_revenue": {
      "max_bucket": {
        "buckets_path": "monthly_sales>total_revenue"
      }
    }
  }
}

データソース              収集               処理/変換            格納              可視化
┌──────────┐          ┌──────────┐      ┌──────────┐      ┌──────────┐      ┌──────────┐
│  App Log │──────────│ Filebeat │──────│ Logstash │──────│  Elastic │──────│  Kibana  │
│  Server  │          │ Metricbt │      │ Pipeline │      │  search  │      │Dashboard │
│  Docker  │          │ Heartbt  │      │ (Filter) │      │ Cluster  │      │ Lens/Map │
│  K8s     │          │ Packetbt │      │          │      │          │      │ Alerting │
└──────────┘          └──────────┘      └──────────┘      └──────────┘      └──────────┘

# /etc/logstash/conf.d/main.conf
input {
  beats {
    port => 5044
  }
  kafka {
    bootstrap_servers => "kafka:9092"
    topics => ["app-logs"]
    codec => json
  }
}

filter {
  # Grokパターンで非構造化ログをパース
  grok {
    match => {
      "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:msg}"
    }
  }

  # 日付パース
  date {
    match => ["timestamp", "ISO8601"]
    target => "@timestamp"
  }

  # GeoIP変換
  geoip {
    source => "client_ip"
    target => "geo"
  }

  # 条件付き処理
  if [level] == "ERROR" {
    mutate {
      add_tag => ["alert"]
      add_field => { "severity" => "high" }
    }
  }
}

output {
  elasticsearch {
    hosts => ["http://elasticsearch:9200"]
    index => "logs-%{+YYYY.MM.dd}"
    user => "elastic"
    password => "changeme"
  }
}

# /etc/filebeat/filebeat.yml
filebeat.inputs:
  - type: log
    enabled: true
    paths:
      - /var/log/app/*.log
    multiline:
      pattern: '^\d{4}-\d{2}-\d{2}'
      negate: true
      match: after

  - type: container
    paths:
      - /var/lib/docker/containers/*/*.log
    processors:
      - add_docker_metadata: ~

output.logstash:
  hosts: ["logstash:5044"]

"自動車" → [0.12, -0.34, 0.56, ..., 0.89]  (768次元)
"車両"   → [0.13, -0.32, 0.55, ..., 0.88]  (類似ベクトル)
"果物"   → [-0.45, 0.67, -0.12, ..., 0.23] (異なるベクトル)

PUT /semantic-search
{
  "mappings": {
    "properties": {
      "title": { "type": "text" },
      "content": { "type": "text" },
      "content_vector": {
        "type": "dense_vector",
        "dims": 768,
        "index": true,
        "similarity": "cosine",
        "index_options": {
          "type": "hnsw",
          "m": 16,
          "ef_construction": 200
        }
      }
    }
  }
}

// 純粋kNN検索
GET /semantic-search/_search
{
  "knn": {
    "field": "content_vector",
    "query_vector": [0.12, -0.34, 0.56],
    "k": 10,
    "num_candidates": 100
  }
}

// ハイブリッド検索（キーワード + kNN）
GET /semantic-search/_search
{
  "query": {
    "match": {
      "content": "自動車 おすすめ"
    }
  },
  "knn": {
    "field": "content_vector",
    "query_vector": [0.12, -0.34, 0.56],
    "k": 10,
    "num_candidates": 100,
    "boost": 0.5
  },
  "size": 10
}

Layer 2 (sparse):  A ──── B
                   │
Layer 1 (medium):  A ── C ── B ── D
                   │    │    │    │
Layer 0 (dense):   A-E-C-F-B-G-D-H

┌─────────────────────────────────────────────────┐
│                  Elasticsearch Cluster            │
│                                                   │
│  ┌─────────────┐  ┌─────────────┐  ┌────────────┐│
│  │ Master Node │  │ Master Node │  │Master Node ││
│  │  (Elected)  │  │ (Eligible)  │  │(Eligible)  ││
│  └─────────────┘  └─────────────┘  └────────────┘│
│                                                   │
│  ┌─────────────┐  ┌─────────────┐  ┌────────────┐│
│  │  Data Node  │  │  Data Node  │  │ Data Node  ││
│  │ (Hot Tier)  │  │ (Hot Tier)  │  │(Warm Tier) ││
│  │  SSD 1TB    │  │  SSD 1TB    │  │ HDD 4TB    ││
│  └─────────────┘  └─────────────┘  └────────────┘│
│                                                   │
│  ┌─────────────┐  ┌─────────────┐                │
│  │ Ingest Node │  │ Coord Node  │                │
│  │ (Pipeline)  │  │ (Routing)   │                │
│  └─────────────┘  └─────────────┘                │
└─────────────────────────────────────────────────┘

PUT /logs-2025.03
{
  "settings": {
    "number_of_shards": 3,
    "number_of_replicas": 1,
    "routing.allocation.require.data": "hot"
  }
}

PUT /_ilm/policy/logs-policy
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_primary_shard_size": "50gb",
            "max_age": "1d"
          },
          "set_priority": { "priority": 100 }
        }
      },
      "warm": {
        "min_age": "7d",
        "actions": {
          "shrink": { "number_of_shards": 1 },
          "forcemerge": { "max_num_segments": 1 },
          "allocate": { "require": { "data": "warm" } },
          "set_priority": { "priority": 50 }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "allocate": { "require": { "data": "cold" } },
          "freeze": {}
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": { "delete": {} }
      }
    }
  }
}

# クラスタ状態
GET /_cluster/health

# ノード統計
GET /_nodes/stats

# インデックス状態
GET /_cat/indices?v&s=store.size:desc

# シャード割り当て状態
GET /_cat/shards?v&s=store:desc

# 割り当て失敗原因
GET /_cluster/allocation/explain

// Bulk API（個別リクエストの10倍以上高速）
POST /_bulk
{"index": {"_index": "products", "_id": "1"}}
{"name": "Product 1", "price": 100}
{"index": {"_index": "products", "_id": "2"}}
{"name": "Product 2", "price": 200}

// 1. Filter Context活用（キャッシュされる）
GET /products/_search
{
  "query": {
    "bool": {
      "filter": [
        { "term": { "category": "electronics" } },
        { "range": { "price": { "gte": 100, "lte": 500 } } }
      ],
      "must": [
        { "match": { "description": "wireless" } }
      ]
    }
  }
}

// 2. Source Filtering（必要なフィールドのみ）
GET /products/_search
{
  "_source": ["name", "price", "category"],
  "query": { "match_all": {} }
}

// 3. Search After（Deep Paginationの代替）
GET /products/_search
{
  "size": 20,
  "sort": [
    { "created_at": "desc" },
    { "_id": "asc" }
  ],
  "search_after": ["2025-03-01T00:00:00", "abc123"]
}

Elasticsearchキャッシュ階層：

1. Node Query Cache（Filter Cache）
   - filter contextの結果をキャッシュ
   - ノードレベル、ヒープの10%（デフォルト）
   - LRU方式除去

2. Shard Request Cache
   - 集約結果をキャッシュ
   - シャードレベル、ヒープの1%（デフォルト）
   - インデックスrefresh時に無効化

3. OS Page Cache
   - Luceneセグメントファイルのキャッシュ
   - ヒープ外メモリを活用
   - 最も重要なキャッシュ！

// Component Template（再利用可能なブロック）
PUT /_component_template/base-settings
{
  "template": {
    "settings": {
      "number_of_shards": 3,
      "number_of_replicas": 1,
      "index.lifecycle.name": "logs-policy"
    }
  }
}

PUT /_component_template/log-mappings
{
  "template": {
    "mappings": {
      "properties": {
        "@timestamp": { "type": "date" },
        "message": { "type": "text" },
        "level": { "type": "keyword" },
        "service": { "type": "keyword" },
        "trace_id": { "type": "keyword" }
      }
    }
  }
}

// Index Template
PUT /_index_template/logs
{
  "index_patterns": ["logs-*"],
  "composed_of": ["base-settings", "log-mappings"],
  "priority": 200
}

// Alias設定（無停止インデックス切り替え）
POST /_aliases
{
  "actions": [
    { "remove": { "index": "products-v1", "alias": "products" } },
    { "add": { "index": "products-v2", "alias": "products" } }
  ]
}

// Reindex（インデックスマイグレーション）
POST /_reindex
{
  "source": {
    "index": "old-index",
    "query": {
      "range": {
        "@timestamp": { "gte": "2025-01-01" }
      }
    }
  },
  "dest": {
    "index": "new-index",
    "pipeline": "enrichment-pipeline"
  }
}

// リポジトリ登録（S3）
PUT /_snapshot/s3-backup
{
  "type": "s3",
  "settings": {
    "bucket": "my-es-backups",
    "region": "ap-northeast-1",
    "base_path": "elasticsearch"
  }
}

// スナップショット作成
PUT /_snapshot/s3-backup/snapshot-2025-03-24
{
  "indices": "logs-*,products",
  "ignore_unavailable": true,
  "include_global_state": false
}

// 復元
POST /_snapshot/s3-backup/snapshot-2025-03-24/_restore
{
  "indices": "products",
  "rename_pattern": "(.+)",
  "rename_replacement": "restored-$1"
}