Split View: OpenSearch 운영·관리·인덱스 설계 실전 가이드 2026

+------------------------------------------------------------------+
|                      Client / Data Prepper                        |
|                        (Ingest Pipeline)                          |
+------------------------------+-----------------------------------+
                               |
          +--------------------+--------------------+
          v                    v                    v
   +------------+       +------------+       +------------+
   | Master-1   |       | Master-2   |       | Master-3   |
   | (AZ-a)     |       | (AZ-b)     |       | (AZ-c)     |
   | Dedicated  |       | Dedicated  |       | Dedicated  |
   +------------+       +------------+       +------------+
          |                    |                    |
          v                    v                    v
   +------------+       +------------+       +------------+
   | Hot-1      |       | Hot-2      |       | Hot-3      |
   | (AZ-a)     |       | (AZ-b)     |       | (AZ-c)     |
   | EBS gp3    |       | EBS gp3    |       | EBS gp3    |
   +------------+       +------------+       +------------+
          |                    |                    |
          v                    v                    v
   +------------+       +------------+       +------------+
   | Warm-1     |       | Warm-2     |       | Warm-3     |
   | UltraWarm  |       | UltraWarm  |       | UltraWarm  |
   | (S3+cache) |       | (S3+cache) |       | (S3+cache) |
   +------------+       +------------+       +------------+

Required Storage = Daily Raw Data
                 x (1 + Number of Replicas)
                 x 1.1 (Indexing Overhead)
                 x 1.15 (OS Reserved + Internal Operations Margin)
                 x Retention Period (days)

Actual Provisioning = Required Storage / 0.75 = 10,120 GiB

Primary Shard Count = (Raw Data Size + Growth Margin) x 1.1 / Target Shard Size

(66 + 198) x 1.1 / 30 = approx. 10 primary shards

PUT _index_template/app-logs-template
{
  "index_patterns": ["app-logs-*"],
  "template": {
    "settings": {
      "number_of_shards": 3,
      "number_of_replicas": 1,
      "refresh_interval": "30s",
      "index.translog.flush_threshold_size": "1024mb",
      "index.codec": "zstd_no_dict",
      "plugins.index_state_management.rollover_alias": "app-logs-write"
    },
    "mappings": {
      "dynamic": "strict",
      "properties": {
        "@timestamp":  { "type": "date" },
        "level":       { "type": "keyword" },
        "message":     { "type": "text", "analyzer": "standard" },
        "service":     { "type": "keyword" },
        "trace_id":    { "type": "keyword" },
        "span_id":     { "type": "keyword" },
        "host":        { "type": "keyword" },
        "duration_ms": { "type": "float" },
        "http_status": { "type": "short" },
        "request_path":{ "type": "keyword" },
        "user_id":     { "type": "keyword" },
        "metadata":    {
          "type": "object",
          "enabled": false
        }
      }
    }
  },
  "composed_of": ["common-settings"],
  "priority": 200
}

// Before indexing: disable replicas
PUT my-bulk-index/_settings
{ "index.number_of_replicas": 0 }

// After indexing complete: restore replicas
PUT my-bulk-index/_settings
{ "index.number_of_replicas": 1 }

PUT _plugins/_ism/policies/app-log-lifecycle
{
  "policy": {
    "description": "App log index lifecycle: hot -> warm -> cold -> delete",
    "default_state": "hot",
    "states": [
      {
        "name": "hot",
        "actions": [
          {
            "rollover": {
              "min_size": "30gb",
              "min_index_age": "1d",
              "min_doc_count": 10000000
            }
          }
        ],
        "transitions": [
          { "state_name": "warm", "conditions": { "min_index_age": "3d" } }
        ]
      },
      {
        "name": "warm",
        "actions": [
          { "replica_count": { "number_of_replicas": 1 } },
          { "force_merge": { "max_num_segments": 1 } },
          { "allocation": {
              "require": { "temp": "warm" },
              "wait_for": true
            }
          }
        ],
        "transitions": [
          { "state_name": "cold", "conditions": { "min_index_age": "30d" } }
        ]
      },
      {
        "name": "cold",
        "actions": [
          { "replica_count": { "number_of_replicas": 0 } },
          { "read_only": {} }
        ],
        "transitions": [
          { "state_name": "delete", "conditions": { "min_index_age": "90d" } }
        ]
      },
      {
        "name": "delete",
        "actions": [
          {
            "notification": {
              "destination": {
                "slack": { "url": "https://hooks.slack.com/services/T.../B.../xxx" }
              },
              "message_template": {
                "source": "Index {{ctx.index}} will be deleted according to retention policy (90 days)."
              }
            }
          },
          { "delete": {} }
        ],
        "transitions": []
      }
    ],
    "ism_template": [
      { "index_patterns": ["app-logs-*"], "priority": 100 }
    ]
  }
}

# Check policy execution status (specific index)
GET _plugins/_ism/explain/app-logs-000001

# Attach policy to existing indexes
POST _plugins/_ism/add/old-logs-2025-*
{ "policy_id": "app-log-lifecycle" }

# Update policy version (apply new policy to managed indexes)
POST _plugins/_ism/change_policy/app-logs-*
{
  "policy_id": "app-log-lifecycle-v2",
  "state": "warm"
}

# Retry failed ISM operations
POST _plugins/_ism/retry/app-logs-000003

# Remove ISM policy from a specific index
POST _plugins/_ism/remove/app-logs-000005

// Step 1: Create initial index + alias setup
PUT app-logs-000001
{
  "aliases": {
    "app-logs-write": { "is_write_index": true },
    "app-logs-read": {}
  }
}

// Step 2: ISM policy's rollover action handles automatically
//   -> Creates app-logs-000002
//   -> Moves app-logs-write alias to 000002
//   -> Sets 000001's is_write_index to false

// Reads always use the read alias (targets all history)
GET app-logs-read/_search
{
  "query": {
    "bool": {
      "must": [
        { "match": { "level": "ERROR" } },
        { "range": { "@timestamp": { "gte": "now-1h" } } }
      ]
    }
  },
  "sort": [{ "@timestamp": "desc" }],
  "size": 100
}

// Writes always use the write alias
POST app-logs-write/_doc
{
  "@timestamp": "2026-03-04T10:00:00Z",
  "level": "ERROR",
  "service": "payment-api",
  "message": "Payment gateway timeout occurred",
  "trace_id": "abc123",
  "duration_ms": 30500,
  "http_status": 504
}

# Check current write index
GET _alias/app-logs-write

# Check full alias structure
GET _cat/aliases/app-logs-*?v&h=alias,index,is_write_index

# Overall cluster status (green/yellow/red)
GET _cluster/health

# Per-index status check
GET _cluster/health?level=indices

# Per-node JVM, CPU, disk details
GET _nodes/stats/jvm,os,fs

# Hot threads check (for high CPU root cause analysis)
GET _nodes/hot_threads

# Thread pool status (check pending and rejected counts)
GET _cat/thread_pool?v&h=node_name,name,active,queue,rejected

# Shard allocation status
GET _cat/shards?v&h=index,shard,prirep,state,docs,store,node&s=state

# Unassigned shard root cause analysis
GET _cluster/allocation/explain
{
  "index": "app-logs-000003",
  "shard": 0,
  "primary": true
}

# Per-index size and document count
GET _cat/indices?v&h=index,health,pri,rep,docs.count,store.size&s=store.size:desc

# Per-node disk usage
GET _cat/allocation?v&h=node,shards,disk.used,disk.avail,disk.percent

# Check pending tasks
GET _cat/pending_tasks?v

PUT app-logs-*/_settings
{
  "index.search.slowlog.threshold.query.warn": "10s",
  "index.search.slowlog.threshold.query.info": "5s",
  "index.search.slowlog.threshold.fetch.warn": "5s",
  "index.indexing.slowlog.threshold.index.warn": "10s",
  "index.indexing.slowlog.threshold.index.info": "5s"
}

POST _plugins/_alerting/monitors
{
  "type": "monitor",
  "name": "Cluster Red Status Alert",
  "monitor_type": "query_level_monitor",
  "enabled": true,
  "schedule": {
    "period": { "interval": 1, "unit": "MINUTES" }
  },
  "inputs": [
    {
      "search": {
        "indices": [".opensearch-sap-log-types-config"],
        "query": {
          "size": 0,
          "query": {
            "match_all": {}
          }
        }
      }
    }
  ],
  "triggers": [
    {
      "query_level_trigger": {
        "name": "Cluster Status Red",
        "severity": "1",
        "condition": {
          "script": {
            "source": "ctx.results[0].hits.total.value >= 0",
            "lang": "painless"
          }
        },
        "actions": [
          {
            "name": "Notify Slack",
            "destination_id": "slack-destination-id",
            "message_template": {
              "source": "Cluster status is RED.\nCluster: {{ctx.monitor.name}}\nTime: {{ctx.periodEnd}}\nImmediate investigation required."
            },
            "throttle_enabled": true,
            "throttle": { "value": 10, "unit": "MINUTES" }
          }
        ]
      }
    }
  ]
}

# 1. Identify the largest indexes
GET _cat/indices?v&h=index,store.size&s=store.size:desc&format=json

# 2. Delete old unnecessary indexes
DELETE old-logs-2024-*

# 3. Remove read-only block (all indexes)
PUT _all/_settings
{
  "index.blocks.read_only_allow_delete": null
}

# 4. Remove cluster-level block
PUT _cluster/settings
{
  "persistent": {
    "cluster.blocks.read_only": false
  }
}

# 5. Check cluster status
GET _cluster/health

// Register S3 repository
PUT _snapshot/s3-backup
{
  "type": "s3",
  "settings": {
    "bucket": "my-opensearch-snapshots",
    "base_path": "daily",
    "region": "ap-northeast-2",
    "server_side_encryption": true,
    "role_arn": "arn:aws:iam::123456789012:role/OpenSearchSnapshotRole"
  }
}

// Create manual snapshot
PUT _snapshot/s3-backup/snapshot-2026-03-04
{
  "indices": "app-logs-*",
  "ignore_unavailable": true,
  "include_global_state": false
}

// Check snapshot status
GET _snapshot/s3-backup/snapshot-2026-03-04/_status

// Restore specific indexes only (rename to avoid conflicts with existing indexes)
POST _snapshot/s3-backup/snapshot-2026-03-04/_restore
{
  "indices": "app-logs-000001,app-logs-000002",
  "ignore_unavailable": true,
  "include_global_state": false,
  "rename_pattern": "(.+)",
  "rename_replacement": "$1_restored",
  "index_settings": {
    "index.number_of_replicas": 0
  }
}

// Restore replicas after restoration is complete
PUT app-logs-000001_restored/_settings
{ "index.number_of_replicas": 1 }

POST _reindex
{
  "source": {
    "remote": {
      "host": "https://old-es-cluster:9200",
      "username": "admin",
      "password": "********",
      "socket_timeout": "60m",
      "connect_timeout": "30s"
    },
    "index": "app-logs-2025-*",
    "size": 5000,
    "query": {
      "range": {
        "@timestamp": {
          "gte": "2025-01-01",
          "lt": "2026-01-01"
        }
      }
    }
  },
  "dest": {
    "index": "app-logs-migrated"
  },
  "conflicts": "proceed"
}

from opensearchpy import OpenSearch, helpers

client = OpenSearch(
    hosts=[{"host": "opensearch-node", "port": 9200}],
    http_auth=("admin", "admin"),
    use_ssl=True,
    verify_certs=False,
)

def generate_actions(file_path):
    """Reads a log file and converts to _bulk actions"""
    import json
    with open(file_path) as f:
        for line in f:
            doc = json.loads(line)
            yield {
                "_index": "app-logs-write",
                "_source": doc,
            }

success, errors = helpers.bulk(
    client,
    generate_actions("/var/log/app/events.jsonl"),
    chunk_size=5000,
    max_retries=3,
    request_timeout=60,
)
print(f"Indexed: {success}, Errors: {len(errors)}")