Cell-Based Architecture Design and Operation: Strategy to Minimize Failure Explosion Radius

전통적 아키텍처 (공유 인프라):
┌─────────────────────────────────────────────┐
│              Load Balancer                  │
├─────────────────────────────────────────────┤
│  App Server 1 | App Server 2 | App Server 3│  <-- 공유 컴퓨팅
├─────────────────────────────────────────────┤
│              Shared Database                │  <-- 단일 장애점
├─────────────────────────────────────────────┤
│              Shared Cache                   │  <-- 단일 장애점
└─────────────────────────────────────────────┘
  장애 시: 전체 사용자 영향 (Blast Radius = 100%)

셀 기반 아키텍처 (격리된 인프라):
┌──────────────┐
│  Cell Router │  <-- 유일한 공유 컴포넌트 (최소화)
└──────┬───────┘
       │
  ┌────┴────┬────────┬────────┐
  ▼         ▼        ▼        ▼
┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐
│Cell 1│ │Cell 2│ │Cell 3│ │Cell 4│
│ App  │ │ App  │ │ App  │ │ App  │
│ DB   │ │ DB   │ │ DB   │ │ DB   │
│Cache │ │Cache │ │Cache │ │Cache │
└──────┘ └──────┘ └──────┘ └──────┘
  장애 시: 해당 셀 사용자만 영향 (Blast Radius = 25%)

단일 셀 장애 시 영향받는 사용자 비율 = 1 / N (N = 셀 수)

예시:
- 셀 4개: 장애 시 최대 25% 사용자 영향
- 셀 10개: 장애 시 최대 10% 사용자 영향
- 셀 20개: 장애 시 최대 5% 사용자 영향

# cell_router.py - Consistent Hashing 기반 셀 라우터

import hashlib
from bisect import bisect_right
from typing import Optional


class CellRouter:
    """Consistent Hashing 기반 셀 라우터.

    가상 노드(virtual node)를 사용하여 해시 링 위에
    셀을 균등하게 분산 배치한다.
    """

    def __init__(self, virtual_nodes: int = 150):
        self.virtual_nodes = virtual_nodes
        self.ring: list[int] = []
        self.ring_to_cell: dict[int, str] = {}
        self.cells: dict[str, dict] = {}

    def _hash(self, key: str) -> int:
        """SHA-256 해시로 키를 정수 값으로 변환한다."""
        digest = hashlib.sha256(key.encode()).hexdigest()
        return int(digest[:16], 16)

    def add_cell(self, cell_id: str, metadata: Optional[dict] = None) -> None:
        """셀을 해시 링에 추가한다.

        Args:
            cell_id: 셀 식별자 (예: "cell-us-east-001")
            metadata: 셀 메타데이터 (endpoint, capacity 등)
        """
        self.cells[cell_id] = metadata or {}
        for i in range(self.virtual_nodes):
            virtual_key = f"{cell_id}:vn{i}"
            hash_value = self._hash(virtual_key)
            self.ring.append(hash_value)
            self.ring_to_cell[hash_value] = cell_id
        self.ring.sort()

    def remove_cell(self, cell_id: str) -> None:
        """셀을 해시 링에서 제거한다.

        제거된 셀의 트래픽은 인접 셀로 자동 재배치된다.
        """
        self.ring = [
            h for h in self.ring
            if self.ring_to_cell.get(h) != cell_id
        ]
        self.ring_to_cell = {
            h: c for h, c in self.ring_to_cell.items()
            if c != cell_id
        }
        del self.cells[cell_id]

    def route(self, partition_key: str) -> str:
        """파티션 키로 대상 셀을 결정한다.

        Args:
            partition_key: 라우팅 키 (예: tenant_id, user_id, org_id)

        Returns:
            대상 셀 ID
        """
        if not self.ring:
            raise ValueError("해시 링에 셀이 없습니다")

        hash_value = self._hash(partition_key)
        idx = bisect_right(self.ring, hash_value)

        if idx == len(self.ring):
            idx = 0

        return self.ring_to_cell[self.ring[idx]]

    def get_cell_distribution(self) -> dict[str, int]:
        """각 셀이 해시 링에서 차지하는 비율을 반환한다."""
        distribution: dict[str, int] = {}
        for cell_id in self.ring_to_cell.values():
            distribution[cell_id] = distribution.get(cell_id, 0) + 1
        return distribution


# 사용 예시
if __name__ == "__main__":
    router = CellRouter(virtual_nodes=150)

    # 셀 등록
    router.add_cell("cell-001", {"region": "us-east-1", "capacity": 10000})
    router.add_cell("cell-002", {"region": "us-east-1", "capacity": 10000})
    router.add_cell("cell-003", {"region": "us-west-2", "capacity": 10000})
    router.add_cell("cell-004", {"region": "eu-west-1", "capacity": 10000})

    # 사용자 라우팅
    user_ids = ["user-12345", "user-67890", "org-acme", "org-globex"]
    for uid in user_ids:
        target_cell = router.route(uid)
        print(f"{uid} -> {target_cell}")

    # 셀 분포 확인
    dist = router.get_cell_distribution()
    total = sum(dist.values())
    for cell_id, count in sorted(dist.items()):
        pct = (count / total) * 100
        print(f"{cell_id}: {pct:.1f}%")

# cell-deployment.yaml
# 각 셀은 독립적인 네임스페이스에 배포된다

apiVersion: v1
kind: Namespace
metadata:
  name: cell-001
  labels:
    cell-id: 'cell-001'
    region: 'us-east-1'
    tier: 'standard'
---
# 셀 전용 리소스 제한 (noisy neighbor 방지)
apiVersion: v1
kind: ResourceQuota
metadata:
  name: cell-001-quota
  namespace: cell-001
spec:
  hard:
    requests.cpu: '16'
    requests.memory: '32Gi'
    limits.cpu: '32'
    limits.memory: '64Gi'
    pods: '100'
    services: '20'
    persistentvolumeclaims: '10'
---
# 셀 애플리케이션 배포
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cell-app
  namespace: cell-001
  labels:
    app: cell-app
    cell-id: 'cell-001'
spec:
  replicas: 3
  selector:
    matchLabels:
      app: cell-app
      cell-id: 'cell-001'
  template:
    metadata:
      labels:
        app: cell-app
        cell-id: 'cell-001'
    spec:
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app: cell-app
              cell-id: 'cell-001'
      containers:
        - name: app
          image: myregistry/cell-app:v2.4.1
          ports:
            - containerPort: 8080
          env:
            - name: CELL_ID
              value: 'cell-001'
            - name: DB_HOST
              value: 'cell-001-db.cell-001.svc.cluster.local'
            - name: CACHE_HOST
              value: 'cell-001-redis.cell-001.svc.cluster.local'
            - name: QUEUE_URL
              value: 'https://sqs.us-east-1.amazonaws.com/123456789/cell-001-queue'
          resources:
            requests:
              cpu: '500m'
              memory: '1Gi'
            limits:
              cpu: '1000m'
              memory: '2Gi'
          readinessProbe:
            httpGet:
              path: /health/ready
              port: 8080
            initialDelaySeconds: 10
            periodSeconds: 5
          livenessProbe:
            httpGet:
              path: /health/live
              port: 8080
            initialDelaySeconds: 30
            periodSeconds: 10
---
# 셀 전용 데이터베이스 (StatefulSet)
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: cell-001-db
  namespace: cell-001
spec:
  serviceName: cell-001-db
  replicas: 3
  selector:
    matchLabels:
      app: cell-db
      cell-id: 'cell-001'
  template:
    metadata:
      labels:
        app: cell-db
        cell-id: 'cell-001'
    spec:
      containers:
        - name: postgres
          image: postgres:16
          ports:
            - containerPort: 5432
          env:
            - name: POSTGRES_DB
              value: 'cell_001'
          volumeMounts:
            - name: data
              mountPath: /var/lib/postgresql/data
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: ['ReadWriteOnce']
        storageClassName: gp3-encrypted
        resources:
          requests:
            storage: 100Gi
---
# 셀 전용 Redis 캐시
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cell-001-redis
  namespace: cell-001
spec:
  replicas: 1
  selector:
    matchLabels:
      app: cell-redis
      cell-id: 'cell-001'
  template:
    metadata:
      labels:
        app: cell-redis
        cell-id: 'cell-001'
    spec:
      containers:
        - name: redis
          image: redis:7-alpine
          ports:
            - containerPort: 6379
          args: ['--maxmemory', '2gb', '--maxmemory-policy', 'allkeys-lru']
---
# 셀 간 트래픽 차단을 위한 NetworkPolicy
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: cell-isolation
  namespace: cell-001
spec:
  podSelector: {}
  policyTypes:
    - Ingress
    - Egress
  ingress:
    # 같은 셀 내부 트래픽만 허용
    - from:
        - namespaceSelector:
            matchLabels:
              cell-id: 'cell-001'
    # Cell Router에서 오는 트래픽 허용
    - from:
        - namespaceSelector:
            matchLabels:
              role: 'cell-router'
  egress:
    # 같은 셀 내부로의 트래픽
    - to:
        - namespaceSelector:
            matchLabels:
              cell-id: 'cell-001'
    # DNS 허용
    - to:
        - namespaceSelector: {}
          podSelector:
            matchLabels:
              k8s-app: kube-dns
      ports:
        - port: 53
          protocol: UDP
    # 외부 AWS 서비스 접근 허용 (SQS, S3 등)
    - to:
        - ipBlock:
            cidr: 0.0.0.0/0
            except:
              - 10.0.0.0/8
      ports:
        - port: 443
          protocol: TCP

# modules/cell/main.tf
# 재사용 가능한 셀 인프라 모듈

variable "cell_id" {
  description = "셀 고유 식별자"
  type        = string
}

variable "cell_cidr" {
  description = "셀 VPC CIDR 블록"
  type        = string
}

variable "environment" {
  description = "배포 환경"
  type        = string
  default     = "production"
}

variable "max_capacity" {
  description = "셀 최대 사용자 수"
  type        = number
  default     = 10000
}

# 셀 전용 VPC
resource "aws_vpc" "cell" {
  cidr_block           = var.cell_cidr
  enable_dns_support   = true
  enable_dns_hostnames = true

  tags = {
    Name        = "cell-${var.cell_id}-vpc"
    CellId      = var.cell_id
    Environment = var.environment
  }
}

# 가용 영역별 프라이빗 서브넷
resource "aws_subnet" "private" {
  count             = 3
  vpc_id            = aws_vpc.cell.id
  cidr_block        = cidrsubnet(var.cell_cidr, 4, count.index)
  availability_zone = data.aws_availability_zones.available.names[count.index]

  tags = {
    Name   = "cell-${var.cell_id}-private-${count.index}"
    CellId = var.cell_id
  }
}

# 셀 전용 ECS 클러스터
resource "aws_ecs_cluster" "cell" {
  name = "cell-${var.cell_id}"

  setting {
    name  = "containerInsights"
    value = "enabled"
  }

  tags = {
    CellId = var.cell_id
  }
}

# 셀 전용 RDS (Multi-AZ)
resource "aws_db_instance" "cell" {
  identifier     = "cell-${var.cell_id}-db"
  engine         = "postgres"
  engine_version = "16.4"
  instance_class = "db.r6g.xlarge"

  allocated_storage     = 100
  max_allocated_storage = 500
  storage_encrypted     = true
  storage_type          = "gp3"

  multi_az            = true
  db_subnet_group_name = aws_db_subnet_group.cell.name
  vpc_security_group_ids = [aws_security_group.cell_db.id]

  db_name  = "cell_${replace(var.cell_id, "-", "_")}"
  username = "cell_admin"
  password = data.aws_secretsmanager_secret_version.db_password.secret_string

  backup_retention_period = 14
  deletion_protection     = true

  tags = {
    CellId = var.cell_id
  }
}

# 셀 전용 ElastiCache Redis
resource "aws_elasticache_replication_group" "cell" {
  replication_group_id = "cell-${var.cell_id}-redis"
  description          = "Redis cluster for cell ${var.cell_id}"

  node_type            = "cache.r6g.large"
  num_cache_clusters   = 2
  engine_version       = "7.1"
  port                 = 6379
  subnet_group_name    = aws_elasticache_subnet_group.cell.name
  security_group_ids   = [aws_security_group.cell_cache.id]

  at_rest_encryption_enabled = true
  transit_encryption_enabled = true
  automatic_failover_enabled = true

  tags = {
    CellId = var.cell_id
  }
}

# 셀 전용 SQS 큐
resource "aws_sqs_queue" "cell" {
  name = "cell-${var.cell_id}-events"

  visibility_timeout_seconds = 60
  message_retention_seconds  = 1209600  # 14일
  receive_wait_time_seconds  = 20

  redrive_policy = jsonencode({
    deadLetterTargetArn = aws_sqs_queue.cell_dlq.arn
    maxReceiveCount     = 3
  })

  tags = {
    CellId = var.cell_id
  }
}

# 셀 Auto Scaling
resource "aws_appautoscaling_target" "cell_ecs" {
  max_capacity       = var.max_capacity / 100  # 인스턴스당 100명 기준
  min_capacity       = 3
  resource_id        = "service/${aws_ecs_cluster.cell.name}/${aws_ecs_service.cell.name}"
  scalable_dimension = "ecs:service:DesiredCount"
  service_namespace  = "ecs"
}

resource "aws_appautoscaling_policy" "cell_cpu" {
  name               = "cell-${var.cell_id}-cpu-scaling"
  policy_type        = "TargetTrackingScaling"
  resource_id        = aws_appautoscaling_target.cell_ecs.resource_id
  scalable_dimension = aws_appautoscaling_target.cell_ecs.scalable_dimension
  service_namespace  = aws_appautoscaling_target.cell_ecs.service_namespace

  target_tracking_scaling_policy_configuration {
    target_value       = 60.0
    scale_in_cooldown  = 300
    scale_out_cooldown = 60

    predefined_metric_specification {
      predefined_metric_type = "ECSServiceAverageCPUUtilization"
    }
  }
}

# 출력
output "cell_vpc_id" {
  value = aws_vpc.cell.id
}

output "cell_db_endpoint" {
  value = aws_db_instance.cell.endpoint
}

output "cell_redis_endpoint" {
  value = aws_elasticache_replication_group.cell.primary_endpoint_address
}

output "cell_sqs_url" {
  value = aws_sqs_queue.cell.url
}

# environments/production/main.tf
# 실제 셀 프로비저닝

module "cell_001" {
  source       = "../../modules/cell"
  cell_id      = "001"
  cell_cidr    = "10.1.0.0/16"
  environment  = "production"
  max_capacity = 10000
}

module "cell_002" {
  source       = "../../modules/cell"
  cell_id      = "002"
  cell_cidr    = "10.2.0.0/16"
  environment  = "production"
  max_capacity = 10000
}

module "cell_003" {
  source       = "../../modules/cell"
  cell_id      = "003"
  cell_cidr    = "10.3.0.0/16"
  environment  = "production"
  max_capacity = 10000
}

# .github/workflows/cell-canary-deploy.yaml
# 셀 단위 카나리 배포 파이프라인

name: Cell Canary Deployment

on:
  push:
    branches: [main]

env:
  IMAGE_TAG: ${{ github.sha }}

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Build and push container image
        run: |
          docker build -t myregistry/cell-app:${{ env.IMAGE_TAG }} .
          docker push myregistry/cell-app:${{ env.IMAGE_TAG }}

  # Phase 1: 카나리 셀에 배포
  deploy-canary:
    needs: build
    runs-on: ubuntu-latest
    environment: canary
    steps:
      - name: Deploy to canary cell (cell-001)
        run: |
          kubectl set image deployment/cell-app \
            app=myregistry/cell-app:${{ env.IMAGE_TAG }} \
            -n cell-001
          kubectl rollout status deployment/cell-app \
            -n cell-001 --timeout=300s

      - name: Run smoke tests against canary cell
        run: |
          ./scripts/smoke-test.sh cell-001

      - name: Monitor canary metrics (10 minutes)
        run: |
          ./scripts/canary-monitor.sh cell-001 600

  # Phase 2: 첫 번째 배치 (30% 셀)
  deploy-batch-1:
    needs: deploy-canary
    runs-on: ubuntu-latest
    environment: production-batch-1
    strategy:
      max-parallel: 2
    steps:
      - name: Deploy to batch 1 cells
        run: |
          for CELL in cell-002 cell-003 cell-004; do
            kubectl set image deployment/cell-app \
              app=myregistry/cell-app:${{ env.IMAGE_TAG }} \
              -n $CELL
            kubectl rollout status deployment/cell-app \
              -n $CELL --timeout=300s
          done

  # Phase 3: 나머지 전체 셀
  deploy-remaining:
    needs: deploy-batch-1
    runs-on: ubuntu-latest
    environment: production-all
    steps:
      - name: Deploy to all remaining cells
        run: |
          REMAINING_CELLS=$(kubectl get ns -l cell-id \
            --no-headers -o custom-columns=":metadata.name" | \
            grep -v -E "cell-001|cell-002|cell-003|cell-004")
          for CELL in $REMAINING_CELLS; do
            kubectl set image deployment/cell-app \
              app=myregistry/cell-app:${{ env.IMAGE_TAG }} \
              -n $CELL
            kubectl rollout status deployment/cell-app \
              -n $CELL --timeout=300s
          done

#!/bin/bash
# cell-health-check.sh
# 전체 셀의 상태를 점검하는 헬스체크 스크립트

set -euo pipefail

CELL_ROUTER_URL="${CELL_ROUTER_URL:-http://cell-router.internal:8080}"
ALERT_WEBHOOK="${ALERT_WEBHOOK:-}"
HEALTH_THRESHOLD=3  # 연속 실패 횟수 임계값

declare -A FAILURE_COUNT

check_cell_health() {
    local cell_id="$1"
    local cell_endpoint="$2"

    # 애플리케이션 헬스 체크
    local http_code
    http_code=$(curl -s -o /dev/null -w "%{http_code}" \
        --connect-timeout 5 --max-time 10 \
        "${cell_endpoint}/health/ready" 2>/dev/null || echo "000")

    if [[ "$http_code" != "200" ]]; then
        FAILURE_COUNT[$cell_id]=$(( ${FAILURE_COUNT[$cell_id]:-0} + 1 ))
        echo "[WARN] ${cell_id}: health check failed (HTTP ${http_code}, consecutive: ${FAILURE_COUNT[$cell_id]})"

        if [[ ${FAILURE_COUNT[$cell_id]} -ge $HEALTH_THRESHOLD ]]; then
            echo "[CRITICAL] ${cell_id}: ${HEALTH_THRESHOLD} consecutive failures - triggering alert"
            send_alert "$cell_id" "$http_code"
        fi
        return 1
    fi

    # 성공 시 카운터 리셋
    FAILURE_COUNT[$cell_id]=0

    # 셀 메트릭 수집
    local metrics
    metrics=$(curl -s --max-time 5 "${cell_endpoint}/metrics/cell" 2>/dev/null || echo "{}")
    local active_connections error_rate p99_latency
    active_connections=$(echo "$metrics" | jq -r '.active_connections // "N/A"')
    error_rate=$(echo "$metrics" | jq -r '.error_rate_percent // "N/A"')
    p99_latency=$(echo "$metrics" | jq -r '.p99_latency_ms // "N/A"')

    echo "[OK] ${cell_id}: connections=${active_connections}, errors=${error_rate}%, p99=${p99_latency}ms"
    return 0
}

send_alert() {
    local cell_id="$1"
    local http_code="$2"
    local timestamp
    timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")

    if [[ -n "$ALERT_WEBHOOK" ]]; then
        curl -s -X POST "$ALERT_WEBHOOK" \
            -H "Content-Type: application/json" \
            -d "{
                \"severity\": \"critical\",
                \"cell_id\": \"${cell_id}\",
                \"message\": \"Cell ${cell_id} health check failed (HTTP ${http_code})\",
                \"timestamp\": \"${timestamp}\",
                \"action\": \"investigate_and_consider_drain\"
            }"
    fi
}

# 셀 목록 조회 및 점검
echo "=== Cell Health Check: $(date -u +"%Y-%m-%d %H:%M:%S UTC") ==="

CELLS=$(curl -s "${CELL_ROUTER_URL}/cells" | jq -r '.[] | "\(.cell_id)|\(.endpoint)"')
TOTAL=0
HEALTHY=0
UNHEALTHY=0

while IFS='|' read -r cell_id endpoint; do
    if check_cell_health "$cell_id" "$endpoint"; then
        HEALTHY=$((HEALTHY + 1))
    else
        UNHEALTHY=$((UNHEALTHY + 1))
    fi
    TOTAL=$((TOTAL + 1))
done <<< "$CELLS"

echo ""
echo "=== Summary: Total=${TOTAL}, Healthy=${HEALTHY}, Unhealthy=${UNHEALTHY} ==="

# 전체 중 50% 이상 비정상이면 글로벌 알림
if [[ $TOTAL -gt 0 ]] && [[ $((UNHEALTHY * 100 / TOTAL)) -ge 50 ]]; then
    echo "[GLOBAL CRITICAL] More than 50% cells unhealthy - possible global issue"
fi

# cell-routing-rules.yaml
# 셀 라우팅 규칙 정의

apiVersion: v1
kind: ConfigMap
metadata:
  name: cell-routing-config
  namespace: cell-router
data:
  routing-rules.yaml: |
    version: "2.0"
    default_strategy: "consistent-hashing"
    partition_key: "X-Tenant-Id"

    cells:
      - id: "cell-001"
        endpoint: "https://cell-001.internal.example.com"
        region: "us-east-1"
        status: "active"
        capacity: 10000
        weight: 100

      - id: "cell-002"
        endpoint: "https://cell-002.internal.example.com"
        region: "us-east-1"
        status: "active"
        capacity: 10000
        weight: 100

      - id: "cell-003"
        endpoint: "https://cell-003.internal.example.com"
        region: "us-west-2"
        status: "active"
        capacity: 10000
        weight: 100

      - id: "cell-004"
        endpoint: "https://cell-004.internal.example.com"
        region: "eu-west-1"
        status: "draining"
        capacity: 10000
        weight: 0
        drain_target: "cell-003"

    # 전용 셀 (대형 테넌트)
    dedicated_cells:
      - tenant_id: "tenant-enterprise-001"
        cell_id: "cell-dedicated-001"
        reason: "SLA 요구사항 - 99.99% 가용성"

      - tenant_id: "tenant-enterprise-002"
        cell_id: "cell-dedicated-002"
        reason: "데이터 레지던시 - EU GDPR 준수"

    # 장애 시 트래픽 전환 규칙
    failover_rules:
      health_check_interval_seconds: 10
      consecutive_failures_threshold: 3
      failover_strategy: "nearest-healthy-cell"
      auto_failback: true
      failback_delay_minutes: 15

    # 셀 용량 제한
    capacity_management:
      overflow_strategy: "reject-with-retry-after"
      overflow_http_status: 503
      retry_after_seconds: 30
      capacity_warning_threshold_percent: 80
      capacity_critical_threshold_percent: 95