Keycloak 관측성 — 메트릭, 감사 로그, 이벤트 기반 모니터링

# 이벤트 설정 (kcadm.sh)
./kcadm.sh update events/config -r myrealm \
  -s eventsEnabled=true \
  -s 'eventsExpiration=2592000' \
  -s 'enabledEventTypes=["LOGIN","LOGIN_ERROR","LOGOUT","REGISTER","UPDATE_PASSWORD","UPDATE_EMAIL","REMOVE_TOTP","UPDATE_TOTP","TOKEN_EXCHANGE","REFRESH_TOKEN_ERROR"]' \
  -s adminEventsEnabled=true \
  -s adminEventsDetailsEnabled=true

# 최근 로그인 실패 이벤트 조회
./kcadm.sh get events -r myrealm \
  -q type=LOGIN_ERROR -q max=50

# 특정 사용자의 admin 변경 이력
./kcadm.sh get admin-events -r myrealm \
  -q resourcePath=users/USER_UUID -q max=20

사고 패턴 1: 무한 보존
  eventsExpiration 미설정 → 테이블이 수억 행으로 비대
  → 이벤트 INSERT 지연 → 로그인 트랜잭션 전체 지연
  → "로그인이 느려요" 티켓 폭주

사고 패턴 2: 일괄 삭제 폭탄
  뒤늦게 만료 설정 → 만료 작업이 수억 행 삭제 시도
  → DB lock 경합, WAL/redo 폭증 → 운영 중 DB 마비

public class KafkaEventListenerProvider implements EventListenerProvider {

  private final KafkaProducer<String, String> producer;
  private final String topic;
  private final ObjectMapper mapper = new ObjectMapper();

  public KafkaEventListenerProvider(KafkaProducer<String, String> producer, String topic) {
    this.producer = producer;
    this.topic = topic;
  }

  @Override
  public void onEvent(Event event) {
    // LOGIN_ERROR 등 보안 이벤트를 비동기 전송
    try {
      String payload = mapper.writeValueAsString(Map.of(
          "category", "USER_EVENT",
          "type", event.getType().name(),
          "realmId", event.getRealmId(),
          "clientId", event.getClientId(),
          "userId", event.getUserId(),
          "ipAddress", event.getIpAddress(),
          "error", event.getError(),
          "time", event.getTime(),
          "details", event.getDetails()
      ));
      producer.send(new ProducerRecord<>(topic, event.getRealmId(), payload));
    } catch (Exception e) {
      // 절대 인증 흐름을 깨뜨리지 않는다 — 로깅 후 진행
      LoggerFactory.getLogger(getClass()).warn("event publish failed", e);
    }
  }

  @Override
  public void onEvent(AdminEvent adminEvent, boolean includeRepresentation) {
    try {
      String payload = mapper.writeValueAsString(Map.of(
          "category", "ADMIN_EVENT",
          "operation", adminEvent.getOperationType().name(),
          "resourceType", String.valueOf(adminEvent.getResourceTypeAsString()),
          "resourcePath", adminEvent.getResourcePath(),
          "realmId", adminEvent.getRealmId(),
          "authUserId", adminEvent.getAuthDetails().getUserId(),
          "time", adminEvent.getTime()
      ));
      producer.send(new ProducerRecord<>(topic, adminEvent.getRealmId(), payload));
    } catch (Exception e) {
      LoggerFactory.getLogger(getClass()).warn("admin event publish failed", e);
    }
  }

  @Override
  public void close() {
  }
}

public class KafkaEventListenerProviderFactory implements EventListenerProviderFactory {

  private KafkaProducer<String, String> producer;
  private String topic;

  @Override
  public EventListenerProvider create(KeycloakSession session) {
    return new KafkaEventListenerProvider(producer, topic);
  }

  @Override
  public void init(Config.Scope config) {
    Properties props = new Properties();
    props.put("bootstrap.servers", config.get("bootstrapServers", "kafka:9092"));
    props.put("key.serializer", StringSerializer.class.getName());
    props.put("value.serializer", StringSerializer.class.getName());
    props.put("acks", "1");
    props.put("linger.ms", "20");
    this.producer = new KafkaProducer<>(props);
    this.topic = config.get("topic", "keycloak-events");
  }

  @Override
  public String getId() {
    return "kafka-event-listener";
  }

  @Override
  public void close() {
    if (producer != null) producer.close();
  }
}

META-INF/services/org.keycloak.events.EventListenerProviderFactory
파일 내용: com.example.keycloak.KafkaEventListenerProviderFactory

cp keycloak-kafka-listener.jar /opt/keycloak/providers/
/opt/keycloak/bin/kc.sh build

./kcadm.sh update events/config -r myrealm \
  -s 'eventsListeners=["jboss-logging","kafka-event-listener"]'

# JSON 로그 출력 + 이벤트 로그 레벨 조정
bin/kc.sh start \
  --log-console-output=json \
  --log-level=INFO,org.keycloak.events:DEBUG

# promtail 설정 발췌 — Keycloak 이벤트만 라벨링
scrape_configs:
  - job_name: keycloak
    static_configs:
      - targets: [localhost]
        labels:
          job: keycloak
          __path__: /var/log/keycloak/*.log
    pipeline_stages:
      - json:
          expressions:
            logger: loggerName
            message: message
      - match:
          selector: '{job="keycloak"} |= "org.keycloak.events"'
          stages:
            - labels:
                logger:

bin/kc.sh start \
  --metrics-enabled=true \
  --event-metrics-user-enabled=true \
  --event-metrics-user-tags=realm,clientId,idp \
  --http-metrics-histograms-enabled=true

# prometheus.yml 발췌
scrape_configs:
  - job_name: keycloak
    metrics_path: /metrics
    static_configs:
      - targets: ['keycloak-0.mgmt:9000', 'keycloak-1.mgmt:9000']
    scheme: https
    tls_config:
      insecure_skip_verify: false

+----------------------------------------------------------------+
| Keycloak Security Operations                                   |
+------------------------+---------------------------------------+
| 로그인 성공/실패 (rate)  | 실패율 % (성공+실패 대비)                |
| realm/client별 스택     | 임계선 표시 (5%, 20%)                   |
+------------------------+---------------------------------------+
| LOGIN_ERROR 사유 분포   | IP별 실패 Top 10 (Loki 쿼리)            |
| invalid_user_credentials| user_not_found 비율 급증 = enumeration |
+------------------------+---------------------------------------+
| 신규 등록/비밀번호 변경  | admin 이벤트 타임라인                    |
+------------------------+---------------------------------------+

+----------------------------------------------------------------+
| Keycloak System Health                                         |
+------------------------+---------------------------------------+
| HTTP p50/p95/p99       | 5xx 비율                               |
+------------------------+---------------------------------------+
| Agroal 풀 사용률/대기   | Infinispan 캐시 히트율/엔트리 수          |
+------------------------+---------------------------------------+
| JVM 힙/GC pause        | 인스턴스별 CPU/스레드                    |
+----------------------------------------------------------------+

# 5분 윈도우 로그인 실패율 (%)
100 *
  sum(rate(keycloak_user_events_total{event="login_error"}[5m]))
/
  sum(rate(keycloak_user_events_total{event=~"login|login_error"}[5m]))

# 토큰 엔드포인트 p99 지연
histogram_quantile(0.99,
  sum by (le) (rate(http_server_requests_seconds_bucket{uri=~".*token.*"}[5m])))

# DB 풀 대기 평균 (ms)
avg(agroal_blocking_time_average_milliseconds)

bin/kc.sh start \
  --tracing-enabled=true \
  --tracing-endpoint=http://otel-collector:4317 \
  --tracing-protocol=grpc \
  --tracing-sampler-type=traceidratio \
  --tracing-sampler-ratio=0.05 \
  --tracing-service-name=keycloak-prod

./kcadm.sh update realms/myrealm \
  -s bruteForceProtected=true \
  -s failureFactor=5 \
  -s waitIncrementSeconds=60 \
  -s maxFailureWaitSeconds=900 \
  -s maxDeltaTimeSeconds=43200 \
  -s permanentLockout=false

groups:
  - name: keycloak-security
    rules:
      - alert: KeycloakLoginFailureRateHigh
        expr: |
          100 *
            sum(rate(keycloak_user_events_total{event="login_error"}[5m]))
          /
            sum(rate(keycloak_user_events_total{event=~"login|login_error"}[5m]))
          > 20
        for: 10m
        labels:
          severity: warning
          team: identity
        annotations:
          summary: "로그인 실패율 20% 초과 (10분 지속)"
          description: "공격 또는 인증 경로 장애 가능성. 보안 대시보드를 확인하세요."

      - alert: KeycloakLoginErrorSpike
        expr: |
          sum(rate(keycloak_user_events_total{event="login_error"}[5m]))
          > 4 * sum(rate(keycloak_user_events_total{event="login_error"}[5m] offset 1d))
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "로그인 실패가 전일 동시간 대비 4배 급증"

  - name: keycloak-health
    rules:
      - alert: KeycloakDbPoolExhausted
        expr: min(agroal_available_count) == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "DB 커넥션 풀 고갈 — 로그인 전면 장애 임박"

      - alert: KeycloakTokenLatencyHigh
        expr: |
          histogram_quantile(0.99,
            sum by (le) (rate(http_server_requests_seconds_bucket{uri=~".*token.*"}[5m])))
          > 1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "토큰 엔드포인트 p99가 1초 초과"

      - alert: KeycloakInstanceDown
        expr: up{job="keycloak"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Keycloak 인스턴스 다운"