Keycloak 可観測性 — メトリクス、監査ログ、イベントベースのモニタリング

# イベント設定 (kcadm.sh)
./kcadm.sh update events/config -r myrealm \
  -s eventsEnabled=true \
  -s 'eventsExpiration=2592000' \
  -s 'enabledEventTypes=["LOGIN","LOGIN_ERROR","LOGOUT","REGISTER","UPDATE_PASSWORD","UPDATE_EMAIL","REMOVE_TOTP","UPDATE_TOTP","TOKEN_EXCHANGE","REFRESH_TOKEN_ERROR"]' \
  -s adminEventsEnabled=true \
  -s adminEventsDetailsEnabled=true

# 直近のログイン失敗イベントを取得
./kcadm.sh get events -r myrealm \
  -q type=LOGIN_ERROR -q max=50

# 特定ユーザーの admin 変更履歴
./kcadm.sh get admin-events -r myrealm \
  -q resourcePath=users/USER_UUID -q max=20

事故パターン 1: 無限保持
  eventsExpiration 未設定 → テーブルが数億行に肥大化
  → イベント INSERT の遅延 → ログイントランザクション全体の遅延
  → 「ログインが遅い」チケットの殺到

事故パターン 2: 一括削除の爆弾
  後から有効期限を設定 → 期限切れ処理が数億行の削除を試行
  → DB ロック競合、WAL/redo の急増 → 稼働中の DB が麻痺

public class KafkaEventListenerProvider implements EventListenerProvider {

  private final KafkaProducer<String, String> producer;
  private final String topic;
  private final ObjectMapper mapper = new ObjectMapper();

  public KafkaEventListenerProvider(KafkaProducer<String, String> producer, String topic) {
    this.producer = producer;
    this.topic = topic;
  }

  @Override
  public void onEvent(Event event) {
    // LOGIN_ERROR などのセキュリティイベントを非同期送信
    try {
      String payload = mapper.writeValueAsString(Map.of(
          "category", "USER_EVENT",
          "type", event.getType().name(),
          "realmId", event.getRealmId(),
          "clientId", event.getClientId(),
          "userId", event.getUserId(),
          "ipAddress", event.getIpAddress(),
          "error", event.getError(),
          "time", event.getTime(),
          "details", event.getDetails()
      ));
      producer.send(new ProducerRecord<>(topic, event.getRealmId(), payload));
    } catch (Exception e) {
      // 認証フローを絶対に壊さない — ログを残して続行
      LoggerFactory.getLogger(getClass()).warn("event publish failed", e);
    }
  }

  @Override
  public void onEvent(AdminEvent adminEvent, boolean includeRepresentation) {
    try {
      String payload = mapper.writeValueAsString(Map.of(
          "category", "ADMIN_EVENT",
          "operation", adminEvent.getOperationType().name(),
          "resourceType", String.valueOf(adminEvent.getResourceTypeAsString()),
          "resourcePath", adminEvent.getResourcePath(),
          "realmId", adminEvent.getRealmId(),
          "authUserId", adminEvent.getAuthDetails().getUserId(),
          "time", adminEvent.getTime()
      ));
      producer.send(new ProducerRecord<>(topic, adminEvent.getRealmId(), payload));
    } catch (Exception e) {
      LoggerFactory.getLogger(getClass()).warn("admin event publish failed", e);
    }
  }

  @Override
  public void close() {
  }
}

public class KafkaEventListenerProviderFactory implements EventListenerProviderFactory {

  private KafkaProducer<String, String> producer;
  private String topic;

  @Override
  public EventListenerProvider create(KeycloakSession session) {
    return new KafkaEventListenerProvider(producer, topic);
  }

  @Override
  public void init(Config.Scope config) {
    Properties props = new Properties();
    props.put("bootstrap.servers", config.get("bootstrapServers", "kafka:9092"));
    props.put("key.serializer", StringSerializer.class.getName());
    props.put("value.serializer", StringSerializer.class.getName());
    props.put("acks", "1");
    props.put("linger.ms", "20");
    this.producer = new KafkaProducer<>(props);
    this.topic = config.get("topic", "keycloak-events");
  }

  @Override
  public String getId() {
    return "kafka-event-listener";
  }

  @Override
  public void close() {
    if (producer != null) producer.close();
  }
}

META-INF/services/org.keycloak.events.EventListenerProviderFactory
ファイル内容: com.example.keycloak.KafkaEventListenerProviderFactory

cp keycloak-kafka-listener.jar /opt/keycloak/providers/
/opt/keycloak/bin/kc.sh build

./kcadm.sh update events/config -r myrealm \
  -s 'eventsListeners=["jboss-logging","kafka-event-listener"]'

# JSON ログ出力 + イベントログレベルの調整
bin/kc.sh start \
  --log-console-output=json \
  --log-level=INFO,org.keycloak.events:DEBUG

# promtail 設定の抜粋 — Keycloak イベントだけをラベリング
scrape_configs:
  - job_name: keycloak
    static_configs:
      - targets: [localhost]
        labels:
          job: keycloak
          __path__: /var/log/keycloak/*.log
    pipeline_stages:
      - json:
          expressions:
            logger: loggerName
            message: message
      - match:
          selector: '{job="keycloak"} |= "org.keycloak.events"'
          stages:
            - labels:
                logger:

bin/kc.sh start \
  --metrics-enabled=true \
  --event-metrics-user-enabled=true \
  --event-metrics-user-tags=realm,clientId,idp \
  --http-metrics-histograms-enabled=true

# prometheus.yml 抜粋
scrape_configs:
  - job_name: keycloak
    metrics_path: /metrics
    static_configs:
      - targets: ['keycloak-0.mgmt:9000', 'keycloak-1.mgmt:9000']
    scheme: https
    tls_config:
      insecure_skip_verify: false

+----------------------------------------------------------------+
| Keycloak Security Operations                                   |
+------------------------+---------------------------------------+
| ログイン成功/失敗 (rate) | 失敗率 % (成功+失敗 比)                 |
| realm/client 別スタック  | しきい値ライン表示 (5%, 20%)            |
+------------------------+---------------------------------------+
| LOGIN_ERROR 理由の分布  | IP 別失敗 Top 10 (Loki クエリ)          |
| invalid_user_credentials| user_not_found 急増 = enumeration     |
+------------------------+---------------------------------------+
| 新規登録/パスワード変更  | admin イベントのタイムライン             |
+------------------------+---------------------------------------+

+----------------------------------------------------------------+
| Keycloak System Health                                         |
+------------------------+---------------------------------------+
| HTTP p50/p95/p99       | 5xx 比率                               |
+------------------------+---------------------------------------+
| Agroal プール使用率/待機 | Infinispan キャッシュヒット率/エントリ数  |
+------------------------+---------------------------------------+
| JVM ヒープ/GC pause    | インスタンス別 CPU/スレッド              |
+----------------------------------------------------------------+

# 5 分ウィンドウのログイン失敗率 (%)
100 *
  sum(rate(keycloak_user_events_total{event="login_error"}[5m]))
/
  sum(rate(keycloak_user_events_total{event=~"login|login_error"}[5m]))

# トークンエンドポイントの p99 レイテンシ
histogram_quantile(0.99,
  sum by (le) (rate(http_server_requests_seconds_bucket{uri=~".*token.*"}[5m])))

# DB プール待機平均 (ms)
avg(agroal_blocking_time_average_milliseconds)

bin/kc.sh start \
  --tracing-enabled=true \
  --tracing-endpoint=http://otel-collector:4317 \
  --tracing-protocol=grpc \
  --tracing-sampler-type=traceidratio \
  --tracing-sampler-ratio=0.05 \
  --tracing-service-name=keycloak-prod

./kcadm.sh update realms/myrealm \
  -s bruteForceProtected=true \
  -s failureFactor=5 \
  -s waitIncrementSeconds=60 \
  -s maxFailureWaitSeconds=900 \
  -s maxDeltaTimeSeconds=43200 \
  -s permanentLockout=false

groups:
  - name: keycloak-security
    rules:
      - alert: KeycloakLoginFailureRateHigh
        expr: |
          100 *
            sum(rate(keycloak_user_events_total{event="login_error"}[5m]))
          /
            sum(rate(keycloak_user_events_total{event=~"login|login_error"}[5m]))
          > 20
        for: 10m
        labels:
          severity: warning
          team: identity
        annotations:
          summary: "ログイン失敗率が 20% 超過（10 分継続）"
          description: "攻撃または認証経路の障害の可能性。セキュリティダッシュボードを確認してください。"

      - alert: KeycloakLoginErrorSpike
        expr: |
          sum(rate(keycloak_user_events_total{event="login_error"}[5m]))
          > 4 * sum(rate(keycloak_user_events_total{event="login_error"}[5m] offset 1d))
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "ログイン失敗が前日同時刻比 4 倍に急増"

  - name: keycloak-health
    rules:
      - alert: KeycloakDbPoolExhausted
        expr: min(agroal_available_count) == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "DB コネクションプール枯渇 — ログイン全面障害が目前"

      - alert: KeycloakTokenLatencyHigh
        expr: |
          histogram_quantile(0.99,
            sum by (le) (rate(http_server_requests_seconds_bucket{uri=~".*token.*"}[5m])))
          > 1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "トークンエンドポイントの p99 が 1 秒超過"

      - alert: KeycloakInstanceDown
        expr: up{job="keycloak"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Keycloak インスタンスダウン"