Split View: Kubernetes RBAC 심화와 OPA Gatekeeper 정책 코드화 운영 가이드

# bad-example: Excessive permissions
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: too-permissive
rules:
  - apiGroups: ['*']
    resources: ['*']
    verbs: ['*']
---
# good-example: Only necessary permissions specified
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: app-deployer
  namespace: production
rules:
  - apiGroups: ['apps']
    resources: ['deployments']
    verbs: ['get', 'list', 'watch', 'create', 'update', 'patch']
  - apiGroups: ['']
    resources: ['services', 'configmaps']
    verbs: ['get', 'list', 'watch', 'create', 'update']
  - apiGroups: ['']
    resources: ['pods']
    verbs: ['get', 'list', 'watch']
  - apiGroups: ['']
    resources: ['pods/log']
    verbs: ['get']

# Audit policy for monitoring privilege escalation related verbs
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
  - level: RequestResponse
    verbs: ['escalate', 'bind', 'impersonate']
    resources:
      - group: 'rbac.authorization.k8s.io'
        resources: ['clusterroles', 'clusterrolebindings', 'roles', 'rolebindings']

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: monitoring-aggregate
  labels:
    rbac.example.com/aggregate-to-monitoring: 'true'
aggregationRule:
  clusterRoleSelectors:
    - matchLabels:
        rbac.example.com/aggregate-to-monitoring: 'true'
rules: [] # rules are automatically populated
---
# This ClusterRole's rules are automatically merged into the aggregate above
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: monitoring-pods
  labels:
    rbac.example.com/aggregate-to-monitoring: 'true'
rules:
  - apiGroups: ['']
    resources: ['pods', 'pods/log']
    verbs: ['get', 'list', 'watch']

# Check actual rules of an Aggregated ClusterRole
kubectl get clusterrole monitoring-aggregate -o jsonpath='{.rules}' | jq .

# Query all ClusterRoles with a specific label
kubectl get clusterrole -l rbac.example.com/aggregate-to-monitoring=true

apiVersion: v1
kind: ServiceAccount
metadata:
  name: my-app-sa
  namespace: production
automountServiceAccountToken: false # Disable token mount if not needed
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-app
  namespace: production
spec:
  template:
    spec:
      serviceAccountName: my-app-sa
      automountServiceAccountToken: false # Also specify at Pod level
      containers:
        - name: app
          image: registry.example.com/my-app:v2.1.0

API Request -> Authentication -> Authorization(RBAC) -> Mutating Admission -> Validating Admission -> etcd Storage
                                                              ^                      ^
                                                        Gatekeeper Mutation    Gatekeeper Validation

# Install with Helm (recommended)
helm repo add gatekeeper https://open-policy-agent.github.io/gatekeeper/charts
helm repo update

helm install gatekeeper gatekeeper/gatekeeper \
  --namespace gatekeeper-system \
  --create-namespace \
  --set replicas=3 \
  --set audit.replicas=1 \
  --set audit.logLevel=INFO \
  --set logDenies=true \
  --set emitAdmissionEvents=true \
  --set emitAuditEvents=true

# Verify installation
kubectl get pods -n gatekeeper-system
kubectl get crd | grep gatekeeper

assign.mutations.gatekeeper.sh
assignmetadata.mutations.gatekeeper.sh
configs.config.gatekeeper.sh
constraintpodstatuses.status.gatekeeper.sh
constrainttemplatepodstatuses.status.gatekeeper.sh
constrainttemplates.templates.gatekeeper.sh
expansiontemplate.expansion.gatekeeper.sh
modifyset.mutations.gatekeeper.sh
mutatorpodstatuses.status.gatekeeper.sh
providers.externaldata.gatekeeper.sh

apiVersion: templates.gatekeeper.sh/v1
kind: ConstraintTemplate
metadata:
  name: k8srequiredlabels
spec:
  crd:
    spec:
      names:
        kind: K8sRequiredLabels
      validation:
        openAPIV3Schema:
          type: object
          properties:
            labels:
              type: array
              description: 'List of label names that must exist'
              items:
                type: string
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8srequiredlabels

        violation[{"msg": msg, "details": {"missing_labels": missing}}] {
          provided := {label | input.review.object.metadata.labels[label]}
          required := {label | label := input.parameters.labels[_]}
          missing := required - provided
          count(missing) > 0
          msg := sprintf("Resource is missing required labels: %v", [missing])
        }
---
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sRequiredLabels
metadata:
  name: deployment-must-have-labels
spec:
  enforcementAction: deny
  match:
    kinds:
      - apiGroups: ['apps']
        kinds: ['Deployment']
    namespaces: ['production', 'staging']
    excludedNamespaces: ['kube-system', 'gatekeeper-system']
  parameters:
    labels:
      - 'app.kubernetes.io/name'
      - 'app.kubernetes.io/owner'

apiVersion: templates.gatekeeper.sh/v1
kind: ConstraintTemplate
metadata:
  name: k8sallowedrepos
spec:
  crd:
    spec:
      names:
        kind: K8sAllowedRepos
      validation:
        openAPIV3Schema:
          type: object
          properties:
            repos:
              type: array
              description: 'List of allowed container registry prefixes'
              items:
                type: string
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8sallowedrepos

        violation[{"msg": msg}] {
          container := input.review.object.spec.containers[_]
          not startswith_any(container.image, input.parameters.repos)
          msg := sprintf("Container '%v' image '%v' is not from an allowed registry. Allowed: %v", [container.name, container.image, input.parameters.repos])
        }

        violation[{"msg": msg}] {
          container := input.review.object.spec.initContainers[_]
          not startswith_any(container.image, input.parameters.repos)
          msg := sprintf("initContainer '%v' image '%v' is not from an allowed registry. Allowed: %v", [container.name, container.image, input.parameters.repos])
        }

        startswith_any(str, prefixes) {
          prefix := prefixes[_]
          startswith(str, prefix)
        }
---
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sAllowedRepos
metadata:
  name: allowed-repos-production
spec:
  enforcementAction: deny
  match:
    kinds:
      - apiGroups: ['']
        kinds: ['Pod']
    namespaces: ['production']
  parameters:
    repos:
      - 'registry.example.com/'
      - 'gcr.io/my-project/'

apiVersion: templates.gatekeeper.sh/v1
kind: ConstraintTemplate
metadata:
  name: k8spspprivileged
spec:
  crd:
    spec:
      names:
        kind: K8sPSPPrivileged
      validation:
        openAPIV3Schema:
          type: object
          properties:
            exemptImages:
              type: array
              description: 'List of images exempt from this policy'
              items:
                type: string
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8spspprivileged

        violation[{"msg": msg}] {
          container := input.review.object.spec.containers[_]
          container.securityContext.privileged == true
          not is_exempt(container.image)
          msg := sprintf("Privileged containers are not allowed: '%v'", [container.name])
        }

        violation[{"msg": msg}] {
          container := input.review.object.spec.initContainers[_]
          container.securityContext.privileged == true
          not is_exempt(container.image)
          msg := sprintf("Privileged initContainers are not allowed: '%v'", [container.name])
        }

        is_exempt(image) {
          exempt := input.parameters.exemptImages[_]
          image == exempt
        }

Phase 1: dryrun  ->  Phase 2: warn  ->  Phase 3: deny
(audit only)         (show warnings)     (actual blocking)

# Check violations for a specific Constraint
kubectl get k8srequiredlabels deployment-must-have-labels -o yaml

# Filter only violating resources (using jq)
kubectl get k8srequiredlabels deployment-must-have-labels -o json | \
  jq '.status.violations[] | {name: .name, namespace: .namespace, message: .message}'

# Check Gatekeeper audit logs
kubectl logs -n gatekeeper-system -l control-plane=audit-controller --tail=100 | \
  grep '"process":"audit"'

#!/bin/bash
# safe-enforcement-switch.sh
# Script to check violations before transitioning dryrun -> deny

CONSTRAINT_KIND=$1
CONSTRAINT_NAME=$2

echo "=== Checking current violation count ==="
VIOLATIONS=$(kubectl get ${CONSTRAINT_KIND} ${CONSTRAINT_NAME} -o json | \
  jq '.status.totalViolations')

echo "Total violations: ${VIOLATIONS}"

if [ "${VIOLATIONS}" -gt 0 ]; then
  echo ""
  echo "=== Violating resource list ==="
  kubectl get ${CONSTRAINT_KIND} ${CONSTRAINT_NAME} -o json | \
    jq -r '.status.violations[] | "\(.namespace)/\(.name): \(.message)"'
  echo ""
  echo "[WARNING] Violating resources exist. Switching to deny will block updates to those resources."
  echo "Fix the violating resources first."
  exit 1
fi

echo ""
echo "No violations found. Switching to deny."
kubectl patch ${CONSTRAINT_KIND} ${CONSTRAINT_NAME} --type=merge \
  -p '{"spec":{"enforcementAction":"deny"}}'
echo "Transition complete."

policies/
├── templates/
│   ├── k8s-required-labels.yaml
│   ├── k8s-allowed-repos.yaml
│   └── k8s-psp-privileged.yaml
├── constraints/
│   ├── production/
│   │   ├── required-labels.yaml
│   │   └── allowed-repos.yaml
│   └── staging/
│       └── required-labels.yaml
├── tests/
│   ├── required-labels_test.rego
│   └── allowed-repos_test.rego
└── Makefile

# tests/required-labels_test.rego
package k8srequiredlabels

test_violation_missing_label {
  input := {
    "review": {
      "object": {
        "metadata": {
          "labels": {
            "app.kubernetes.io/name": "myapp"
          }
        }
      }
    },
    "parameters": {
      "labels": ["app.kubernetes.io/name", "app.kubernetes.io/owner"]
    }
  }
  results := violation with input as input
  count(results) > 0
}

test_no_violation_all_labels_present {
  input := {
    "review": {
      "object": {
        "metadata": {
          "labels": {
            "app.kubernetes.io/name": "myapp",
            "app.kubernetes.io/owner": "team-platform"
          }
        }
      }
    },
    "parameters": {
      "labels": ["app.kubernetes.io/name", "app.kubernetes.io/owner"]
    }
  }
  results := violation with input as input
  count(results) == 0
}

# Run tests
opa test ./policies/templates/ ./policies/tests/ -v

# Makefile targets for CI
# Makefile
.PHONY: test-rego lint-rego apply-dryrun

test-rego:
	opa test ./policies/templates/ ./policies/tests/ -v --fail

lint-rego:
	opa check ./policies/templates/ --strict

apply-dryrun:
	kubectl apply -f ./policies/templates/ --dry-run=server
	kubectl apply -f ./policies/constraints/ --dry-run=server

# .github/workflows/policy-ci.yaml
name: Policy CI
on:
  pull_request:
    paths:
      - 'policies/**'

jobs:
  test-and-validate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup OPA
        uses: open-policy-agent/setup-opa@v2
        with:
          version: latest

      - name: Rego Lint
        run: |
          opa check ./policies/templates/ --strict

      - name: Rego Unit Tests
        run: |
          opa test ./policies/templates/ ./policies/tests/ -v --fail

      - name: Validate YAML syntax
        run: |
          for f in $(find policies/ -name '*.yaml'); do
            echo "Validating: $f"
            kubectl apply -f "$f" --dry-run=client 2>&1 || exit 1
          done

      - name: Conftest Policy Check
        uses: instrumenta/conftest-action@main
        with:
          files: policies/constraints/

# Check webhook configuration
kubectl get validatingwebhookconfiguration gatekeeper-validating-webhook-configuration -o yaml | \
  grep failurePolicy

# failurePolicy: Fail -> All requests blocked during Gatekeeper outage (dangerous!)
# failurePolicy: Ignore -> Policy validation skipped during Gatekeeper outage

# 1. Temporarily disable webhook (emergency)
kubectl delete validatingwebhookconfiguration gatekeeper-validating-webhook-configuration

# 2. Check Gatekeeper Pod status and recover
kubectl get pods -n gatekeeper-system
kubectl describe pod -n gatekeeper-system -l control-plane=controller-manager

# 3. Re-register webhook after Pod recovery (Helm reapply)
helm upgrade gatekeeper gatekeeper/gatekeeper \
  --namespace gatekeeper-system \
  --reuse-values

# 4. Verify webhook re-registration
kubectl get validatingwebhookconfiguration | grep gatekeeper

# Check ConstraintTemplate status
kubectl get constrainttemplate k8srequiredlabels -o yaml | grep -A 20 status

# Common cause: Rego syntax errors
# Check compilation errors in Controller Manager logs
kubectl logs -n gatekeeper-system -l control-plane=controller-manager --tail=50 | \
  grep -i "error\|compile\|template"

# Check if resource sync is configured in Config
kubectl get config config -n gatekeeper-system -o yaml

# Resources must be registered in Config for audit

# Gatekeeper Config: Register resources for Audit reference
apiVersion: config.gatekeeper.sh/v1alpha1
kind: Config
metadata:
  name: config
  namespace: gatekeeper-system
spec:
  sync:
    syncOnly:
      - group: ''
        version: 'v1'
        kind: 'Namespace'
      - group: ''
        version: 'v1'
        kind: 'Pod'
      - group: 'apps'
        version: 'v1'
        kind: 'Deployment'

# Check excludedNamespaces in the match block
spec:
  match:
    excludedNamespaces:
      - 'kube-system'
      - 'gatekeeper-system'
      - 'cert-manager' # System component namespace
      - 'monitoring' # Monitoring stack

# Set global exclusions via Helm values
helm upgrade gatekeeper gatekeeper/gatekeeper \
  --namespace gatekeeper-system \
  --set 'exemptNamespaces={kube-system,gatekeeper-system}'

# Enable VAP integration in Gatekeeper 3.22+
helm upgrade gatekeeper gatekeeper/gatekeeper \
  --namespace gatekeeper-system \
  --set 'controllerManager.extraArgs={--sync-vap-enforcement-scope=true}'

# Step 1: Assess current state - check which registry images are in use
kubectl get pods --all-namespaces -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\t"}{range .spec.containers[*]}{.image}{"\n"}{end}{end}' | \
  sort | uniq -c | sort -rn | head -20

# Step 2: Deploy ConstraintTemplate
kubectl apply -f policies/templates/k8s-allowed-repos.yaml

# Step 3: Deploy Constraint in dryrun mode
cat <<EOF | kubectl apply -f -
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sAllowedRepos
metadata:
  name: allowed-repos-production
spec:
  enforcementAction: dryrun
  match:
    kinds:
      - apiGroups: [""]
        kinds: ["Pod"]
    namespaces: ["production"]
  parameters:
    repos:
      - "registry.example.com/"
      - "gcr.io/my-project/"
EOF

# Step 4: Check violations after 1-2 days
kubectl get k8sallowedrepos allowed-repos-production -o json | \
  jq '.status.totalViolations'

# Step 5: Fix violating resources then switch to warn
kubectl patch k8sallowedrepos allowed-repos-production --type=merge \
  -p '{"spec":{"enforcementAction":"warn"}}'

# Step 6: Switch to deny after collecting dev team feedback
kubectl patch k8sallowedrepos allowed-repos-production --type=merge \
  -p '{"spec":{"enforcementAction":"deny"}}'

# Step 7: Verify policy enforcement
kubectl run test-blocked --image=docker.io/nginx:latest -n production
# Error: admission webhook "validation.gatekeeper.sh" denied the request