Split View: GitHub Actions Self-Hosted Runner 대규모 운영과 보안 하드닝 가이드

┌─────────────────┐     ┌──────────────────────┐
│   GitHub.com    │     │   Kubernetes Cluster  │
│                 │     │                       │
│  Job Queue      │◄───►│  ARC Controller       │
│  (workflow_job)  │     │    │                  │
│                 │     │    ▼                  │
│  Scale Set API  │◄───►│  ScaleSet Listener    │
│                 │     │    │                  │
└─────────────────┘     │    ▼                  │
                        │  EphemeralRunnerSet   │
                        │    │                  │
                        │    ├─► Runner Pod 1   │
                        │    ├─► Runner Pod 2   │
                        │    └─► Runner Pod N   │
                        └──────────────────────┘

# Create namespace for ARC controller
kubectl create namespace arc-systems

# Install via Helm
helm install arc \
  --namespace arc-systems \
  oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \
  --version 0.10.1

# Create GitHub App secret
kubectl create secret generic github-app-secret \
  --namespace arc-runners \
  --from-literal=github_app_id=12345 \
  --from-literal=github_app_installation_id=67890 \
  --from-file=github_app_private_key=./private-key.pem

# values.yaml - Runner Scale Set configuration
githubConfigUrl: 'https://github.com/my-org'
githubConfigSecret: github-app-secret

# Autoscaling settings
minRunners: 2 # Minimum standby runners (prevents cold starts)
maxRunners: 30 # Maximum runners (consider cluster resources)

# Runner group assignment (Enterprise/Org level)
runnerGroup: 'production-runners'

# Container mode settings
containerMode:
  type: 'kubernetes'
  kubernetesModeWorkVolumeClaim:
    accessModes: ['ReadWriteOnce']
    storageClassName: 'gp3'
    resources:
      requests:
        storage: 50Gi

# Pod template customization
template:
  spec:
    containers:
      - name: runner
        image: ghcr.io/actions/actions-runner:latest
        resources:
          requests:
            cpu: '2'
            memory: '4Gi'
          limits:
            cpu: '4'
            memory: '8Gi'
        env:
          - name: RUNNER_GRACEFUL_STOP_TIMEOUT
            value: '60'
    # Node selection (dedicated build node pool)
    nodeSelector:
      workload-type: ci-runner
    tolerations:
      - key: 'ci-runner'
        operator: 'Equal'
        value: 'true'
        effect: 'NoSchedule'

# Deploy Runner Scale Set
helm install arc-runner-set \
  --namespace arc-runners \
  --create-namespace \
  -f values.yaml \
  oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \
  --version 0.10.1

# Dockerfile.runner - Custom GitHub Actions Runner image
FROM ubuntu:22.04 AS base

# Install system packages (keep minimal)
RUN apt-get update && apt-get install -y --no-install-recommends \
    curl \
    ca-certificates \
    git \
    jq \
    unzip \
    zip \
    build-essential \
    && rm -rf /var/lib/apt/lists/*

# Create dedicated runner user
RUN useradd -m -d /home/runner -s /bin/bash runner

# Install Runner binary
ARG RUNNER_VERSION=2.321.0
RUN curl -fsSL -o runner.tar.gz \
    "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" \
    && mkdir -p /home/runner/actions-runner \
    && tar xzf runner.tar.gz -C /home/runner/actions-runner \
    && rm runner.tar.gz \
    && /home/runner/actions-runner/bin/installdependencies.sh

# Install Node.js 22 LTS
RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
    && apt-get install -y nodejs \
    && rm -rf /var/lib/apt/lists/*

# Install Docker CLI (CLI only, not DinD)
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker.gpg \
    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu jammy stable" \
    > /etc/apt/sources.list.d/docker.list \
    && apt-get update && apt-get install -y docker-ce-cli \
    && rm -rf /var/lib/apt/lists/*

# Disable auto-updates (manage versions through image builds)
ENV RUNNER_MANUALLY_TRAP_SIG=1
ENV ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT=1

# Set permissions and switch user
RUN chown -R runner:runner /home/runner
USER runner
WORKDIR /home/runner/actions-runner

ENTRYPOINT ["./run.sh"]

# Build and push image
docker build -t ghcr.io/my-org/actions-runner:v2.321.0-custom -f Dockerfile.runner .
docker push ghcr.io/my-org/actions-runner:v2.321.0-custom

# Verify ephemeral runner usage in workflows
runs-on: arc-runner-set # ARC is ephemeral by default

# For VM-based self-hosted runners
# Register with the --ephemeral flag via ./config.sh

# network-policy.yaml - Runner Pod network restrictions
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: runner-network-policy
  namespace: arc-runners
spec:
  podSelector:
    matchLabels:
      app.kubernetes.io/component: runner
  policyTypes:
    - Egress
    - Ingress
  ingress: [] # Block inbound traffic from outside to Runner
  egress:
    # GitHub API and Actions services
    - to:
        - ipBlock:
            cidr: 140.82.112.0/20 # github.com
        - ipBlock:
            cidr: 185.199.108.0/22 # GitHub Pages/CDN
      ports:
        - protocol: TCP
          port: 443
    # Internal container registry
    - to:
        - namespaceSelector:
            matchLabels:
              name: registry
      ports:
        - protocol: TCP
          port: 5000
    # DNS
    - to: []
      ports:
        - protocol: UDP
          port: 53
        - protocol: TCP
          port: 53

# rbac.yaml - Runner ServiceAccount minimum permissions
apiVersion: v1
kind: ServiceAccount
metadata:
  name: runner-sa
  namespace: arc-runners
automountServiceAccountToken: false # Disable auto-mounting K8s API token
---
# Only bind minimal Role when necessary
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: runner-minimal
  namespace: arc-runners
rules: [] # No permissions by default

# Secure workflow writing patterns
name: Secure CI Pipeline
on:
  pull_request:
    branches: [main]

# Minimum privilege tokens
permissions:
  contents: read
  packages: read

jobs:
  build:
    runs-on: arc-runner-set
    steps:
      # Use Actions with SHA pinning (not tags)
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      # Avoid directly exposing secrets in environment variables
      - name: Build
        run: |
          echo "Building..."
        env:
          # Inject secrets only in the steps that need them
          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}

jobs:
  build:
    runs-on: arc-runner-set
    steps:
      - uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
        with:
          egress-policy: audit # First use audit to understand traffic patterns
          # Switch to block after understanding patterns
          # egress-policy: block
          # allowed-endpoints: >
          #   github.com:443
          #   registry.npmjs.org:443
          #   ghcr.io:443

# Add PVC to ARC Runner Scale Set values.yaml
template:
  spec:
    containers:
      - name: runner
        image: ghcr.io/my-org/actions-runner:latest
        volumeMounts:
          - name: cache-volume
            mountPath: /opt/cache
        env:
          - name: RUNNER_TOOL_CACHE
            value: /opt/cache/tool-cache
          - name: npm_config_cache
            value: /opt/cache/npm
          - name: GOPATH
            value: /opt/cache/go
    volumes:
      - name: cache-volume
        persistentVolumeClaim:
          claimName: runner-cache-pvc
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: runner-cache-pvc
  namespace: arc-runners
spec:
  accessModes:
    - ReadWriteMany # Multiple Runner Pods access simultaneously
  storageClassName: efs # AWS EFS or NFS
  resources:
    requests:
      storage: 100Gi

# MinIO-based cache server deployment (within the same VPC/region)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: actions-cache-server
  namespace: arc-systems
spec:
  replicas: 1
  selector:
    matchLabels:
      app: actions-cache
  template:
    spec:
      containers:
        - name: minio
          image: minio/minio:latest
          args: ['server', '/data', '--console-address', ':9001']
          env:
            - name: MINIO_ROOT_USER
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: user
            - name: MINIO_ROOT_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: password
          volumeMounts:
            - name: data
              mountPath: /data
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: minio-data-pvc

# Use BuildKit registry cache in workflows
- name: Build and Push
  uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6
  with:
    push: true
    tags: ghcr.io/my-org/my-app:${{ github.sha }}
    cache-from: type=registry,ref=ghcr.io/my-org/my-app:buildcache
    cache-to: type=registry,ref=ghcr.io/my-org/my-app:buildcache,mode=max

# Prometheus ServiceMonitor configuration
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: arc-controller-monitor
  namespace: arc-systems
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: gha-runner-scale-set-controller
  endpoints:
    - port: metrics
      interval: 30s
      path: /metrics

# Alertmanager rules
groups:
  - name: arc-runner-alerts
    rules:
      # Runner pool exhaustion warning
      - alert: RunnerPoolExhausted
        expr: |
          gha_runner_scale_set_desired_replicas
          >= gha_runner_scale_set_max_replicas * 0.9
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'Runner pool is over 90% utilized'
          description: 'Increase maxRunners or optimize workflows'

      # Runner registration failure detection
      - alert: RunnerRegistrationFailed
        expr: |
          rate(gha_runner_scale_set_registration_failures_total[5m]) > 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: 'Runner registration failure detected'
          description: 'Check GitHub App authentication or network'

      # Prolonged Pod Pending state
      - alert: RunnerPodPending
        expr: |
          kube_pod_status_phase{namespace="arc-runners", phase="Pending"} > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: 'Runner Pod has been Pending for over 10 minutes'
          description: 'Possible node resource shortage or PVC binding failure'

# 1. Check Listener Pod logs
kubectl logs -n arc-systems -l app.kubernetes.io/component=runner-scale-set-listener --tail=100

# 2. Common cause: GitHub App authentication expiry
# - Check private key file
# - Check App installation status (org settings > GitHub Apps)

# 3. Network issue: Cannot reach GitHub API
kubectl exec -n arc-systems deploy/arc-gha-runner-scale-set-controller -- \
  curl -s https://api.github.com/meta | jq '.actions[]'

kubectl create secret generic github-app-secret \
  --namespace arc-runners \
  --from-literal=github_app_id=12345 \
  --from-literal=github_app_installation_id=67890 \
  --from-file=github_app_private_key=./new-private-key.pem \
  --dry-run=client -o yaml | kubectl apply -f -

# Restart Controller
kubectl rollout restart deployment -n arc-systems arc-gha-runner-scale-set-controller

# Check Pod events
kubectl describe pod -n arc-runners -l actions.github.com/scale-set-name=arc-runner-set

# Response per common cause
# 1. Node resource shortage
kubectl top nodes
# -> Verify Cluster Autoscaler is working, or lower maxRunners

# 2. PVC binding waiting
kubectl get pvc -n arc-runners
# -> Check StorageClass settings, availability zone mismatch

# 3. Image pull failure
kubectl get events -n arc-runners --sort-by='.lastTimestamp' | grep -i pull
# -> Check image tag, registry authentication

# Check Runner registration status
kubectl get ephemeralrunner -n arc-runners

# Check Runner labels (must match runs-on)
kubectl get autoscalingrunnersets -n arc-runners -o yaml | grep -A5 labels

# Check Runner group settings on GitHub
# Settings > Actions > Runner groups > Verify the repository is included in the group

# Check current Runner version
kubectl exec -n arc-runners -it <runner-pod> -- ./config.sh --version

# Update image (after modifying values.yaml)
helm upgrade arc-runner-set \
  --namespace arc-runners \
  -f values.yaml \
  oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set

# Separate Runner Scale Sets by purpose
# 1. General CI (lightweight tests, lint)
# values-ci-light.yaml
minRunners: 2
maxRunners: 20
template:
  spec:
    containers:
      - name: runner
        resources:
          requests:
            cpu: "1"
            memory: "2Gi"

# 2. Build-dedicated (compilation, Docker builds)
# values-ci-build.yaml
minRunners: 1
maxRunners: 10
template:
  spec:
    containers:
      - name: runner
        resources:
          requests:
            cpu: "4"
            memory: "8Gi"

# 3. GPU workloads (ML model testing)
# values-gpu.yaml
minRunners: 0
maxRunners: 4
template:
  spec:
    containers:
      - name: runner
        resources:
          limits:
            nvidia.com/gpu: 1
    nodeSelector:
      accelerator: nvidia-a10g

template:
  spec:
    terminationGracePeriodSeconds: 3600 # Wait up to 1 hour
    containers:
      - name: runner
        env:
          - name: RUNNER_GRACEFUL_STOP_TIMEOUT
            value: '3500' # Slightly shorter than terminationGracePeriodSeconds

# Karpenter NodePool example (dedicated to CI Runners)
apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: ci-runners
spec:
  template:
    metadata:
      labels:
        workload-type: ci-runner
    spec:
      taints:
        - key: ci-runner
          value: 'true'
          effect: NoSchedule
      requirements:
        - key: kubernetes.io/arch
          operator: In
          values: ['amd64']
        - key: karpenter.sh/capacity-type
          operator: In
          values: ['on-demand', 'spot']
        - key: node.kubernetes.io/instance-type
          operator: In
          values: ['m7i.xlarge', 'm7i.2xlarge', 'm6i.xlarge', 'm6i.2xlarge']
  limits:
    cpu: 200
    memory: 400Gi
  disruption:
    consolidationPolicy: WhenEmpty
    consolidateAfter: 60s