GitHub Actions Self-Hosted Runner 대규모 운영과 보안 하드닝 가이드

┌─────────────────┐     ┌──────────────────────┐
│   GitHub.com    │     │   Kubernetes Cluster  │
│                 │     │                       │
│  Job Queue      │◄───►│  ARC Controller       │
│  (workflow_job)  │     │    │                  │
│                 │     │    ▼                  │
│  Scale Set API  │◄───►│  ScaleSet Listener    │
│                 │     │    │                  │
└─────────────────┘     │    ▼                  │
                        │  EphemeralRunnerSet   │
                        │    │                  │
                        │    ├─► Runner Pod 1   │
                        │    ├─► Runner Pod 2   │
                        │    └─► Runner Pod N   │
                        └──────────────────────┘

# ARC 컨트롤러용 네임스페이스 생성
kubectl create namespace arc-systems

# Helm 리포지토리 추가
helm install arc \
  --namespace arc-systems \
  oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \
  --version 0.10.1

# GitHub App 시크릿 생성
kubectl create secret generic github-app-secret \
  --namespace arc-runners \
  --from-literal=github_app_id=12345 \
  --from-literal=github_app_installation_id=67890 \
  --from-file=github_app_private_key=./private-key.pem

# values.yaml - Runner Scale Set 설정
githubConfigUrl: 'https://github.com/my-org'
githubConfigSecret: github-app-secret

# 오토스케일링 설정
minRunners: 2 # 최소 대기 러너 (콜드 스타트 방지)
maxRunners: 30 # 최대 러너 수 (클러스터 리소스 고려)

# 러너 그룹 지정 (Enterprise/Org 수준)
runnerGroup: 'production-runners'

# 컨테이너 모드 설정
containerMode:
  type: 'kubernetes'
  kubernetesModeWorkVolumeClaim:
    accessModes: ['ReadWriteOnce']
    storageClassName: 'gp3'
    resources:
      requests:
        storage: 50Gi

# Pod 템플릿 커스터마이징
template:
  spec:
    containers:
      - name: runner
        image: ghcr.io/actions/actions-runner:latest
        resources:
          requests:
            cpu: '2'
            memory: '4Gi'
          limits:
            cpu: '4'
            memory: '8Gi'
        env:
          - name: RUNNER_GRACEFUL_STOP_TIMEOUT
            value: '60'
    # 노드 선택 (빌드 전용 노드풀)
    nodeSelector:
      workload-type: ci-runner
    tolerations:
      - key: 'ci-runner'
        operator: 'Equal'
        value: 'true'
        effect: 'NoSchedule'

# Runner Scale Set 배포
helm install arc-runner-set \
  --namespace arc-runners \
  --create-namespace \
  -f values.yaml \
  oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \
  --version 0.10.1

# Dockerfile.runner - 커스텀 GitHub Actions Runner 이미지
FROM ubuntu:22.04 AS base

# 시스템 패키지 설치 (최소한으로 유지)
RUN apt-get update && apt-get install -y --no-install-recommends \
    curl \
    ca-certificates \
    git \
    jq \
    unzip \
    zip \
    build-essential \
    && rm -rf /var/lib/apt/lists/*

# Runner 전용 사용자 생성
RUN useradd -m -d /home/runner -s /bin/bash runner

# Runner 바이너리 설치
ARG RUNNER_VERSION=2.321.0
RUN curl -fsSL -o runner.tar.gz \
    "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" \
    && mkdir -p /home/runner/actions-runner \
    && tar xzf runner.tar.gz -C /home/runner/actions-runner \
    && rm runner.tar.gz \
    && /home/runner/actions-runner/bin/installdependencies.sh

# Node.js 22 LTS 설치
RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
    && apt-get install -y nodejs \
    && rm -rf /var/lib/apt/lists/*

# Docker CLI 설치 (DinD가 아닌 CLI만)
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker.gpg \
    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu jammy stable" \
    > /etc/apt/sources.list.d/docker.list \
    && apt-get update && apt-get install -y docker-ce-cli \
    && rm -rf /var/lib/apt/lists/*

# 자동 업데이트 비활성화 (이미지 빌드로 버전 관리)
ENV RUNNER_MANUALLY_TRAP_SIG=1
ENV ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT=1

# 권한 설정 및 사용자 전환
RUN chown -R runner:runner /home/runner
USER runner
WORKDIR /home/runner/actions-runner

ENTRYPOINT ["./run.sh"]

# 이미지 빌드 및 푸시
docker build -t ghcr.io/my-org/actions-runner:v2.321.0-custom -f Dockerfile.runner .
docker push ghcr.io/my-org/actions-runner:v2.321.0-custom

# 워크플로우에서 ephemeral runner 사용 확인
runs-on: arc-runner-set # ARC는 기본적으로 ephemeral

# VM 기반 self-hosted runner의 경우
# ./config.sh --ephemeral 플래그로 등록

# network-policy.yaml - Runner Pod 네트워크 제한
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: runner-network-policy
  namespace: arc-runners
spec:
  podSelector:
    matchLabels:
      app.kubernetes.io/component: runner
  policyTypes:
    - Egress
    - Ingress
  ingress: [] # 외부에서 Runner로의 인바운드 차단
  egress:
    # GitHub API 및 Actions 서비스
    - to:
        - ipBlock:
            cidr: 140.82.112.0/20 # github.com
        - ipBlock:
            cidr: 185.199.108.0/22 # GitHub Pages/CDN
      ports:
        - protocol: TCP
          port: 443
    # 내부 컨테이너 레지스트리
    - to:
        - namespaceSelector:
            matchLabels:
              name: registry
      ports:
        - protocol: TCP
          port: 5000
    # DNS
    - to: []
      ports:
        - protocol: UDP
          port: 53
        - protocol: TCP
          port: 53

# rbac.yaml - Runner ServiceAccount 최소 권한
apiVersion: v1
kind: ServiceAccount
metadata:
  name: runner-sa
  namespace: arc-runners
automountServiceAccountToken: false # K8s API 토큰 자동 마운트 비활성화
---
# 필요한 경우에만 최소 권한의 Role 바인딩
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: runner-minimal
  namespace: arc-runners
rules: [] # 기본적으로 권한 없음

# 안전한 워크플로우 작성 패턴
name: Secure CI Pipeline
on:
  pull_request:
    branches: [main]

# 최소 권한 토큰
permissions:
  contents: read
  packages: read

jobs:
  build:
    runs-on: arc-runner-set
    steps:
      # SHA 고정으로 Action 사용 (태그가 아님)
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      # 환경 변수에 시크릿 직접 노출 금지
      - name: Build
        run: |
          echo "Building..."
        env:
          # 시크릿은 필요한 스텝에서만 주입
          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}

jobs:
  build:
    runs-on: arc-runner-set
    steps:
      - uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
        with:
          egress-policy: audit # 먼저 audit으로 트래픽 패턴 파악
          # 패턴 파악 후 block으로 전환
          # egress-policy: block
          # allowed-endpoints: >
          #   github.com:443
          #   registry.npmjs.org:443
          #   ghcr.io:443

# ARC Runner Scale Set values.yaml에 PVC 추가
template:
  spec:
    containers:
      - name: runner
        image: ghcr.io/my-org/actions-runner:latest
        volumeMounts:
          - name: cache-volume
            mountPath: /opt/cache
        env:
          - name: RUNNER_TOOL_CACHE
            value: /opt/cache/tool-cache
          - name: npm_config_cache
            value: /opt/cache/npm
          - name: GOPATH
            value: /opt/cache/go
    volumes:
      - name: cache-volume
        persistentVolumeClaim:
          claimName: runner-cache-pvc
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: runner-cache-pvc
  namespace: arc-runners
spec:
  accessModes:
    - ReadWriteMany # 여러 Runner Pod가 동시 접근
  storageClassName: efs # AWS EFS 또는 NFS
  resources:
    requests:
      storage: 100Gi

# MinIO 기반 캐시 서버 배포 (같은 VPC/리전 내)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: actions-cache-server
  namespace: arc-systems
spec:
  replicas: 1
  selector:
    matchLabels:
      app: actions-cache
  template:
    spec:
      containers:
        - name: minio
          image: minio/minio:latest
          args: ['server', '/data', '--console-address', ':9001']
          env:
            - name: MINIO_ROOT_USER
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: user
            - name: MINIO_ROOT_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: password
          volumeMounts:
            - name: data
              mountPath: /data
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: minio-data-pvc

# 워크플로우에서 BuildKit 레지스트리 캐시 활용
- name: Build and Push
  uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6
  with:
    push: true
    tags: ghcr.io/my-org/my-app:${{ github.sha }}
    cache-from: type=registry,ref=ghcr.io/my-org/my-app:buildcache
    cache-to: type=registry,ref=ghcr.io/my-org/my-app:buildcache,mode=max

# Prometheus ServiceMonitor 설정
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: arc-controller-monitor
  namespace: arc-systems
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: gha-runner-scale-set-controller
  endpoints:
    - port: metrics
      interval: 30s
      path: /metrics

# Alertmanager 규칙
groups:
  - name: arc-runner-alerts
    rules:
      # 러너 풀 고갈 경고
      - alert: RunnerPoolExhausted
        expr: |
          gha_runner_scale_set_desired_replicas
          >= gha_runner_scale_set_max_replicas * 0.9
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'Runner pool이 90% 이상 사용 중'
          description: 'maxRunners 증설 또는 워크플로우 최적화 필요'

      # 러너 등록 실패 감지
      - alert: RunnerRegistrationFailed
        expr: |
          rate(gha_runner_scale_set_registration_failures_total[5m]) > 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: 'Runner 등록 실패 발생'
          description: 'GitHub App 인증 또는 네트워크 확인 필요'

      # Pending Pod 장시간 대기
      - alert: RunnerPodPending
        expr: |
          kube_pod_status_phase{namespace="arc-runners", phase="Pending"} > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: 'Runner Pod가 10분 이상 Pending 상태'
          description: '노드 리소스 부족 또는 PVC 바인딩 실패 가능성'

# 1. Listener Pod 로그 확인
kubectl logs -n arc-systems -l app.kubernetes.io/component=runner-scale-set-listener --tail=100

# 2. 흔한 원인: GitHub App 인증 만료
# - Private key 파일 확인
# - App installation 상태 확인 (org settings > GitHub Apps)

# 3. 네트워크 문제: GitHub API 접근 불가
kubectl exec -n arc-systems deploy/arc-gha-runner-scale-set-controller -- \
  curl -s https://api.github.com/meta | jq '.actions[]'

kubectl create secret generic github-app-secret \
  --namespace arc-runners \
  --from-literal=github_app_id=12345 \
  --from-literal=github_app_installation_id=67890 \
  --from-file=github_app_private_key=./new-private-key.pem \
  --dry-run=client -o yaml | kubectl apply -f -

# Controller 재시작
kubectl rollout restart deployment -n arc-systems arc-gha-runner-scale-set-controller

# Pod 이벤트 확인
kubectl describe pod -n arc-runners -l actions.github.com/scale-set-name=arc-runner-set

# 흔한 원인별 대응
# 1. 노드 리소스 부족
kubectl top nodes
# -> Cluster Autoscaler가 동작하는지 확인, 또는 maxRunners 하향

# 2. PVC 바인딩 대기
kubectl get pvc -n arc-runners
# -> StorageClass 설정, 가용영역 불일치 확인

# 3. 이미지 Pull 실패
kubectl get events -n arc-runners --sort-by='.lastTimestamp' | grep -i pull
# -> 이미지 태그, 레지스트리 인증 확인

# Runner 등록 상태 확인
kubectl get ephemeralrunner -n arc-runners

# Runner labels 확인 (runs-on과 일치해야 함)
kubectl get autoscalingrunnersets -n arc-runners -o yaml | grep -A5 labels

# GitHub에서 Runner 그룹 설정 확인
# Settings > Actions > Runner groups > 해당 그룹에 리포지토리가 포함되어 있는지 확인

# 현재 Runner 버전 확인
kubectl exec -n arc-runners -it <runner-pod> -- ./config.sh --version

# 이미지 업데이트 (values.yaml 수정 후)
helm upgrade arc-runner-set \
  --namespace arc-runners \
  -f values.yaml \
  oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set

# 용도별 Runner Scale Set 분리
# 1. 일반 CI (가벼운 테스트, 린트)
# values-ci-light.yaml
minRunners: 2
maxRunners: 20
template:
  spec:
    containers:
      - name: runner
        resources:
          requests:
            cpu: "1"
            memory: "2Gi"

# 2. 빌드 전용 (컴파일, Docker 빌드)
# values-ci-build.yaml
minRunners: 1
maxRunners: 10
template:
  spec:
    containers:
      - name: runner
        resources:
          requests:
            cpu: "4"
            memory: "8Gi"

# 3. GPU 워크로드 (ML 모델 테스트)
# values-gpu.yaml
minRunners: 0
maxRunners: 4
template:
  spec:
    containers:
      - name: runner
        resources:
          limits:
            nvidia.com/gpu: 1
    nodeSelector:
      accelerator: nvidia-a10g

template:
  spec:
    terminationGracePeriodSeconds: 3600 # 최대 1시간 대기
    containers:
      - name: runner
        env:
          - name: RUNNER_GRACEFUL_STOP_TIMEOUT
            value: '3500' # terminationGracePeriodSeconds보다 약간 짧게

# Karpenter NodePool 예시 (CI Runner 전용)
apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: ci-runners
spec:
  template:
    metadata:
      labels:
        workload-type: ci-runner
    spec:
      taints:
        - key: ci-runner
          value: 'true'
          effect: NoSchedule
      requirements:
        - key: kubernetes.io/arch
          operator: In
          values: ['amd64']
        - key: karpenter.sh/capacity-type
          operator: In
          values: ['on-demand', 'spot']
        - key: node.kubernetes.io/instance-type
          operator: In
          values: ['m7i.xlarge', 'm7i.2xlarge', 'm6i.xlarge', 'm6i.2xlarge']
  limits:
    cpu: 200
    memory: 400Gi
  disruption:
    consolidationPolicy: WhenEmpty
    consolidateAfter: 60s