Kubernetes Gateway API Production Guide: From Ingress Migration to HTTPRoute, GRPCRoute, and Envoy Gateway Deployment

# Tier 1: GatewayClass - Managed by infrastructure provider (cluster-scoped)
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: envoy-gateway
spec:
  controllerName: gateway.envoyproxy.io/gatewayclass-controller
---
# Tier 2: Gateway - Managed by platform team (namespace-scoped)
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: production-gateway
  namespace: infra-gateway
spec:
  gatewayClassName: envoy-gateway
  listeners:
    - name: https
      protocol: HTTPS
      port: 443
      tls:
        mode: Terminate
        certificateRefs:
          - name: wildcard-tls-cert
    - name: http
      protocol: HTTP
      port: 80
---
# Tier 3: HTTPRoute - Managed by dev teams (per application namespace)
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: api-route
  namespace: app-team-a
spec:
  parentRefs:
    - name: production-gateway
      namespace: infra-gateway
      sectionName: https
  hostnames:
    - 'api.example.com'
  rules:
    - matches:
        - path:
            type: PathPrefix
            value: /v1/users
      backendRefs:
        - name: user-service
          port: 8080

# Install Envoy Gateway Helm chart
helm install eg oci://docker.io/envoyproxy/gateway-helm \
  --version v1.3.0 \
  -n envoy-gateway-system \
  --create-namespace \
  --set config.envoyGateway.logging.level.default=info

# Verify installation
kubectl get pods -n envoy-gateway-system
kubectl get gatewayclass

# Verify CRDs
kubectl get crd | grep gateway.networking.k8s.io

apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: envoy-production
spec:
  controllerName: gateway.envoyproxy.io/gatewayclass-controller
  parametersRef:
    group: gateway.envoyproxy.io
    kind: EnvoyProxy
    name: production-config
    namespace: envoy-gateway-system
---
apiVersion: gateway.envoyproxy.io/v1alpha1
kind: EnvoyProxy
metadata:
  name: production-config
  namespace: envoy-gateway-system
spec:
  provider:
    type: Kubernetes
    kubernetes:
      envoyDeployment:
        replicas: 3
        pod:
          annotations:
            prometheus.io/scrape: 'true'
            prometheus.io/port: '19001'
        container:
          resources:
            requests:
              cpu: '500m'
              memory: '512Mi'
            limits:
              cpu: '2000m'
              memory: '2Gi'
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: production-gateway
  namespace: infra-gateway
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
spec:
  gatewayClassName: envoy-production
  listeners:
    - name: https-wildcard
      protocol: HTTPS
      port: 443
      hostname: '*.example.com'
      tls:
        mode: Terminate
        certificateRefs:
          - name: wildcard-example-tls
      allowedRoutes:
        namespaces:
          from: Selector
          selector:
            matchLabels:
              gateway-access: 'true'
    - name: http-redirect
      protocol: HTTP
      port: 80
      allowedRoutes:
        namespaces:
          from: Same

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: app-routes
  namespace: production
spec:
  parentRefs:
    - name: production-gateway
      namespace: infra-gateway
      sectionName: https-wildcard
  hostnames:
    - 'app.example.com'
  rules:
    # Exact match - highest priority
    - matches:
        - path:
            type: Exact
            value: /healthz
      backendRefs:
        - name: health-check-service
          port: 8080
    # PathPrefix match - API version routing
    - matches:
        - path:
            type: PathPrefix
            value: /api/v2
      backendRefs:
        - name: api-v2-service
          port: 8080
    - matches:
        - path:
            type: PathPrefix
            value: /api/v1
      backendRefs:
        - name: api-v1-service
          port: 8080
    # RegularExpression match
    - matches:
        - path:
            type: RegularExpression
            value: '/users/[0-9]+/profile'
      backendRefs:
        - name: user-profile-service
          port: 8080

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: header-based-routing
  namespace: production
spec:
  parentRefs:
    - name: production-gateway
      namespace: infra-gateway
  hostnames:
    - 'api.example.com'
  rules:
    # Route to different backends based on header values
    - matches:
        - headers:
            - name: x-api-version
              value: 'beta'
            - name: x-user-tier
              value: 'premium'
      backendRefs:
        - name: api-beta-premium
          port: 8080
    # A/B testing via header-based routing
    - matches:
        - headers:
            - name: x-experiment-group
              value: 'treatment'
      backendRefs:
        - name: api-experiment
          port: 8080
    # Default routing (no match conditions)
    - backendRefs:
        - name: api-stable
          port: 8080

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: canary-route
  namespace: production
spec:
  parentRefs:
    - name: production-gateway
      namespace: infra-gateway
  hostnames:
    - 'app.example.com'
  rules:
    - matches:
        - path:
            type: PathPrefix
            value: /
      backendRefs:
        # Stable version: 90% traffic
        - name: app-stable
          port: 8080
          weight: 90
        # Canary version: 10% traffic
        - name: app-canary
          port: 8080
          weight: 10

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: redirect-and-rewrite
  namespace: production
spec:
  parentRefs:
    - name: production-gateway
      namespace: infra-gateway
  hostnames:
    - 'app.example.com'
  rules:
    # HTTP -> HTTPS redirect
    - matches:
        - path:
            type: PathPrefix
            value: /
      filters:
        - type: RequestRedirect
          requestRedirect:
            scheme: https
            statusCode: 301
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: url-rewrite
  namespace: production
spec:
  parentRefs:
    - name: production-gateway
      namespace: infra-gateway
  hostnames:
    - 'api.example.com'
  rules:
    # /old-api/* -> /new-api/* path rewrite
    - matches:
        - path:
            type: PathPrefix
            value: /old-api
      filters:
        - type: URLRewrite
          urlRewrite:
            path:
              type: ReplacePrefixMatch
              replacePrefixMatch: /new-api
      backendRefs:
        - name: api-service
          port: 8080

apiVersion: gateway.networking.k8s.io/v1
kind: GRPCRoute
metadata:
  name: order-service-route
  namespace: production
spec:
  parentRefs:
    - name: production-gateway
      namespace: infra-gateway
      sectionName: https-wildcard
  hostnames:
    - 'grpc.example.com'
  rules:
    # Service-based matching
    - matches:
        - method:
            service: 'order.v1.OrderService'
      backendRefs:
        - name: order-service-grpc
          port: 50051
    # Method-based matching
    - matches:
        - method:
            service: 'order.v1.OrderService'
            method: 'CreateOrder'
      backendRefs:
        - name: order-write-service
          port: 50051
    # Header-based matching
    - matches:
        - headers:
            - name: x-region
              value: 'asia'
          method:
            service: 'order.v1.OrderService'
      backendRefs:
        - name: order-service-asia
          port: 50051

apiVersion: gateway.networking.k8s.io/v1
kind: GRPCRoute
metadata:
  name: grpc-canary
  namespace: production
spec:
  parentRefs:
    - name: production-gateway
      namespace: infra-gateway
  hostnames:
    - 'grpc.example.com'
  rules:
    - matches:
        - method:
            service: 'payment.v1.PaymentService'
      backendRefs:
        - name: payment-service-stable
          port: 50051
          weight: 95
        - name: payment-service-canary
          port: 50051
          weight: 5

apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-prod
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: admin@example.com
    privateKeySecretRef:
      name: letsencrypt-prod-key
    solvers:
      - http01:
          gatewayHTTPRoute:
            parentRefs:
              - name: production-gateway
                namespace: infra-gateway
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: wildcard-example-tls
  namespace: infra-gateway
spec:
  secretName: wildcard-example-tls
  issuerRef:
    name: letsencrypt-prod
    kind: ClusterIssuer
  dnsNames:
    - '*.example.com'
    - 'example.com'

apiVersion: gateway.networking.k8s.io/v1alpha3
kind: BackendTLSPolicy
metadata:
  name: backend-mtls
  namespace: production
spec:
  targetRefs:
    - group: ''
      kind: Service
      name: secure-backend
  validation:
    caCertificateRefs:
      - name: backend-ca-cert
        group: ''
        kind: ConfigMap
    hostname: secure-backend.production.svc.cluster.local

apiVersion: gateway.envoyproxy.io/v1alpha1
kind: BackendTrafficPolicy
metadata:
  name: global-rate-limit
  namespace: infra-gateway
spec:
  targetRefs:
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: production-gateway
  rateLimit:
    type: Global
    global:
      rules:
        - clientSelectors:
            - headers:
                - name: x-api-key
                  type: Distinct
          limit:
            requests: 100
            unit: Minute
        - limit:
            requests: 1000
            unit: Minute

apiVersion: gateway.envoyproxy.io/v1alpha1
kind: BackendTrafficPolicy
metadata:
  name: api-rate-limit
  namespace: production
spec:
  targetRefs:
    - group: gateway.networking.k8s.io
      kind: HTTPRoute
      name: api-route
  rateLimit:
    type: Global
    global:
      rules:
        - clientSelectors:
            - headers:
                - name: x-user-tier
                  value: 'free'
          limit:
            requests: 10
            unit: Minute
        - clientSelectors:
            - headers:
                - name: x-user-tier
                  value: 'premium'
          limit:
            requests: 1000
            unit: Minute

# Allow HTTPRoutes in app-team-a namespace to reference
# Services in shared-services namespace
apiVersion: gateway.networking.k8s.io/v1beta1
kind: ReferenceGrant
metadata:
  name: allow-cross-ns-routing
  namespace: shared-services
spec:
  from:
    - group: gateway.networking.k8s.io
      kind: HTTPRoute
      namespace: app-team-a
    - group: gateway.networking.k8s.io
      kind: HTTPRoute
      namespace: app-team-b
  to:
    - group: ''
      kind: Service

# Install ingress2gateway
go install github.com/kubernetes-sigs/ingress2gateway@latest

# Convert existing Ingress resources to Gateway API
ingress2gateway print \
  --input-file existing-ingress.yaml \
  --providers ingress-nginx \
  --all-resources

# Convert directly from cluster
ingress2gateway print \
  --providers ingress-nginx \
  --all-resources \
  --namespace production

# Existing Ingress (keep running)
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: legacy-app-ingress
  namespace: production
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
spec:
  ingressClassName: nginx
  tls:
    - hosts:
        - app.example.com
      secretName: app-tls
  rules:
    - host: app.example.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: app-service
                port:
                  number: 8080
---
# New Gateway API HTTPRoute (parallel deployment)
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: app-route
  namespace: production
spec:
  parentRefs:
    - name: production-gateway
      namespace: infra-gateway
  hostnames:
    - 'app.example.com'
  rules:
    - matches:
        - path:
            type: PathPrefix
            value: /
      backendRefs:
        - name: app-service
          port: 8080

# Step 1: Verify Gateway API CRDs are installed
kubectl get crd gateways.gateway.networking.k8s.io

# Step 2: Install Envoy Gateway
helm install eg oci://docker.io/envoyproxy/gateway-helm \
  --version v1.3.0 \
  -n envoy-gateway-system --create-namespace

# Step 3: Create GatewayClass and Gateway
kubectl apply -f gateway-class.yaml
kubectl apply -f gateway.yaml

# Step 4: Convert existing Ingress to HTTPRoute
ingress2gateway print --providers ingress-nginx --all-resources > routes.yaml

# Step 5: Deploy converted HTTPRoutes (parallel operation)
kubectl apply -f routes.yaml

# Step 6: Switch DNS to new Gateway LoadBalancer
GATEWAY_IP=$(kubectl get gateway production-gateway -n infra-gateway \
  -o jsonpath='{.status.addresses[0].value}')
echo "Update DNS A record to: $GATEWAY_IP"

# Step 7: Monitor traffic then remove legacy Ingress
kubectl delete ingress legacy-app-ingress -n production

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: envoy-gateway-metrics
  namespace: envoy-gateway-system
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: envoy
  endpoints:
    - port: metrics
      interval: 15s
      path: /stats/prometheus
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: gateway-api-alerts
  namespace: monitoring
spec:
  groups:
    - name: gateway-api
      rules:
        - alert: GatewayHighErrorRate
          expr: |
            sum(rate(envoy_http_downstream_rq_xx{envoy_response_code_class="5"}[5m])) by (envoy_http_conn_manager_prefix)
            /
            sum(rate(envoy_http_downstream_rq_total[5m])) by (envoy_http_conn_manager_prefix)
            > 0.05
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: 'Gateway error rate exceeds 5%'
        - alert: GatewayHighLatency
          expr: |
            histogram_quantile(0.99,
              sum(rate(envoy_http_downstream_rq_time_bucket[5m])) by (le, envoy_http_conn_manager_prefix)
            ) > 1000
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: 'Gateway p99 latency exceeds 1s'

# Key Envoy metrics
# Requests per second
sum(rate(envoy_http_downstream_rq_total[5m])) by (envoy_http_conn_manager_prefix)

# Error rate (5xx)
sum(rate(envoy_http_downstream_rq_xx{envoy_response_code_class="5"}[5m]))

# Average response time
sum(rate(envoy_http_downstream_rq_time_sum[5m])) / sum(rate(envoy_http_downstream_rq_time_count[5m]))

# Active connections
envoy_http_downstream_cx_active

# Symptom: 503 errors, TLS handshake failures
# Diagnosis
kubectl get certificate -n infra-gateway
kubectl describe certificate wildcard-example-tls -n infra-gateway

# Recovery: Force cert-manager renewal
kubectl delete certificaterequest -n infra-gateway --all
kubectl annotate certificate wildcard-example-tls \
  -n infra-gateway \
  cert-manager.io/renew-before="720h" --overwrite

# Emergency: Manual certificate replacement
kubectl create secret tls wildcard-example-tls \
  --cert=fullchain.pem --key=privkey.pem \
  -n infra-gateway --dry-run=client -o yaml | kubectl apply -f -

# Symptom: New HTTPRoutes not being applied
# Diagnosis
kubectl get pods -n envoy-gateway-system
kubectl logs -n envoy-gateway-system deploy/envoy-gateway -f

# Check Envoy Proxy status
kubectl get pods -l app.kubernetes.io/name=envoy -A

# Recovery: Restart controller
kubectl rollout restart deployment/envoy-gateway -n envoy-gateway-system

# Verify Gateway status
kubectl get gateway production-gateway -n infra-gateway -o yaml

# Symptom: All requests to a specific path return 404
# Diagnosis: Check HTTPRoute status
kubectl get httproute -A
kubectl describe httproute app-route -n production

# Check Accepted/ResolvedRefs conditions in status
# Cause: backendRef points to a non-existent service

# Recovery: Fix service name and reapply
kubectl apply -f corrected-httproute.yaml

# Verify Envoy config sync
kubectl exec -n envoy-gateway-system deploy/envoy-gateway -- \
  curl -s localhost:19000/config_dump | python3 -m json.tool | head -100

# Symptom: Normal users receiving 429 Too Many Requests
# Diagnosis
kubectl get backendtrafficpolicy -A
kubectl describe backendtrafficpolicy global-rate-limit -n infra-gateway

# Check Redis-based global rate limiter status
kubectl logs -n envoy-gateway-system deploy/envoy-ratelimit

# Recovery: Temporarily remove rate limiting policy
kubectl delete backendtrafficpolicy global-rate-limit -n infra-gateway

# Reapply corrected policy after stabilization
kubectl apply -f corrected-rate-limit.yaml