Fluent Bit Complete Guide: Architecture, Configuration, and Kubernetes Integration for the Lightweight Log Processor

+------------------------------------------------------------------+
|                        Fluent Bit Engine                         |
|                                                                  |
|  +--------+   +--------+   +--------+   +--------+   +--------+ |
|  | Input  |-->| Parser |-->| Filter |-->| Buffer |-->| Output | |
|  +--------+   +--------+   +--------+   +--------+   +--------+ |
|  | tail   |   | json   |   | k8s    |   | memory |   | es     | |
|  | systemd|   | regex  |   | grep   |   | filesys|   | loki   | |
|  | forward|   | logfmt |   | modify |   |        |   | s3     | |
|  | http   |   | cri    |   | lua    |   |        |   | kafka  | |
|  | tcp    |   | docker |   | nest   |   |        |   | stdout | |
|  +--------+   +--------+   +--------+   +--------+   +--------+ |
|                                                                  |
|  [Scheduler]  [Router / Tag Matching]  [HTTP Server / Monitoring]|
+------------------------------------------------------------------+

  [Data Source]
       |
       v
  +---------+     +---------+     +---------+     +---------+     +---------+
  |  INPUT  | --> | PARSER  | --> | FILTER  | --> | BUFFER  | --> | OUTPUT  |
  |         |     |         |     |         |     |         |     |         |
  | Data    |     | Unstruc |     | Data    |     | Memory/ |     | Final   |
  | Collect |     | -> Struc|     | Process |     | Disk    |     | Deliver |
  +---------+     +---------+     +---------+     +---------+     +---------+
       |               |               |               |               |
    Tag assign    Structuring      Enrichment      Reliability     Destination
                  conversion       Filtering       guarantee       (Tag match)

Pipeline A: [tail: app-*.log] --tag:app--> [filter:k8s] --> [output:elasticsearch]
Pipeline B: [tail: sys-*.log] --tag:sys--> [filter:grep] --> [output:loki]
Pipeline C: [forward:24224]   --tag:fwd--> [filter:lua]  --> [output:s3]

# Add GPG key and repository
curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh

# Or manual installation
wget -qO - https://packages.fluentbit.io/fluentbit.key | sudo apt-key add -
echo "deb https://packages.fluentbit.io/ubuntu/$(lsb_release -cs) $(lsb_release -cs) main" | \
  sudo tee /etc/apt/sources.list.d/fluent-bit.list

sudo apt-get update
sudo apt-get install -y fluent-bit

# Start service
sudo systemctl start fluent-bit
sudo systemctl enable fluent-bit

cat > /etc/yum.repos.d/fluent-bit.repo << 'EOF'
[fluent-bit]
name=Fluent Bit
baseurl=https://packages.fluentbit.io/centos/$releasever/
gpgcheck=1
gpgkey=https://packages.fluentbit.io/fluentbit.key
enabled=1
EOF

sudo yum install -y fluent-bit
sudo systemctl start fluent-bit
sudo systemctl enable fluent-bit

# Run latest version
docker run -ti cr.fluentbit.io/fluent/fluent-bit:latest

# Mount configuration file
docker run -ti \
  -v /path/to/fluent-bit.yaml:/fluent-bit/etc/fluent-bit.yaml \
  -v /var/log:/var/log \
  cr.fluentbit.io/fluent/fluent-bit:latest \
  /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.yaml

# Docker Compose example
cat > docker-compose.yaml << 'EOF'
version: '3.8'
services:
  fluent-bit:
    image: cr.fluentbit.io/fluent/fluent-bit:latest
    volumes:
      - ./fluent-bit.yaml:/fluent-bit/etc/fluent-bit.yaml
      - /var/log:/var/log:ro
    ports:
      - "2020:2020"   # HTTP monitoring
      - "24224:24224"  # Forward protocol
EOF

brew install fluent-bit

# Run
fluent-bit -c /opt/homebrew/etc/fluent-bit/fluent-bit.conf

# Or run with YAML configuration file
fluent-bit -c /path/to/fluent-bit.yaml

# Add Helm repository
helm repo add fluent https://fluent.github.io/helm-charts
helm repo update

# Basic installation
helm install fluent-bit fluent/fluent-bit \
  --namespace logging \
  --create-namespace

# Install with custom values.yaml
helm install fluent-bit fluent/fluent-bit \
  --namespace logging \
  --create-namespace \
  -f custom-values.yaml

# Upgrade
helm upgrade fluent-bit fluent/fluent-bit \
  --namespace logging \
  -f custom-values.yaml

# Download binary from GitHub Releases
FLUENT_BIT_VERSION=3.2.2
wget https://github.com/fluent/fluent-bit/releases/download/v${FLUENT_BIT_VERSION}/fluent-bit-${FLUENT_BIT_VERSION}-linux-x86_64.tar.gz

tar xzf fluent-bit-${FLUENT_BIT_VERSION}-linux-x86_64.tar.gz
cd fluent-bit-${FLUENT_BIT_VERSION}-linux-x86_64/

# Run
./bin/fluent-bit -c conf/fluent-bit.yaml

# Check version
./bin/fluent-bit --version

# fluent-bit.conf - Classic INI format

[SERVICE]
    Flush         5
    Daemon        Off
    Log_Level     info
    Parsers_File  parsers.conf
    HTTP_Server   On
    HTTP_Listen   0.0.0.0
    HTTP_Port     2020
    Hot_Reload    On

[INPUT]
    Name          tail
    Path          /var/log/containers/*.log
    Parser        cri
    Tag           kube.*
    Mem_Buf_Limit 5MB
    Skip_Long_Lines On
    Refresh_Interval 10
    DB            /var/log/flb_kube.db

[FILTER]
    Name          kubernetes
    Match         kube.*
    Kube_URL      https://kubernetes.default.svc:443
    Kube_CA_File  /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
    Merge_Log     On
    K8S-Logging.Parser On
    K8S-Logging.Exclude On

[OUTPUT]
    Name          es
    Match         kube.*
    Host          elasticsearch.logging.svc.cluster.local
    Port          9200
    Logstash_Format On
    Logstash_Prefix kube
    Retry_Limit   False

# fluent-bit.yaml - YAML format (v3.2+ standard)
service:
  flush: 5
  daemon: off
  log_level: info
  parsers_file: parsers.conf
  http_server: on
  http_listen: 0.0.0.0
  http_port: 2020
  hot_reload: on

pipeline:
  inputs:
    - name: tail
      path: /var/log/containers/*.log
      parser: cri
      tag: kube.*
      mem_buf_limit: 5MB
      skip_long_lines: on
      refresh_interval: 10
      db: /var/log/flb_kube.db

  filters:
    - name: kubernetes
      match: kube.*
      kube_url: https://kubernetes.default.svc:443
      kube_ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      kube_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      merge_log: on
      k8s-logging.parser: on
      k8s-logging.exclude: on

  outputs:
    - name: es
      match: kube.*
      host: elasticsearch.logging.svc.cluster.local
      port: 9200
      logstash_format: on
      logstash_prefix: kube
      retry_limit: false

# YAML environment variable example
pipeline:
  outputs:
    - name: es
      match: '*'
      host: ${ELASTICSEARCH_HOST}
      port: ${ELASTICSEARCH_PORT}
      http_user: ${ES_USER}
      http_passwd: ${ES_PASSWORD}
      tls: ${ES_TLS_ENABLED}

# Classic environment variable example
[OUTPUT]
    Name          es
    Match         *
    Host          ${ELASTICSEARCH_HOST}
    Port          ${ELASTICSEARCH_PORT}
    HTTP_User     ${ES_USER}
    HTTP_Passwd   ${ES_PASSWORD}

pipeline:
  outputs:
    - name: es
      http_passwd: file:///run/secrets/es-password

# fluent-bit.conf (Classic)
[SERVICE]
    Flush 5

@INCLUDE inputs.conf
@INCLUDE filters.conf
@INCLUDE outputs.conf

# fluent-bit.yaml
includes:
  - inputs.yaml
  - filters.yaml
  - outputs.yaml

service:
  flush: 5

pipeline:
  inputs:
    - name: tail
      tag: app.logs
      path: /var/log/app/*.log
      path_key: filename # Include file path in records
      exclude_path: /var/log/app/debug.log # Exclude specific files
      parser: json # Default parser
      db: /var/log/flb_app.db # Offset storage DB (resume after restart)
      db.sync: normal # DB sync mode
      refresh_interval: 10 # File list refresh interval (seconds)
      read_from_head: false # true: read from beginning of file
      skip_long_lines: on # Skip very long lines
      mem_buf_limit: 5MB # Memory buffer limit
      rotate_wait: 5 # Rotation wait time (seconds)
      multiline.parser: docker, cri # Multiline parser

pipeline:
  inputs:
    - name: systemd
      tag: host.systemd
      systemd_filter: _SYSTEMD_UNIT=docker.service
      systemd_filter: _SYSTEMD_UNIT=kubelet.service
      read_from_tail: on
      strip_underscores: on
      db: /var/log/flb_systemd.db

pipeline:
  inputs:
    - name: forward
      tag: forward.incoming
      listen: 0.0.0.0
      port: 24224
      buffer_chunk_size: 1M
      buffer_max_size: 6M

pipeline:
  inputs:
    # HTTP reception
    - name: http
      tag: http.logs
      listen: 0.0.0.0
      port: 9880
      successful_response_code: 201

    # TCP reception
    - name: tcp
      tag: tcp.logs
      listen: 0.0.0.0
      port: 5170
      format: json

    # UDP reception
    - name: udp
      tag: udp.logs
      listen: 0.0.0.0
      port: 5170
      format: json

pipeline:
  inputs:
    - name: kubernetes_events
      tag: kube.events
      kube_url: https://kubernetes.default.svc:443
      kube_ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      kube_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      interval_sec: 1
      retention_time: 1h

pipeline:
  inputs:
    # Node metrics collection
    - name: node_exporter_metrics
      tag: node.metrics
      scrape_interval: 30

    # Prometheus endpoint scraping
    - name: prometheus_scrape
      tag: prom.metrics
      host: 127.0.0.1
      port: 9090
      metrics_path: /metrics
      scrape_interval: 10s

pipeline:
  inputs:
    - name: fluentbit_metrics
      tag: fb.metrics
      scrape_interval: 30
      scrape_on_start: true

# parsers.conf

# Nginx error log parser
[PARSER]
    Name        nginx_error
    Format      regex
    Regex       ^(?<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?<level>\w+)\] (?<pid>\d+).(?<tid>\d+): (?<message>.*)$
    Time_Key    time
    Time_Format %Y/%m/%d %H:%M:%S

# Spring Boot log parser
[PARSER]
    Name        spring_boot
    Format      regex
    Regex       ^(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+)\s+(?<level>\w+)\s+(?<pid>\d+)\s+---\s+\[(?<thread>[^\]]+)\]\s+(?<logger>\S+)\s+:\s+(?<message>.*)$
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%L

# Apache Combined log parser
[PARSER]
    Name        apache_combined
    Format      regex
    Regex       ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
    Time_Key    time
    Time_Format %d/%b/%Y:%H:%M:%S %z

# parsers.yaml
parsers:
  - name: nginx_error
    format: regex
    regex: '^(?<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?<level>\w+)\] (?<pid>\d+).(?<tid>\d+): (?<message>.*)$'
    time_key: time
    time_format: '%Y/%m/%d %H:%M:%S'

  - name: spring_boot
    format: regex
    regex: '^(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+)\s+(?<level>\w+)\s+(?<pid>\d+)\s+---\s+\[(?<thread>[^\]]+)\]\s+(?<logger>\S+)\s+:\s+(?<message>.*)$'
    time_key: time
    time_format: '%Y-%m-%dT%H:%M:%S.%L'

# Multiline parser definition
[MULTILINE_PARSER]
    Name          java_stacktrace
    Type          regex
    Flush_Timeout 1000
    # First line pattern: starts with timestamp
    Rule          "start_state"  "/^\d{4}-\d{2}-\d{2}/"  "cont"
    # Continuation line pattern: starts with whitespace or Caused by
    Rule          "cont"         "/^\s+|^Caused by:/"      "cont"

[MULTILINE_PARSER]
    Name          python_traceback
    Type          regex
    Flush_Timeout 1000
    Rule          "start_state"  "/^Traceback/"            "python_tb"
    Rule          "python_tb"    "/^\s+/"                   "python_tb"
    Rule          "python_tb"    "/^\w+Error/"              "end"

pipeline:
  inputs:
    - name: tail
      tag: app.java
      path: /var/log/app/application.log
      multiline.parser: java_stacktrace
      read_from_head: true

[PARSER]
    Name        custom_time
    Format      regex
    Regex       ^(?<time>[^ ]+) (?<message>.*)$
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep   On    # Keep the time field after conversion
    Time_Offset +0900  # KST timezone

# Configuration for parser testing
service:
  flush: 1
  log_level: debug

pipeline:
  inputs:
    - name: tail
      tag: test
      path: /tmp/test.log
      parser: my_custom_parser

  outputs:
    - name: stdout
      match: test
      format: json_lines

# Generate test log and verify
echo '2026-03-01T12:00:00.000Z ERROR [main] App - Connection failed' >> /tmp/test.log
fluent-bit -c test.yaml

pipeline:
  filters:
    - name: kubernetes
      match: kube.*
      kube_url: https://kubernetes.default.svc:443
      kube_ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      kube_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      merge_log: on # Merge JSON logs to top level
      merge_log_key: log_parsed # Merge key name
      keep_log: off # Remove original log field
      k8s-logging.parser: on # Use parser settings from Pod annotations
      k8s-logging.exclude: on # Exclude logs via Pod annotations
      labels: on # Include Pod Labels
      annotations: off # Whether to include Pod Annotations
      buffer_size: 0 # API response buffer (0=unlimited)
      kube_meta_cache_ttl: 300 # Metadata cache TTL (seconds)
      use_kubelet: false # Whether to use kubelet API

{
  "log": "Connection established to database",
  "kubernetes": {
    "pod_name": "api-server-7d4b8c6f5-x2k9j",
    "namespace_name": "production",
    "pod_id": "abc-123-def",
    "container_name": "api-server",
    "container_image": "myregistry/api-server:v2.1.0",
    "labels": {
      "app": "api-server",
      "version": "v2.1.0",
      "team": "backend"
    },
    "host": "node-01"
  }
}

pipeline:
  filters:
    - name: modify
      match: "*"
      # Add fields
      add: environment production
      add: cluster_name main-cluster
      # Rename fields
      rename: log message
      rename: stream source
      # Remove fields
      remove: unwanted_field
      # Conditional field addition (only when key doesn't exist)
      set: default_level INFO
      # Hard copy
      copy: source source_backup

pipeline:
  filters:
    # Pass only ERROR or WARN levels
    - name: grep
      match: app.*
      regex: level (ERROR|WARN)

    # Exclude healthcheck paths
    - name: grep
      match: access.*
      exclude: path /health

    # Pass only specific namespaces
    - name: grep
      match: kube.*
      regex: $kubernetes['namespace_name'] ^(production|staging)$

    # Combine multiple conditions (AND)
    - name: grep
      match: app.*
      regex: level ERROR
      regex: message .*timeout.*

pipeline:
  filters:
    - name: record_modifier
      match: "*"
      record: hostname ${HOSTNAME}
      record: service_name my-application
      remove_key: unnecessary_field
      allowlist_key: timestamp
      allowlist_key: level
      allowlist_key: message

pipeline:
  filters:
    # Nest: flat -> nested
    - name: nest
      match: '*'
      operation: nest
      wildcard: 'app_*'
      nest_under: application
      # app_name, app_version -> application: { name, version }

    # Lift: nested -> flat
    - name: nest
      match: '*'
      operation: lift
      nested_under: kubernetes
      # kubernetes: { pod_name, namespace } -> pod_name, namespace

pipeline:
  filters:
    - name: lua
      match: app.*
      script: /fluent-bit/scripts/transform.lua
      call: process_log

-- /fluent-bit/scripts/transform.lua

function process_log(tag, timestamp, record)
    -- Normalize log level
    if record["level"] then
        record["level"] = string.upper(record["level"])
    end

    -- Mask sensitive information
    if record["message"] then
        record["message"] = string.gsub(
            record["message"],
            "%d%d%d%d%-%d%d%d%d%-%d%d%d%d%-%d%d%d%d",
            "****-****-****-****"
        )
    end

    -- Add grade based on response time
    if record["response_time"] then
        local rt = tonumber(record["response_time"])
        if rt > 5000 then
            record["performance"] = "critical"
        elseif rt > 1000 then
            record["performance"] = "slow"
        else
            record["performance"] = "normal"
        end
    end

    -- Add timestamp field
    record["processed_at"] = os.date("!%Y-%m-%dT%H:%M:%SZ")

    -- 2 = MODIFIED
    return 2, timestamp, record
end

pipeline:
  filters:
    - name: rewrite_tag
      match: kube.*
      rule: $kubernetes['namespace_name'] ^(production)$ prod.$TAG false
      rule: $kubernetes['namespace_name'] ^(staging)$    stg.$TAG  false
      rule: $level                        ^(ERROR)$      alert.$TAG false

pipeline:
  filters:
    - name: throttle
      match: app.*
      rate: 1000 # Allowed per window
      window: 5 # Window size (seconds)
      interval: 1s # Evaluation interval
      print_status: true # Print status

pipeline:
  filters:
    - name: multiline
      match: app.*
      multiline.parser: java_stacktrace
      multiline.key_content: log

pipeline:
  outputs:
    # Elasticsearch
    - name: es
      match: kube.*
      host: ${ES_HOST}
      port: 9200
      index: logs
      type: _doc
      http_user: ${ES_USER}
      http_passwd: ${ES_PASSWORD}
      logstash_format: on
      logstash_prefix: kube-logs
      logstash_dateformat: %Y.%m.%d
      time_key: '@timestamp'
      include_tag_key: true
      tag_key: fluentbit_tag
      generate_id: on # Generate deduplication ID
      buffer_size: 512KB
      tls: on
      tls.verify: on
      tls.ca_file: /certs/ca.pem
      retry_limit: 5
      workers: 2
      suppress_type_name: on # ES 8.x compatibility

    # OpenSearch
    - name: opensearch
      match: app.*
      host: opensearch.logging.svc
      port: 9200
      index: app-logs
      http_user: admin
      http_passwd: ${OPENSEARCH_PASSWORD}
      tls: on
      suppress_type_name: on
      trace_output: off

pipeline:
  outputs:
    - name: loki
      match: kube.*
      host: loki-gateway.logging.svc
      port: 3100
      uri: /loki/api/v1/push
      tenant_id: my-tenant
      labels: job=fluent-bit
      label_keys: $kubernetes['namespace_name'],$kubernetes['pod_name'],$kubernetes['container_name']
      label_map_path: /fluent-bit/etc/loki-labelmap.json
      remove_keys: kubernetes,stream
      auto_kubernetes_labels: on
      line_format: json
      drop_single_key: on
      http_user: ${LOKI_USER}
      http_passwd: ${LOKI_PASSWORD}
      tls: on
      tls.verify: on
      workers: 2

{
  "kubernetes": {
    "namespace_name": "namespace",
    "pod_name": "pod",
    "container_name": "container",
    "labels": {
      "app": "app"
    }
  },
  "stream": "stream"
}

pipeline:
  outputs:
    - name: s3
      match: archive.*
      region: ap-northeast-2
      bucket: my-log-bucket
      s3_key_format: /logs/$TAG/%Y/%m/%d/%H/$UUID.gz
      s3_key_format_tag_delimiters: .
      total_file_size: 50M # Upload when this size is reached
      upload_timeout: 10m # Upload after this time even if size not met
      use_put_object: on
      compression: gzip
      content_type: application/gzip
      store_dir: /tmp/fluent-bit-s3 # Local buffer directory
      store_dir_limit_size: 512M
      retry_limit: 5
      # When using IAM roles (IRSA / EKS Pod Identity)
      role_arn: arn:aws:iam::123456789012:role/fluent-bit-s3
      # Static credentials (not recommended)
      # endpoint:
      # sts_endpoint:

pipeline:
  outputs:
    - name: kafka
      match: app.*
      brokers: kafka-0:9092,kafka-1:9092,kafka-2:9092
      topics: application-logs
      timestamp_key: '@timestamp'
      timestamp_format: iso8601
      format: json
      message_key: log
      queue_full_retries: 10
      rdkafka.request.required.acks: 1
      rdkafka.log.connection.close: false
      rdkafka.compression.codec: snappy

pipeline:
  outputs:
    - name: cloudwatch_logs
      match: kube.*
      region: ap-northeast-2
      log_group_name: /eks/my-cluster/containers
      log_group_template: /eks/my-cluster/$kubernetes['namespace_name']
      log_stream_prefix: from-fluent-bit-
      log_stream_template: $kubernetes['pod_name']
      auto_create_group: on
      extra_user_agent: fluent-bit
      retry_limit: 5
      workers: 1

pipeline:
  outputs:
    - name: stdout
      match: '*'
      format: json_lines # json_lines | msgpack

pipeline:
  outputs:
    - name: forward
      match: '*'
      host: fluentd-aggregator.logging.svc
      port: 24224
      time_as_integer: off
      send_options: true
      require_ack_response: true
      # TLS
      tls: on
      tls.verify: on
      tls.ca_file: /certs/ca.pem
      # Shared key authentication
      shared_key: my-shared-secret
      self_hostname: fluent-bit-node01

pipeline:
  outputs:
    - name: prometheus_exporter
      match: metrics.*
      host: 0.0.0.0
      port: 2021
      add_label: app fluent-bit

service:
  flush: 1
  log_level: info
  # Enable filesystem buffer
  storage.path: /var/log/fluent-bit/buffer/
  storage.sync: normal # normal | full
  storage.checksum: off # Data integrity verification
  storage.max_chunks_up: 128 # Max chunks to keep in memory
  storage.backlog.mem_limit: 5M # Backlog memory limit
  storage.metrics: on # Enable storage metrics

pipeline:
  inputs:
    # Memory buffer (default)
    - name: tail
      tag: app.mem
      path: /var/log/app/*.log
      mem_buf_limit: 10MB # Memory buffer limit

    # Filesystem buffer
    - name: tail
      tag: app.fs
      path: /var/log/app/*.log
      storage.type: filesystem # filesystem | memory
      storage.pause_on_chunks_overlimit: off # Continue writing to disk when limit exceeded

pipeline:
  outputs:
    - name: es
      match: '*'
      host: elasticsearch
      port: 9200
      storage.total_limit_size: 1G # Per-Output filesystem buffer total limit
      retry_limit: 10 # Retry count
      # retry_limit: false          # Unlimited retries

  [Input: Data Collection]
         |
         v
  [Store in Memory Buffer]
         |
    Memory limit reached?
    /              \
   No              Yes
   |                |
   v                v
 [Continue]   storage.type=filesystem?
                /              \
              No               Yes
              |                 |
              v                 v
       [Pause Input]       [Write to Disk]
       (Risk of data loss)      |
                          Disk limit reached?
                          /              \
                        No              Yes
                        |                |
                        v                v
                   [Continue]     [Pause Input]

service:
  storage.path: /var/log/fluent-bit/buffer/
  storage.sync: normal
  storage.max_chunks_up: 128
  storage.backlog.mem_limit: 10M

pipeline:
  inputs:
    - name: tail
      tag: kube.*
      path: /var/log/containers/*.log
      storage.type: filesystem
      storage.pause_on_chunks_overlimit: off

  outputs:
    - name: es
      match: kube.*
      host: elasticsearch
      storage.total_limit_size: 5G
      retry_limit: false
      net.keepalive: on
      net.keepalive_idle_timeout: 30

+---------------------------------------------------------------------+
|                      Kubernetes Cluster                             |
|                                                                     |
|  +-------------------+  +-------------------+  +-------------------+|
|  |     Node 1        |  |     Node 2        |  |     Node 3        ||
|  |                    |  |                    |  |                    ||
|  | +------+ +------+ |  | +------+ +------+ |  | +------+ +------+ ||
|  | |Pod A | |Pod B | |  | |Pod C | |Pod D | |  | |Pod E | |Pod F | ||
|  | +--+---+ +--+---+ |  | +--+---+ +--+---+ |  | +--+---+ +--+---+ ||
|  |    |        |      |  |    |        |      |  |    |        |      ||
|  |    v        v      |  |    v        v      |  |    v        v      ||
|  | /var/log/containers|  | /var/log/containers|  | /var/log/containers||
|  |         |          |  |         |          |  |         |          ||
|  |    +----+-----+    |  |    +----+-----+    |  |    +----+-----+    ||
|  |    |Fluent Bit|    |  |    |Fluent Bit|    |  |    |Fluent Bit|    ||
|  |    |(DaemonSet)|   |  |    |(DaemonSet)|   |  |    |(DaemonSet)|   ||
|  |    +----+-----+    |  |    +----+-----+    |  |    +----+-----+    ||
|  +---------|----------+  +---------|----------+  +---------|----------+|
|            |                       |                       |           |
|            +----------+------------+----------+------------+           |
|                       |                       |                        |
|               +-------v--------+     +--------v--------+              |
|               | Elasticsearch  |     |  Grafana Loki   |              |
|               +----------------+     +-----------------+              |
+---------------------------------------------------------------------+

# Add repository
helm repo add fluent https://fluent.github.io/helm-charts
helm repo update

# Basic installation
helm install fluent-bit fluent/fluent-bit \
  --namespace logging \
  --create-namespace

# Verify installation
kubectl get pods -n logging -l app.kubernetes.io/name=fluent-bit
kubectl get ds -n logging

# custom-values.yaml
kind: DaemonSet

image:
  repository: cr.fluentbit.io/fluent/fluent-bit
  tag: '3.2.2'
  pullPolicy: IfNotPresent

resources:
  requests:
    cpu: 100m
    memory: 128Mi
  limits:
    cpu: 500m
    memory: 256Mi

tolerations:
  - operator: Exists # Deploy on all nodes (including master)

serviceAccount:
  create: true
  annotations:
    # IRSA (EKS)
    eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/fluent-bit

# Volume mounts
volumeMounts:
  - name: varlog
    mountPath: /var/log
  - name: varlibdockercontainers
    mountPath: /var/lib/docker/containers
    readOnly: true
  - name: etcmachineid
    mountPath: /etc/machine-id
    readOnly: true

volumes:
  - name: varlog
    hostPath:
      path: /var/log
  - name: varlibdockercontainers
    hostPath:
      path: /var/lib/docker/containers
  - name: etcmachineid
    hostPath:
      path: /etc/machine-id

# Environment variables
env:
  - name: ELASTICSEARCH_HOST
    value: 'elasticsearch-master.logging.svc.cluster.local'
  - name: ELASTICSEARCH_PORT
    value: '9200'

# Fluent Bit configuration
config:
  service: |
    [SERVICE]
        Flush         5
        Log_Level     info
        Daemon        off
        Parsers_File  /fluent-bit/etc/parsers.conf
        HTTP_Server   On
        HTTP_Listen   0.0.0.0
        HTTP_Port     2020
        storage.path  /var/log/fluent-bit/buffer/
        storage.sync  normal
        storage.max_chunks_up 128
        Hot_Reload    On

  inputs: |
    [INPUT]
        Name              tail
        Tag               kube.*
        Path              /var/log/containers/*.log
        multiline.parser  docker, cri
        DB                /var/log/flb_kube.db
        Mem_Buf_Limit     5MB
        Skip_Long_Lines   On
        Refresh_Interval  10
        storage.type      filesystem

  filters: |
    [FILTER]
        Name                kubernetes
        Match               kube.*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     kube.var.log.containers.
        Merge_Log           On
        Keep_Log            Off
        K8S-Logging.Parser  On
        K8S-Logging.Exclude On
        Labels              On
        Annotations         Off

  outputs: |
    [OUTPUT]
        Name            es
        Match           kube.*
        Host            ${ELASTICSEARCH_HOST}
        Port            ${ELASTICSEARCH_PORT}
        Logstash_Format On
        Logstash_Prefix kube
        Retry_Limit     False
        Suppress_Type_Name On

# ServiceMonitor for Prometheus metrics collection
serviceMonitor:
  enabled: true
  interval: 30s
  scrapeTimeout: 10s

# Liveness/Readiness Probe
livenessProbe:
  httpGet:
    path: /
    port: http
readinessProbe:
  httpGet:
    path: /api/v1/health
    port: http

/var/log/containers/<pod-name>_<namespace>_<container-name>-<container-id>.log
  -> symlink -> /var/log/pods/<namespace>_<pod-name>_<pod-uid>/<container-name>/0.log

2026-03-01T12:00:00.123456789Z stdout F This is a log message
2026-03-01T12:00:00.123456789Z stderr P This is a partial log line

# Route to different Outputs based on Namespace
pipeline:
  filters:
    - name: kubernetes
      match: kube.*
      merge_log: on
      labels: on

    # Change tags for production namespace
    - name: rewrite_tag
      match: kube.*
      rule: $kubernetes['namespace_name'] ^(production)$ prod.$TAG false
      rule: $kubernetes['namespace_name'] ^(staging)$    stg.$TAG  false
      rule: $kubernetes['namespace_name'] ^(monitoring)$ mon.$TAG  false

  outputs:
    # Production logs -> Elasticsearch
    - name: es
      match: prod.*
      host: es-production.logging.svc
      port: 9200
      logstash_format: on
      logstash_prefix: prod-logs

    # Staging logs -> Loki
    - name: loki
      match: stg.*
      host: loki.logging.svc
      port: 3100
      labels:
        env=staging

    # Monitoring logs -> S3 archiving
    - name: s3
      match: mon.*
      bucket: monitoring-logs-archive
      region: ap-northeast-2
      total_file_size: 100M
      upload_timeout: 10m
      compression: gzip

    # Other logs -> Default Elasticsearch
    - name: es
      match: kube.*
      host: es-default.logging.svc
      port: 9200
      logstash_format: on
      logstash_prefix: default-logs

pipeline:
  filters:
    - name: kubernetes
      match: kube.*
      merge_log: on
      labels: on

    # Tag rewriting based on team Label
    - name: rewrite_tag
      match: kube.*
      rule: $kubernetes['labels']['team'] ^(backend)$  team.backend.$TAG  false
      rule: $kubernetes['labels']['team'] ^(frontend)$ team.frontend.$TAG false
      rule: $kubernetes['labels']['team'] ^(data)$     team.data.$TAG     false

  outputs:
    # Backend team -> Dedicated Elasticsearch index
    - name: es
      match: team.backend.*
      host: elasticsearch.logging.svc
      logstash_format: on
      logstash_prefix: team-backend

    # Frontend team -> Dedicated Loki tenant
    - name: loki
      match: team.frontend.*
      host: loki.logging.svc
      tenant_id: frontend-team
      labels:
        team=frontend

    # Data team -> S3 + Elasticsearch dual
    - name: es
      match: team.data.*
      host: elasticsearch.logging.svc
      logstash_format: on
      logstash_prefix: team-data

    - name: s3
      match: team.data.*
      bucket: data-team-logs
      region: ap-northeast-2
      compression: gzip

# fluent-bit-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluent-bit
  namespace: logging
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluent-bit
rules:
  - apiGroups: ['']
    resources:
      - namespaces
      - pods
      - pods/logs
      - nodes
      - nodes/proxy
    verbs: ['get', 'list', 'watch']
  - apiGroups: ['']
    resources:
      - events
    verbs: ['get', 'list', 'watch']
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: fluent-bit
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: fluent-bit
subjects:
  - kind: ServiceAccount
    name: fluent-bit
    namespace: logging

# fluent-bit-efk.yaml
service:
  flush: 5
  log_level: info
  parsers_file: /fluent-bit/etc/parsers.conf
  http_server: on
  http_listen: 0.0.0.0
  http_port: 2020
  storage.path: /var/log/fluent-bit/buffer/
  storage.sync: normal
  storage.max_chunks_up: 128

pipeline:
  inputs:
    - name: tail
      tag: kube.*
      path: /var/log/containers/*.log
      multiline.parser: docker, cri
      db: /var/log/flb_kube.db
      mem_buf_limit: 5MB
      skip_long_lines: on
      refresh_interval: 10
      storage.type: filesystem

  filters:
    - name: kubernetes
      match: kube.*
      kube_url: https://kubernetes.default.svc:443
      kube_ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      kube_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      merge_log: on
      keep_log: off
      k8s-logging.parser: on
      k8s-logging.exclude: on
      labels: on

    # Exclude kube-system and logging namespace logs
    - name: grep
      match: kube.*
      exclude: $kubernetes['namespace_name'] ^(kube-system|logging)$

    # Add environment info
    - name: modify
      match: kube.*
      add: cluster_name my-eks-cluster
      add: environment production

  outputs:
    - name: es
      match: kube.*
      host: elasticsearch-master.logging.svc.cluster.local
      port: 9200
      http_user: elastic
      http_passwd: ${ES_PASSWORD}
      logstash_format: on
      logstash_prefix: kube-logs
      logstash_dateformat: "%Y.%m.%d"
      time_key: "@timestamp"
      include_tag_key: true
      tag_key: fluentbit_tag
      suppress_type_name: on
      generate_id: on
      retry_limit: false
      workers: 2
      tls: on
      tls.verify: on
      tls.ca_file: /certs/es-ca.pem

# fluent-bit-loki.yaml
service:
  flush: 5
  log_level: info
  http_server: on
  http_listen: 0.0.0.0
  http_port: 2020

pipeline:
  inputs:
    - name: tail
      tag: kube.*
      path: /var/log/containers/*.log
      multiline.parser: docker, cri
      db: /var/log/flb_kube.db
      mem_buf_limit: 5MB
      skip_long_lines: on

  filters:
    - name: kubernetes
      match: kube.*
      merge_log: on
      keep_log: off
      labels: on
      annotations: off

  outputs:
    - name: loki
      match: kube.*
      host: loki-gateway.logging.svc.cluster.local
      port: 3100
      uri: /loki/api/v1/push
      labels: job=fluent-bit,cluster=my-cluster
      label_keys: $kubernetes['namespace_name'],$kubernetes['pod_name'],$kubernetes['container_name']
      auto_kubernetes_labels: on
      line_format: json
      workers: 2
      retry_limit: false

# fluent-bit-dual.yaml
service:
  flush: 5
  log_level: info
  storage.path: /var/log/fluent-bit/buffer/
  storage.sync: normal
  storage.max_chunks_up: 128

pipeline:
  inputs:
    - name: tail
      tag: kube.*
      path: /var/log/containers/*.log
      multiline.parser: docker, cri
      db: /var/log/flb_kube.db
      storage.type: filesystem

  filters:
    - name: kubernetes
      match: kube.*
      merge_log: on
      keep_log: off
      labels: on

  outputs:
    # Elasticsearch for real-time search/analysis
    - name: es
      match: kube.*
      host: elasticsearch.logging.svc
      port: 9200
      logstash_format: on
      logstash_prefix: kube-logs
      suppress_type_name: on
      retry_limit: false
      workers: 2

    # S3 archiving for long-term retention
    - name: s3
      match: kube.*
      region: ap-northeast-2
      bucket: log-archive-bucket
      s3_key_format: /kubernetes/$TAG/%Y/%m/%d/%H/$UUID.gz
      s3_key_format_tag_delimiters: .
      total_file_size: 100M
      upload_timeout: 10m
      compression: gzip
      content_type: application/gzip
      store_dir: /tmp/fluent-bit-s3
      store_dir_limit_size: 1G
      use_put_object: on
      retry_limit: 5
      workers: 1

# fluent-bit-routing.yaml
service:
  flush: 5
  log_level: info
  storage.path: /var/log/fluent-bit/buffer/
  storage.sync: normal

pipeline:
  inputs:
    - name: tail
      tag: kube.*
      path: /var/log/containers/*.log
      multiline.parser: docker, cri
      db: /var/log/flb_kube.db
      storage.type: filesystem

  filters:
    - name: kubernetes
      match: kube.*
      merge_log: on
      labels: on

    # Per-Namespace tag rewriting
    - name: rewrite_tag
      match: kube.*
      rule: $kubernetes['namespace_name'] ^(production)$  route.prod.$TAG  false
      rule: $kubernetes['namespace_name'] ^(staging)$     route.stg.$TAG   false
      rule: $kubernetes['namespace_name'] ^(kube-system)$ route.sys.$TAG   false

    # Extract only ERRORs from production
    - name: rewrite_tag
      match: route.prod.*
      rule: $log ^.*ERROR.*$ alert.prod.$TAG true

  outputs:
    # All production logs -> Elasticsearch
    - name: es
      match: route.prod.*
      host: es-prod.logging.svc
      port: 9200
      logstash_format: on
      logstash_prefix: prod-all
      suppress_type_name: on
      workers: 2

    # Production ERROR alerts -> Separate index
    - name: es
      match: alert.prod.*
      host: es-prod.logging.svc
      port: 9200
      logstash_format: on
      logstash_prefix: prod-alerts
      suppress_type_name: on

    # Staging logs -> Loki (cost-efficient)
    - name: loki
      match: route.stg.*
      host: loki.logging.svc
      port: 3100
      labels:
        env=staging
      auto_kubernetes_labels: on
      line_format: json

    # System logs -> S3 archiving (long-term retention)
    - name: s3
      match: route.sys.*
      bucket: system-logs-archive
      region: ap-northeast-2
      total_file_size: 50M
      upload_timeout: 15m
      compression: gzip
      s3_key_format: /kube-system/%Y/%m/%d/$UUID.gz

    # Unclassified logs -> Default Elasticsearch
    - name: es
      match: kube.*
      host: es-default.logging.svc
      port: 9200
      logstash_format: on
      logstash_prefix: default-logs
      suppress_type_name: on

pipeline:
  outputs:
    - name: es
      match: '*'
      host: elasticsearch
      port: 9200
      workers: 4 # 4 parallel Workers
      net.keepalive: on
      net.keepalive_idle_timeout: 30

service:
  flush: 1 # Flush every 1 second (prioritize real-time)
  # flush: 5  # Flush every 5 seconds (prioritize throughput)

pipeline:
  inputs:
    - name: tail
      path: /var/log/containers/*.log
      # High throughput environment
      buffer_chunk_size: 512KB # Chunk unit size (default 32KB)
      buffer_max_size: 5MB # Maximum buffer size (default 32KB)
      mem_buf_limit: 50MB # Memory buffer limit

  outputs:
    - name: es
      match: '*'
      buffer_size: 512KB # HTTP buffer size

pipeline:
  inputs:
    # Application log pipeline
    - name: tail
      tag: app.*
      path: /var/log/containers/app-*.log
      multiline.parser: docker, cri
      threaded: on # Run in a separate thread

    # System log pipeline
    - name: tail
      tag: sys.*
      path: /var/log/containers/kube-*.log
      multiline.parser: docker, cri
      threaded: on

  outputs:
    - name: es
      match: app.*
      host: elasticsearch
      workers: 4

    - name: es
      match: sys.*
      host: elasticsearch
      workers: 2

# Enable Hot Reload
service:
  hot_reload: on

# Method 1: SIGHUP signal
kill -SIGHUP $(pidof fluent-bit)

# Method 2: HTTP API (v4.0+)
curl -X POST http://localhost:2020/api/v2/reload

# Method 3: Command line option
fluent-bit -c fluent-bit.yaml -Y   # --enable-hot-reload

# Check Fluent Bit process memory
ps aux | grep fluent-bit

# Check internal metrics via HTTP API
curl -s http://localhost:2020/api/v1/storage | jq .

# Check via Prometheus metrics
curl -s http://localhost:2020/api/v2/metrics/prometheus | grep fluentbit_input

service:
  http_server: on
  http_listen: 0.0.0.0
  http_port: 2020
  health_check: on
  hc_errors_count: 5 # Unhealthy when this many errors occur
  hc_retry_failure_count: 5 # Unhealthy when this many retry failures occur
  hc_period: 60 # Health check interval (seconds)

# Health check
curl -s http://localhost:2020/api/v1/health
# Response: ok (HTTP 200) or error (HTTP 500)

# Check storage status
curl -s http://localhost:2020/api/v1/storage | jq .
# {
#   "storage_layer": {
#     "chunks": {
#       "total_chunks": 15,
#       "mem_chunks": 10,
#       "fs_chunks": 5,
#       "fs_chunks_up": 3,
#       "fs_chunks_down": 2
#     }
#   }
# }

# Check Prometheus metrics
curl -s http://localhost:2020/api/v2/metrics/prometheus

# Expose Fluent Bit's own metrics to Prometheus
pipeline:
  inputs:
    - name: fluentbit_metrics
      tag: fb.metrics
      scrape_interval: 30

  outputs:
    - name: prometheus_exporter
      match: fb.metrics
      host: 0.0.0.0
      port: 2021

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: fluent-bit
  namespace: logging
  labels:
    release: prometheus
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: fluent-bit
  endpoints:
    - port: http
      path: /api/v2/metrics/prometheus
      interval: 30s
      scrapeTimeout: 10s
  namespaceSelector:
    matchNames:
      - logging

# 1. Check Fluent Bit process status
kubectl get pods -n logging -l app.kubernetes.io/name=fluent-bit
kubectl logs -n logging <fluent-bit-pod> --tail=50

# 2. Validate configuration file syntax
fluent-bit -c fluent-bit.yaml --dry-run

# 3. Verify log file paths
kubectl exec -n logging <fluent-bit-pod> -- ls -la /var/log/containers/

# 4. Check DB file (offset)
kubectl exec -n logging <fluent-bit-pod> -- ls -la /var/log/flb_kube.db

# 5. Check permissions
kubectl exec -n logging <fluent-bit-pod> -- cat /var/log/containers/<target-log>

# Memory limit configuration
pipeline:
  inputs:
    - name: tail
      path: /var/log/containers/*.log
      mem_buf_limit: 5MB # Input memory limit
      skip_long_lines: on # Skip long lines
      buffer_chunk_size: 32KB # Chunk size limit
      buffer_max_size: 32KB # Maximum buffer

service:
  storage.path: /var/log/fluent-bit/buffer/
  storage.max_chunks_up: 64 # Limit memory chunk count (default 128)

# Check for Backpressure
curl -s http://localhost:2020/api/v1/storage | jq .

# Check via metrics
curl -s http://localhost:2020/api/v2/metrics/prometheus | grep -E "retries|errors|dropped"

# Backpressure response configuration example
service:
  flush: 1
  storage.path: /var/log/fluent-bit/buffer/
  storage.sync: normal
  storage.max_chunks_up: 256

pipeline:
  inputs:
    - name: tail
      tag: kube.*
      path: /var/log/containers/*.log
      storage.type: filesystem
      storage.pause_on_chunks_overlimit: off

  outputs:
    - name: es
      match: kube.*
      host: elasticsearch
      port: 9200
      workers: 4
      storage.total_limit_size: 10G
      retry_limit: false
      net.keepalive: on
      net.keepalive_idle_timeout: 15

# TLS connection test
kubectl exec -n logging <fluent-bit-pod> -- \
  curl -v --cacert /certs/ca.pem https://elasticsearch:9200

# Check certificate expiration
kubectl exec -n logging <fluent-bit-pod> -- \
  openssl x509 -in /certs/ca.pem -noout -enddate

service:
  log_level: debug # error, warn, info, debug, trace

pipeline:
  outputs:
    # For debugging: all records to standard output
    - name: stdout
      match: '*'
      format: json_lines

    # Actual Output
    - name: es
      match: '*'
      host: elasticsearch

# Test Input only
fluent-bit -i tail -p path=/var/log/test.log -o stdout

# Test Parser
fluent-bit -i tail -p path=/var/log/test.log -p parser=json -o stdout

# Test full configuration (dry-run)
fluent-bit -c fluent-bit.yaml --dry-run

# Stream Fluent Bit logs
kubectl logs -n logging -l app.kubernetes.io/name=fluent-bit -f --tail=100

# Specific Pod logs
kubectl logs -n logging fluent-bit-xxxxx -f

# Previous container logs (on crash)
kubectl logs -n logging fluent-bit-xxxxx --previous

# Kubernetes Pod Security
securityContext:
  runAsNonRoot: true
  runAsUser: 1000
  readOnlyRootFilesystem: true
  capabilities:
    drop: ['ALL']

# Mount only required volumes
volumes:
  - name: varlog
    hostPath:
      path: /var/log
      type: ''
  - name: buffer
    emptyDir:
      sizeLimit: 2Gi

# Enforce TLS (Output)
pipeline:
  outputs:
    - name: es
      tls: on
      tls.verify: on
      tls.ca_file: /certs/ca.pem

pipeline:
  inputs:
    - name: tail
      path: /var/log/app/*.log
      db: /var/log/flb_app.db # Required: offset tracking
      rotate_wait: 5 # Wait time after rotation (seconds)
      refresh_interval: 10 # File list refresh interval