fluentbit-configmap-json.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: logging
  labels:
    k8s-app: fluent-bit
data:
  # Configuration files: server, input, filters, and output
  # =======================================================
  fluent-bit.conf: |
    [SERVICE]
        Flush         5
        Grace         120
        Log_Level     info
        Daemon        off
        Parsers_File  parsers.conf
        HTTP_Server   On
        HTTP_Listen   0.0.0.0
        HTTP_Port     2020
        # Plugins_File /fluent-bit/etc/plugins.conf
  
    @INCLUDE input-kubernetes.conf
    @INCLUDE filter-kubernetes.conf
    @INCLUDE output-stackdriver.conf

  input-kubernetes.conf: |
    [INPUT]
        Name                    tail
        Path                    /var/log/containers/*.log
        # System logs should be captured by the default system logging already
        Exclude_Path            /var/log/containers/*_kube-system_*.log,/var/log/containers/*_istio-system_*.log,/var/log/containers/*_knative-serving_*.log,/var/log/containers/*_gke-system_*.log,/var/log/containers/*_config-management-system_*.log
        Tag                     kube.*
        Parser                  docker
        DB                      /var/log/flb_kube.db
        Mem_Buf_Limit           20MB
        Skip_Long_Lines         On
        Refresh_Interval        10
        # Control the log       line length
        Buffer_Chunk_Size       256k
        Buffer_Max_Size         10240k
        # Using the docker mode to deal with multiline messages emitted by docker
        Docker_Mode             On

  filter-kubernetes.conf: |
    [FILTER]
        Name                kubernetes
        Match               kube.*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     kube.var.log.containers.
        # Try to merge the log messages
        Merge_Log           On
        Merge_Log_Key       log_processed
        K8S-Logging.Parser  On
        K8S-Logging.Exclude Off

    [FILTER]
        Name         parser
        Match        kube.*
        Key_Name     log
        Reserve_Data True
        Parser       containerd
        
    [FILTER]
        Name         parser
        Match        kube.*
        Key_Name     log
        Parser       json


  output-stackdriver.conf: |

    [OUTPUT]
        # write the log records that still have the 'kube.*' tags to Cloud Logging
        Name                    stackdriver
        Match                   kube.*
        # The following fields are necessary. They allow filtering in Cloud Logging
        # based on resource types. Change them accordingly based on your setup.
        resource                k8s_container
        k8s_cluster_name        custom-fluentbit
        k8s_cluster_location    us-east1-b

  parsers.conf: |

    [PARSER]
        Name        json
        Format      json

    [PARSER]
        Name        docker
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
        Time_Keep   On

    [PARSER]
        Name        containerd
        Format      regex
        Regex       ^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z

    [PARSER]
        Name        syslog
        Format      regex
        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
        Time_Key    time
        Time_Format %b %d %H:%M:%S