A decent devloop

Displaying unparsed log entries using
kubectl apply -k base; kubectl rollout status daemonset fluent-bit; kubectl logs -f -l app.kubernetes.io/name=fluent-bit

Issues like fluent#88 and fluent/fluent-bit#1902 (comment)
indicate that depending on /var/log/containers symlinks cause
quite a few issues.

/var/log/pods/ is the path stated in
https://github.com/kubernetes/kubernetes/blob/v1.22.0/pkg/kubelet/kuberuntime/kuberuntime_manager.go#L63
and I've verified on GKE cos-containerd, GKE ubuntu-dockerd and k3s
that the path contains the actual files, not symliks.

Also using /var/log/pods makes it trivial to exclude logs from any
container named fluent-bit. Doing so reduces the risk of endless log loops.

base/filter-kubernetes.conf

@@ -4,7 +4,7 @@
     Kube_URL            https://kubernetes.default.svc:443
     Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
-    Kube_Tag_Prefix     kube.var.log.containers.
+    Kube_Tag_Prefix     kube.var.log.pods.
     Merge_Log           On
     Merge_Log_Key       log_processed
     K8S-Logging.Parser  On

base/fluent-bit-daemonset.yaml

@@ -28,6 +28,9 @@ spec:
       containers:
       - name: fluent-bit
         image: docker.io/fluent/fluent-bit
+        env:
+        - name: LOG_LEVEL
+          value: debug
         ports:
         - name: http
           containerPort: 2020
@@ -40,20 +43,27 @@ spec:
             path: /api/v1/health
             port: http
         volumeMounts:
-        - name: varlog
-          mountPath: /var/log
-        - name: varlibdockercontainers
-          mountPath: /var/lib/docker/containers
+        - name: varlogpods
+          mountPath: /var/log/pods
           readOnly: true
+        - name: db
+          mountPath: /var/run/fluent-bit
         - name: config
           mountPath: /fluent-bit/etc/
+        # - mountPath: /etc/machine-id
+        #   name: etcmachineid
+        #   readOnly: true
       volumes:
-      - name: varlog
+      - name: varlogpods
         hostPath:
-          path: /var/log
-      - name: varlibdockercontainers
+          path: /var/log/pods
+      - name: db
         hostPath:
-          path: /var/lib/docker/containers
+          path: /var/run/fluent-bit
+      # - name: etcmachineid
+      #   hostPath:
+      #     path: /etc/machine-id
+      #     type: File
       - name: config
         configMap:
           name: fluent-bit

base/fluent-bit.conf

@@ -1,12 +1,12 @@
 [SERVICE]
     Flush         1
-    Log_Level     info
+    Log_Level     ${LOG_LEVEL}
     Daemon        off
     Parsers_File  parsers.conf
     HTTP_Server   On
     HTTP_Listen   0.0.0.0
     HTTP_Port     2020
 
 @INCLUDE input-kubernetes.conf
-@INCLUDE filter-kubernetes.conf
-@INCLUDE output-elasticsearch.conf
+#@INCLUDE filter-kubernetes.conf
+@INCLUDE outputs.conf

base/input-kubernetes.conf

@@ -1,9 +1,10 @@
 [INPUT]
     Name              tail
     Tag               kube.*
-    Path              /var/log/containers/*.log
-    Parser            docker
-    DB                /var/log/flb_kube.db
+    DB                /var/run/fluent-bit/tail-positions.db
     Mem_Buf_Limit     5MB
     Skip_Long_Lines   On
-    Refresh_Interval  10
+    DB.locking        true
+    Path              /var/log/pods/*/*/*.log
+    Exclude_Path      /var/log/pods/*/fluent-bit/*.log
+    #Parser            cri

base/kustomization.yaml

@@ -20,5 +20,5 @@ configMapGenerator:
   - fluent-bit.conf
   - input-kubernetes.conf
   - filter-kubernetes.conf
-  - output-elasticsearch.conf
+  - outputs.conf
   - parsers.conf

base/outputs.conf

@@ -0,0 +1,8 @@
+[OUTPUT]
+    Name             stdout
+    Format           json_lines
+    json_date_key    t
+    json_date_format iso8601
+    # Name    file
+    # Path    /var/run/fluent-bit
+    # File    file-output.txt