From 5db626b12c5fe383d46cd4fda935df17e2915d30 Mon Sep 17 00:00:00 2001 From: xieshujian Date: Mon, 18 Apr 2022 16:06:43 +0800 Subject: [PATCH 1/2] fluent bit on OpenShift to OpenSearch Signed-off-by: xieshujian --- README.md | 14 ++ .../opensearch/fluent-bit-ds-openshift.yaml | 79 ++++++++++++ ...nt-bit-opensearch-openshift-configmap.yaml | 120 ++++++++++++++++++ 3 files changed, 213 insertions(+) create mode 100644 output/opensearch/fluent-bit-ds-openshift.yaml create mode 100644 output/opensearch/fluent-bit-opensearch-openshift-configmap.yaml diff --git a/README.md b/README.md index 11f30e9..a89e5a1 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,20 @@ If you are using Minikube for testing purposes, use the following alternative Da $ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/elasticsearch/fluent-bit-ds-minikube.yaml ``` +#### Fluent Bit to OpenSearch on OpenShift4 +If you are use OpenSearch and OpenShift, user the following ConfigMap and DaemonSet. +- For the OpenSearch, add tsl setting, as it use https default +- For the OpenShift, add security context on daemonset, and increase buffer size to 1Mb on kuberneters filter +``` +$ oc new-project logging +$ oc apply -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/fluent-bit-service-account.yaml +$ oc apply -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/fluent-bit-role-1.22.yaml +$ oc apply -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/fluent-bit-role-binding-1.22.yaml +$ oc apply -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/fluent-bit-openshift-security-context-constraints.yaml +$ oc apply -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/opensearch/fluent-bit-opensearch-openshift-configmap.yaml +$ oc apply -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/opensearch/fluent-bit-ds-openshift.yaml +``` + ## Details The default configuration of Fluent Bit makes sure of the following: diff --git a/output/opensearch/fluent-bit-ds-openshift.yaml b/output/opensearch/fluent-bit-ds-openshift.yaml new file mode 100644 index 0000000..1121c77 --- /dev/null +++ b/output/opensearch/fluent-bit-ds-openshift.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluent-bit + namespace: logging + labels: + k8s-app: fluent-bit-logging + version: v1 + kubernetes.io/cluster-service: "true" +spec: + selector: + matchLabels: + k8s-app: fluent-bit-logging + template: + metadata: + labels: + k8s-app: fluent-bit-logging + version: v1 + kubernetes.io/cluster-service: "true" + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "2020" + prometheus.io/path: /api/v1/metrics/prometheus + spec: + containers: + - name: fluent-bit + image: fluent/fluent-bit:1.9.2 + imagePullPolicy: IfNotPresent + securityContext: + capabilities: + drop: + - CHOWN + - DAC_OVERRIDE + - FOWNER + - FSETID + - KILL + - NET_BIND_SERVICE + - SETGID + - SETPCAP + - SETUID + seLinuxOptions: + type: spc_t + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + ports: + - containerPort: 2020 + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: "elasticsearch" + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" + volumeMounts: + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc/ + terminationGracePeriodSeconds: 10 + volumes: + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: fluent-bit-config + configMap: + name: fluent-bit-config + serviceAccountName: fluent-bit + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - operator: "Exists" + effect: "NoExecute" + - operator: "Exists" + effect: "NoSchedule" diff --git a/output/opensearch/fluent-bit-opensearch-openshift-configmap.yaml b/output/opensearch/fluent-bit-opensearch-openshift-configmap.yaml new file mode 100644 index 0000000..e9b1c0b --- /dev/null +++ b/output/opensearch/fluent-bit-opensearch-openshift-configmap.yaml @@ -0,0 +1,120 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluent-bit-config + namespace: logging + labels: + k8s-app: fluent-bit +data: + # Configuration files: server, input, filters and output + # ====================================================== + fluent-bit.conf: | + [SERVICE] + Flush 1 + Log_Level info + Daemon off + Parsers_File parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + + @INCLUDE input-kubernetes.conf + @INCLUDE filter-kubernetes.conf + @INCLUDE output-elasticsearch.conf + + input-kubernetes.conf: | + [INPUT] + Name tail + Tag kube.* + Path /var/log/containers/*.log + Parser cri + DB /var/log/flb_kube.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + + filter-kubernetes.conf: | + [FILTER] + Name kubernetes + Match kube.* + Kube_URL https://kubernetes.default.svc:443 + Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token + Kube_Tag_Prefix kube.var.log.containers. + Merge_Log On + Merge_Log_Key log_processed + K8S-Logging.Parser On + K8S-Logging.Exclude Off + Buffer_Size 1Mb + + output-elasticsearch.conf: | + [OUTPUT] + Name opensearch + Match kube.* + Host {OPENSEARCH_HOST} + Port {OPENSEARCH_PORT} + HTTP_User {OPENSEARCH_USER} + HTTP_Passwd {OPENSEARCH_PASSWORD} + tls On + tls.verify Off + Replace_Dots On + Logstash_Format On + Logstash_Prefix {MY_PREFIX} + Logstash_Prefix_Key {MY_PREFIX} + Index {MY_INDEX} + Type {MY_TYPE} + + parsers.conf: | + [PARSER] + Name apache + Format regex + Regex ^(?[^ ]*) [^ ]* (?[^ ]*) \[(?