diff --git a/examples/xray/logging/README.md b/examples/xray/logging/README.md new file mode 100644 index 000000000..31fd79835 --- /dev/null +++ b/examples/xray/logging/README.md @@ -0,0 +1,22 @@ +# Setting up a logging sidecar +Due to the nature of Xray running with many services, each writing multiple logs to the file system, it's hard to collect them all in a Kubernetes based deployment.
+The example in this directory has an example of using a [fluent-bit](https://fluentbit.io/) sidecar that collects all logs from Xray's `log/` directory and writes them to STDOUT in a nice json formatted way + +See the [values-logging-fluent-bit.yaml](values-logging-fluent-bit.yaml) for the configuration example + +## Deploy +Install Xray with the following command +```shell +helm upgrade --install xray jfrog/xray -f values-logging-fluent-bit.yaml +``` + +## Fluent-bit STDOUT +Once running, the `fluent-bit` sidecar tails the logs in the configured directories and outputs them to the container's STDOUT in a json format.
+Each line had a `"file"` key that lists the source file, which later can be used to separate the sources.
+The actual log line is in the `"log"` key. +```json +{"date":1700135001.943258,"file":"/var/opt/jfrog/xray/log/xray-server-service.log","log":"2023-11-16T11:43:21.942Z \u001b[33m[jfxr ]\u001b[0m \u001b[34m[INFO ]\u001b[0m [0e860ce4f1fd6552] [migrate:36 ] [MainServer ] Data migration that should run always: starting for name V1_Update_Features_table"} +``` + +## Cluster log collector +Once this is setup, you need to configure your cluster log collector (probably running as a DaemonSet) to collect logs from this container only. \ No newline at end of file diff --git a/examples/xray/logging/values-logging-fluent-bit.yaml b/examples/xray/logging/values-logging-fluent-bit.yaml new file mode 100644 index 000000000..5025054c6 --- /dev/null +++ b/examples/xray/logging/values-logging-fluent-bit.yaml @@ -0,0 +1,68 @@ +common: + + # Create extra config maps with the required fluent-bit configuration + configMaps: | + fluent-bit-input.conf: |- + [INPUT] + Name tail + Path /var/opt/jfrog/xray/log/*.log + DB /var/opt/jfrog/fluentdb/fluentdb.db + Exclude_Path *router-request.log,*console.log,*-metrics.log + Path_Key file + Refresh_Interval 20 + Multiline On + Parser_Firstline multiline_pattern + Skip_Long_Lines On + + fluent-bit-output.conf: |- + [OUTPUT] + Name stdout + Match * + Format json_lines + + fluent-bit-service.conf: |- + [SERVICE] + Flush 1 + Daemon Off + Log_Level info + Parsers_File parsers.conf + + fluent-bit.conf: |- + @INCLUDE fluent-bit-service.conf + @INCLUDE fluent-bit-input.conf + @INCLUDE fluent-bit-output.conf + + parsers.conf: |- + [PARSER] + Name multiline_pattern + Format regex + Regex ^(?\d{2,4}\-\d{2,4}\-\d{2,4}T\d{2,4}\:\d{2,4}\:\d{2,4}\.\d{1,6}?.*) + + # Create the extra sidecar container + customSidecarContainers: | + - name: xray-fluent-bit + image: "fluent/fluent-bit:2.1.2" + volumeMounts: + - mountPath: /var/opt/jfrog/xray + name: data-volume + - mountPath: /var/opt/jfrog/fluentdb + name: fluentdb + - mountPath: /fluent-bit/etc/fluent-bit.conf + name: xray-configmaps + subPath: fluent-bit.conf + - mountPath: /fluent-bit/etc/fluent-bit-service.conf + name: xray-configmaps + subPath: fluent-bit-service.conf + - mountPath: /fluent-bit/etc/fluent-bit-input.conf + name: xray-configmaps + subPath: fluent-bit-input.conf + - mountPath: /fluent-bit/etc/fluent-bit-output.conf + name: xray-configmaps + subPath: fluent-bit-output.conf + - mountPath: /fluent-bit/etc/parsers.conf + name: xray-configmaps + subPath: parsers.conf + + customVolumes: | + - name: fluentdb + emptyDir: {}