Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: skorpworks/logbus
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: master
Choose a base ref
...
head repository: erik-stephens/logbus
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: master
Choose a head ref
Can’t automatically merge. Don’t worry, you can still create the pull request.
Loading
Showing with 11,570 additions and 7,979 deletions.
  1. +0 −13 .eslintrc
  2. +286 −0 .eslintrc.yml
  3. +3 −1 .gitignore
  4. +23 −32 Dockerfile
  5. +46 −0 Dockerfile.bionic
  6. +53 −54 Makefile
  7. +157 −12 README.md
  8. +13 −23 config/example.yml
  9. +19 −0 config/templates.yml
  10. +65 −0 examples/elasticsearch-etl/conf.yml
  11. +216 −265 index.js
  12. +165 −97 lib/plugins/agg.js
  13. +102 −0 lib/plugins/cast-spec.js
  14. +32 −43 lib/plugins/cast.js
  15. +63 −70 lib/plugins/dns.js
  16. +29 −0 lib/plugins/drop-fields-spec.js
  17. +9 −20 lib/plugins/drop-fields.js
  18. +0 −110 lib/plugins/elasticsearch.js
  19. +38 −0 lib/plugins/errors-spec.js
  20. +53 −43 lib/plugins/errors.js
  21. +22 −19 lib/plugins/gc.js
  22. +66 −41 lib/plugins/geoip.js
  23. +68 −91 lib/plugins/geopop.js
  24. +162 −0 lib/plugins/input/elasticsearch-spec.js
  25. +194 −0 lib/plugins/input/elasticsearch.js
  26. +62 −0 lib/plugins/input/file-spec.js
  27. +45 −38 lib/plugins/input/file.js
  28. +38 −0 lib/plugins/input/journal-spec.js
  29. +102 −86 lib/plugins/input/journal.js
  30. +0 −17 lib/plugins/input/json.js
  31. +136 −142 lib/plugins/input/kafka-librd.js
  32. +9 −7 lib/plugins/input/kafka.js
  33. +0 −32 lib/plugins/input/lines.js
  34. +54 −26 lib/plugins/input/stdin.js
  35. +56 −0 lib/plugins/input/tail-spec.js
  36. +110 −82 lib/plugins/input/tail.js
  37. +0 −19 lib/plugins/input/yaml.js
  38. +75 −0 lib/plugins/js-spec.js
  39. +32 −34 lib/plugins/js.js
  40. +33 −0 lib/plugins/keep-fields-spec.js
  41. +24 −34 lib/plugins/keep-fields.js
  42. +44 −46 lib/plugins/log.js
  43. +4 −0 lib/plugins/old/README.md
  44. +2 −6 lib/plugins/{input/kafka-node.js → old/kafka-node-input.js}
  45. +2 −3 lib/plugins/{output/kafka-node.js → old/kafka-node-output.js}
  46. +120 −0 lib/plugins/output/elasticsearch-spec.js
  47. +136 −88 lib/plugins/output/elasticsearch.js
  48. +25 −43 lib/plugins/output/file.js
  49. +0 −25 lib/plugins/output/json.js
  50. +144 −142 lib/plugins/output/kafka-librd.js
  51. +9 −7 lib/plugins/output/kafka.js
  52. +37 −14 lib/plugins/output/stdout.js
  53. +0 −19 lib/plugins/output/yaml.js
  54. +114 −0 lib/plugins/parse/elasticsearch.js
  55. +14 −0 lib/plugins/parse/json.js
  56. +145 −0 lib/plugins/parse/kafka-broker.js
  57. +30 −0 lib/plugins/parse/lines.js
  58. +16 −0 lib/plugins/parse/yaml.js
  59. +64 −17 lib/plugins/pass.js
  60. +33 −0 lib/plugins/rename-fields-spec.js
  61. +15 −19 lib/plugins/rename-fields.js
  62. +90 −0 lib/plugins/sample-spec.js
  63. +49 −31 lib/plugins/sample.js
  64. +39 −0 lib/plugins/serialize/json-spec.js
  65. +20 −0 lib/plugins/serialize/json.js
  66. +26 −0 lib/plugins/serialize/yaml-spec.js
  67. +17 −0 lib/plugins/serialize/yaml.js
  68. +40 −40 lib/plugins/sql.js
  69. +27 −0 lib/plugins/stats-spec.js
  70. +86 −89 lib/plugins/stats.js
  71. +1 −0 lib/test/fixtures/input-file/a.csv
  72. +3 −0 lib/test/fixtures/input-file/a.txt
  73. +3 −0 lib/test/fixtures/input-file/b.txt
  74. +1 −0 lib/test/fixtures/input-tail/a.csv
  75. +3 −0 lib/test/fixtures/input-tail/a.txt
  76. +3 −0 lib/test/fixtures/input-tail/b.txt
  77. +35 −0 lib/test/journalctl
  78. +61 −0 lib/test/logbus.js
  79. +6 −0 misc/elasticsearch/README.md
  80. +123 −0 misc/elasticsearch/kafka-broker.json
  81. +8 −0 misc/kibana/README.md
  82. +19 −0 misc/kibana/logbus-dashboard.json
  83. +50 −0 misc/kibana/logbus-search.json
  84. +92 −0 misc/kibana/logbus-vizualization.json
  85. +0 −5,851 package-lock.json
  86. +41 −33 package.json
  87. +77 −0 stage.js
  88. +0 −29 test/elasticsearch/multiline.log
  89. +3 −6 test/kafka/consumer.yml
  90. +2 −2 test/kafka/producer.yml
  91. +150 −0 test/pipeline/agg/conf.yml
  92. +33 −0 test/pipeline/agg/expected.json
  93. +14 −0 test/pipeline/agg/test.log
  94. +18 −13 test/{ → pipeline}/elasticsearch/conf.yml
  95. 0 test/{ → pipeline}/elasticsearch/corrupt.log
  96. +56 −1 test/{ → pipeline}/elasticsearch/expected.json
  97. +65 −0 test/pipeline/elasticsearch/multiline.log
  98. +99 −0 test/pipeline/elasticsearch/shard-failure.log
  99. +1 −1 test/{ → pipeline}/elasticsearch/test.log
  100. +148 −0 test/pipeline/kafka-broker/conf.yml
  101. +140 −0 test/pipeline/kafka-broker/expected.json
  102. +24 −0 test/pipeline/kafka-broker/server.log
  103. +3 −3 test/{ → pipeline}/skeleton/conf.yml
  104. 0 test/{ → pipeline}/skeleton/expected.json
  105. 0 test/{ → pipeline}/skeleton/test.log
  106. +6,280 −0 yarn.lock
13 changes: 0 additions & 13 deletions .eslintrc

This file was deleted.

286 changes: 286 additions & 0 deletions .eslintrc.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,286 @@
---
env:
node: true
es2021: true
extends:
- eslint:recommended
parserOptions:
ecmaVersion: 12
sourceType: module
plugins:
- jest
overrides:
- files: "**/*-spec.js"
extends:
- "plugin:jest/recommended"
rules:
accessor-pairs: error
array-bracket-newline: error
array-bracket-spacing:
- error
- never
array-callback-return: error
array-element-newline: off
arrow-body-style: off
arrow-parens:
- error
- as-needed
arrow-spacing:
- error
- after: true
before: true
block-scoped-var: error
block-spacing: error
brace-style:
- error
- 1tbs
callback-return: error
camelcase: error
capitalized-comments: off
class-methods-use-this: error
comma-dangle: off
comma-spacing:
- error
- after: true
before: false
comma-style:
- error
- last
# complexity: error
computed-property-spacing:
- error
- never
consistent-return: error
consistent-this: error
curly: error
default-case: error
default-case-last: error
default-param-last: error
dot-location: error
dot-notation: error
eol-last: error
eqeqeq: off
func-call-spacing: error
func-name-matching: error
func-names: off
func-style: off
function-paren-newline: error
generator-star-spacing: error
global-require: off
grouped-accessor-pairs: error
guard-for-in: error
handle-callback-err: error
id-blacklist: error
id-denylist: error
id-length: off
id-match: error
implicit-arrow-linebreak:
- error
- beside
indent:
- warn
- 2
indent-legacy: off
init-declarations: error
jsx-quotes: error
key-spacing: error
keyword-spacing:
- error
- after: true
before: true
line-comment-position: off
linebreak-style:
- error
- unix
lines-around-comment: error
lines-around-directive: error
lines-between-class-members: error
max-classes-per-file: off
max-depth: error
max-len: off
max-lines: off
max-lines-per-function: off
max-nested-callbacks: error
max-params:
- error
- max: 5
max-statements: off
max-statements-per-line: error
multiline-comment-style:
- error
- separate-lines
multiline-ternary: off
new-parens: error
newline-after-var: off
newline-before-return: off
newline-per-chained-call: off
no-alert: error
no-array-constructor: error
no-await-in-loop: off
no-bitwise: error
no-buffer-constructor: error
no-caller: error
no-catch-shadow: error
no-confusing-arrow: error
no-console: off
no-constructor-return: error
no-continue: off
no-div-regex: error
no-duplicate-imports: error
no-else-return: error
no-empty-function: error
no-eq-null: off
no-eval: error
no-extend-native: error
no-extra-bind: error
no-extra-label: error
no-extra-parens:
- error
- all
- ignoreJSX: all
no-floating-decimal: error
no-implicit-coercion: error
no-implicit-globals: error
no-implied-eval: error
no-inline-comments: off
no-invalid-this: error
no-iterator: error
no-label-var: error
no-labels: error
no-lone-blocks: error
no-lonely-if: error
no-loop-func: error
no-loss-of-precision: error
no-magic-numbers: off
no-mixed-operators: error
no-mixed-requires: error
no-multi-assign: error
no-multi-spaces: error
no-multi-str: error
no-multiple-empty-lines: error
no-native-reassign: error
no-negated-condition: off
no-negated-in-lhs: error
no-nested-ternary: error
no-new: error
no-new-func: error
no-new-object: error
no-new-require: error
no-new-wrappers: error
no-nonoctal-decimal-escape: error
no-octal-escape: error
no-param-reassign: error
no-path-concat: error
no-plusplus: off
no-process-env: off
no-process-exit: off
no-promise-executor-return: error
no-proto: error
no-restricted-exports: error
no-restricted-globals: error
no-restricted-imports: error
no-restricted-modules: error
no-restricted-properties: error
no-restricted-syntax: error
no-return-assign: error
no-return-await: off
no-script-url: error
no-self-compare: error
no-sequences: error
no-shadow: error
no-spaced-func: error
no-sync: error
no-tabs: error
no-template-curly-in-string: error
no-ternary: off
no-throw-literal: error
no-trailing-spaces: error
no-undef-init: error
no-undefined: error
no-underscore-dangle: off
no-unmodified-loop-condition: error
no-unneeded-ternary: error
no-unreachable-loop: error
no-unused-expressions: error
no-unused-vars: error
no-use-before-define: error
no-useless-backreference: error
no-useless-call: error
no-useless-computed-key: error
no-useless-concat: error
no-useless-constructor: error
no-useless-rename: error
no-useless-return: error
no-var: error
no-void: error
no-warning-comments: off
no-whitespace-before-property: error
nonblock-statement-body-position: error
object-curly-newline: error
object-curly-spacing:
- error
- never
object-shorthand: error
one-var: off
one-var-declaration-per-line: error
operator-assignment: error
operator-linebreak: error
padded-blocks: off
padding-line-between-statements: error
prefer-arrow-callback: off
prefer-const: error
prefer-destructuring: off
prefer-exponentiation-operator: error
prefer-named-capture-group: off
prefer-numeric-literals: error
prefer-object-spread: error
prefer-promise-reject-errors: error
prefer-reflect: off
prefer-regex-literals: error
prefer-rest-params: error
prefer-spread: error
prefer-template: off
quote-props: off
quotes: off
radix:
- error
- as-needed
require-atomic-updates: error
require-await: error
require-jsdoc: off
require-unicode-regexp: error
rest-spread-spacing: error
semi:
- error
- never
semi-spacing: error
semi-style: error
sort-imports: off
sort-keys: off
sort-vars: error
space-before-blocks: error
space-before-function-paren: off
space-in-parens:
- error
- never
space-infix-ops: error
space-unary-ops: error
spaced-comment:
- error
- always
strict: error
switch-colon-spacing: error
symbol-description: error
template-curly-spacing:
- error
- never
template-tag-spacing: error
unicode-bom:
- error
- never
valid-jsdoc: error
vars-on-top: error
wrap-iife: error
wrap-regex: error
yield-star-spacing: error
yoda: error
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
test/*/out.json
test/pipeline/*/out.json
lib/test/journal-db.json
.DS_Store
.grunt
.logbus
@@ -12,3 +13,4 @@ node_modules
pkg/opt/logbus
rpm
shippers/*.rpm
working
55 changes: 23 additions & 32 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,33 +1,24 @@

FROM node:6-alpine

ARG KAFKA
RUN if test -n "${KAFKA}"; then \
apk add --update git alpine-sdk python-dev zlib-dev bash && \
cd /opt && git clone -b v2.1.1 --recursive https://github.com/Blizzard/node-rdkafka.git; \
fi
RUN if test -n "${KAFKA}"; then \
cd /opt/node-rdkafka && sed -i'' -E -e "s#-l(crypto|ssl)#-lz#g" deps/librdkafka.gyp && npm install -g --unsafe; \
fi

ARG ELASTICSEARCH
RUN if test -n "${ELASTICSEARCH}"; then npm install -g elasticsearch@13.0.1; fi

ARG ALASQL
RUN if test -n "${ALASQL}"; then npm install -g alasql@0.3.3; fi

ARG MAXMIND
RUN if test -n "${MAXMIND}"; then npm install -g maxmind-db-reader@0.2.1; fi

# Add node modules in a way that will allow Docker to cache them.
ADD package.json /opt/logbus/package.json
ADD . /opt/logbus
RUN cd /opt/logbus && npm install -g --no-optional

# The `bin` in package.json doesn't work since node_modules in .dockerignore
#
# npm ERR! enoent ENOENT: no such file or directory, chmod '/usr/local/lib/node_modules/logbus/index.js'
#
RUN ln -s /usr/local/lib/node_modules/logbus/index.js /usr/bin/logbus

CMD ["bash"]
FROM node:16-slim AS build
WORKDIR /app
ADD package.json .
ADD yarn.lock .
RUN yarn --ignore-optional --prod --frozen-lockfile
ADD lib lib
ADD stage.js .
ADD index.js .

FROM build AS test
RUN yarn install --frozen-lockfile
ADD test .
# fail if needs linting
ADD .eslintrc.yml .
RUN yarn eslint --format unix lib *.js
# fail if any vulnerabilities >= moderate
RUN yarn audit --groups dependencies --level moderate || test $? -le 2
# fail if unit tests fail
RUN yarn jest --coverage --color

FROM build AS prod
ENV NODE_ENV production
CMD ["index.js"]
Loading