Merge branch 'edenhill-master'

.appveyor.yml

@@ -86,9 +86,9 @@ after_test:
 deploy:
 - provider: S3
   access_key_id:
-    secure: iBK0xb23FMYOrOsOb8cw3YGyU+6vvPX5BF+PXuMub8M=
+    secure: 3SmFFB3J1WWjLqxouvH8zLdcmrFNVHHbkROb+2BBVJE=
   secret_access_key:
-    secure: jJsj373UiOtuXf/u0LLL0Q8XQMyu4s/ucx0+vH4GpKbAfZJUwYB4dEO1//mQDNuC
+    secure: VT0D5uzlaJI6gfZbemKCnf0MMh6qnlcmioVADK0oCkW6syz+n17VzWScRjvAifPm
   region: us-west-1
   bucket: librdkafka-ci-packages
   folder: librdkafka/p-librdkafka__bld-appveyor__plat-windows__arch-$(platform)__bldtype-$(configuration)__tag-$(APPVEYOR_REPO_TAG_NAME)__sha-$(APPVEYOR_REPO_COMMIT)__bid-$(APPVEYOR_BUILD_ID)

.clang-format

@@ -0,0 +1,53 @@
+---
+BasedOnStyle: LLVM
+AlignAfterOpenBracket: Align
+AlignConsecutiveMacros: 'true'
+AlignConsecutiveAssignments: 'true'
+AlignConsecutiveDeclarations: 'false'
+AlignEscapedNewlines: Right
+AlignOperands: 'true'
+AlignTrailingComments: 'true'
+AllowAllArgumentsOnNextLine: 'true'
+AllowAllConstructorInitializersOnNextLine: 'true'
+AllowAllParametersOfDeclarationOnNextLine: 'false'
+AllowShortBlocksOnASingleLine: 'false'
+AllowShortCaseLabelsOnASingleLine: 'false'
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLoopsOnASingleLine: 'false'
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: 'true'
+BinPackArguments: 'true'
+BinPackParameters: 'false'
+BreakBeforeBraces: Custom
+BreakBeforeTernaryOperators: 'true'
+BreakConstructorInitializers: AfterColon
+BreakStringLiterals: 'true'
+ColumnLimit: '80'
+DerivePointerAlignment: 'false'
+SortIncludes: 'false'
+IncludeBlocks: Preserve
+IndentCaseLabels: 'false'
+IndentPPDirectives: None
+IndentWidth: '8'
+Language: Cpp
+MaxEmptyLinesToKeep: '3'
+PointerAlignment: Right
+ReflowComments: 'true'
+SpaceAfterCStyleCast: 'false'
+SpaceAfterLogicalNot: 'false'
+SpaceBeforeAssignmentOperators: 'true'
+SpaceBeforeCpp11BracedList: 'true'
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: 'true'
+SpaceInEmptyParentheses: 'false'
+SpacesBeforeTrailingComments: '2'
+SpacesInAngles: 'false'
+SpacesInCStyleCastParentheses: 'false'
+SpacesInContainerLiterals: 'false'
+SpacesInParentheses: 'false'
+SpacesInSquareBrackets: 'false'
+TabWidth: '8'
+UseTab: Never
+
+...

.clang-format-cpp

@@ -0,0 +1,52 @@
+---
+BasedOnStyle: Google
+AlignConsecutiveMacros: 'true'
+AlignConsecutiveAssignments: 'true'
+AlignConsecutiveDeclarations: 'false'
+AlignEscapedNewlines: Right
+AlignOperands: 'true'
+AlignTrailingComments: 'true'
+AllowAllArgumentsOnNextLine: 'true'
+AllowAllConstructorInitializersOnNextLine: 'true'
+AllowAllParametersOfDeclarationOnNextLine: 'false'
+AllowShortBlocksOnASingleLine: 'false'
+AllowShortCaseLabelsOnASingleLine: 'false'
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLoopsOnASingleLine: 'false'
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: 'true'
+BinPackArguments: 'true'
+BinPackParameters: 'false'
+BreakBeforeBraces: Custom
+BreakBeforeTernaryOperators: 'true'
+BreakConstructorInitializers: AfterColon
+BreakStringLiterals: 'true'
+ColumnLimit: '80'
+DerivePointerAlignment: 'false'
+SortIncludes: 'false'
+IncludeBlocks: Preserve
+IndentCaseLabels: 'false'
+IndentPPDirectives: None
+IndentWidth: '2'
+Language: Cpp
+MaxEmptyLinesToKeep: '3'
+PointerAlignment: Right
+ReflowComments: 'true'
+SpaceAfterCStyleCast: 'false'
+SpaceAfterLogicalNot: 'false'
+SpaceBeforeAssignmentOperators: 'true'
+SpaceBeforeCpp11BracedList: 'true'
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: 'true'
+SpaceInEmptyParentheses: 'false'
+SpacesBeforeTrailingComments: '2'
+SpacesInAngles: 'false'
+SpacesInCStyleCastParentheses: 'false'
+SpacesInContainerLiterals: 'false'
+SpacesInParentheses: 'false'
+SpacesInSquareBrackets: 'false'
+TabWidth: '8'
+UseTab: Never
+
+...

.formatignore

@@ -0,0 +1,18 @@
+# Files to not check/fix coding style for.
+# These files are imported from other sources and we want to maintain
+# them in the original form to make future updates easier.
+src/lz4.c
+src/lz4.h
+src/lz4frame.c
+src/lz4frame.h
+src/lz4hc.c
+src/lz4hc.h
+src/queue.h
+src/crc32c.c
+src/crc32c.h
+src/snappy.c
+src/snappy.h
+src/snappy_compat.h
+src/tinycthread.c
+src/tinycthread.h
+src/regexp.h

.github/ISSUE_TEMPLATE

@@ -1,5 +1,7 @@
 Read the FAQ first: https://github.com/edenhill/librdkafka/wiki/FAQ
 
+Do NOT create issues for questions, use the discussion forum: https://github.com/edenhill/librdkafka/discussions
+
 
 
 Description

.github/workflows/base.yml

@@ -0,0 +1,30 @@
+name: check
+on: [push, pull_request]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          sudo apt install -y python3 python3-pip python3-setuptools libcurl4-openssl-dev libssl-dev libsasl2-dev
+          python3 -m pip install -r tests/requirements.txt
+      - run: |
+          ./configure --CFLAGS="-std=c99" --CXXFLAGS="-std=c++98" --install-deps --enable-devel --disable-lz4-ext --prefix="$PWD/dest"
+      - run: |
+          make -j
+          make -C tests -j build
+      - run: |
+          examples/rdkafka_example -V || true
+          examples/rdkafka_example -X builtin.features
+      - run: |
+          make -C tests run_local_quick
+
+  style:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          sudo apt install -y python3 python3-pip python3-setuptools clang-format
+          python3 -m pip install -r packaging/tools/requirements.txt
+      - name: Style checker
+        run: make style-check

.gitignore

@@ -31,4 +31,5 @@ cov-int
 gdbrun*.gdb
 TAGS
 nbproject
-.vscode
+.vscode
+vcpkg_installed

.travis.yml

@@ -1,5 +1,5 @@
 language: c
-dist: trusty
+dist: xenial
 cache: ccache
 
 addons:
@@ -13,8 +13,8 @@ addons:
 
 env:
   global:
-   - secure: "N6iA5+uNcMVvl6KWIiNMdm8MbTX1sviOG2cKBehL+t36ebQOmsi6amRhNseVHdgboALHeWqKf7N8OdMZGbkE8MkgC4mZk+oBunCAZCyBFqwFmVTYuCvsk5Zr1RFw/GbeiGHTmHQYKOol5ct18L6zQnsxV136uxDOOgVk2TizRO8="
-   - secure: "XcfZ5u15+saEPVmqM6hBfrdNcZdAZgZhRwTSpTMtFagByb7kE/XtIL/n3qNoDrmgAG2cu8chWI6Tj8Jd1gQU6TjbePfoDK3hRH7gtRumw4vGwFf3bCalOCED6ekYy9qTE+Eymfax8QsH4Qs1e7duFK4d6AeIljG3M01bjnutCIY="
+  - secure: "q7DQ6KCiQyMEpBf8mxPFl6hY9JEoaOUdIaLh1IuYn5TctiNIA+J6O/bL/dyDSy2Yjor61WAiiMOh77eMykm1wPl72kqjR97ui0uCq7BQQn4MWtKrXXi0eWLF3bYt2FbUGJZvrM0xeoWzSYT6np7CKu8ssgL8Fvr4bmf152IpdQ8="
+  - secure: "XpFExynXwbSr6vTuGsZVyqF4sti+UmRxX2sztjpTdaIH0yo60d6KYT0SRW7BLdZNA6/XI1l1GPTAwcDwTM1XasnnFrD7i88uZsAneA/xEgZTGXtnVVWPJAcVoX/75Rxeibc8CfSc5MO9QmBMiGGuI3S6HHCj4RzCJacBhOjIhfA="
 
 matrix:
  include:
@@ -32,7 +32,7 @@ matrix:
    compiler: gcc
    env: ADDITIONAL_BUILDS="centos" SKIP_MAKE=y
 
- - name: "Linux clang: +alpine +manylinux"
+ - name: "Linux clang: +alpine +manylinux +werror"
    os: linux
    compiler: clang
    env: ADDITIONAL_BUILDS="alpine manylinux2010_x86_64" LINKAGE=std
@@ -51,19 +51,17 @@ matrix:
    if: tag IS PRESENT
    os: osx
    compiler: gcc
-   env: LINKAGE=std
+   env: LINKAGE=std HOMEBREW_NO_AUTO_UPDATE=1
    before_script:
-    - ./configure --install-deps --disable-lz4-ext --prefix="$PWD/dest" --enable-werror --enable-strip
+    - ./configure --install-deps --source-deps-only --disable-lz4-ext --prefix="$PWD/dest" --enable-strip
 
  - name: "OSX clang: +static"
+   if: tag IS PRESENT
    os: osx
-   # Use an older image to disable syslog and for broader compatibility
-   # with old and new osx versions.
-   osx_image: xcode9.2
    compiler: clang
    env: LINKAGE=static HOMEBREW_NO_AUTO_UPDATE=1
    before_script:
-    - ./configure --install-deps --disable-lz4-ext --prefix="$PWD/dest" --enable-static --disable-syslog --enable-strip
+    - ./configure --install-deps --source-deps-only --disable-lz4-ext --prefix="$PWD/dest" --enable-static --enable-strip
 
  - name: "Windows MinGW-w64 Dynamic"
    if: tag IS PRESENT
@@ -110,6 +108,7 @@ matrix:
     - ./configure --disable-gssapi --install-deps --source-deps-only --enable-static --disable-lz4-ext --prefix="$PWD/dest" --enable-strip
 
  - name: "Linux GCC s390x: +devel"
+   if: tag IS PRESENT
    os: linux
    arch: s390x
    dist: bionic
@@ -143,13 +142,14 @@ script:
 - if [[ $DOC_CHECK == y ]]; then make docs || travis_terminate 1 ; fi
 - if [[ -z $TRAVIS_TAG && $RUN_INTEGRATION_TESTS == y ]]; then (cd tests && travis_retry ./interactive_broker_version.py -c "make quick" 2.7.0) || travis_terminate 1 ; fi
 - if [[ -f tests/core ]] && (which gdb >/dev/null); then (cd tests && LD_LIBRARY_PATH=../src:../src-cpp gdb ./test-runner core < backtrace.gdb) ; fi
+- sha256sum artifacts/* || true
 
 deploy:
   provider: s3
   access_key_id:
-    secure: "m8FQrFesK0xSS1wHo2S7cuWkpO7VB91dBmj1XIYLRXZSkbMpKBJATcFcHNbrAp3slEp7wLAnT7CHrQ4ccQi4H68Z7mjEwdq4VKRE+7zqJ/feK8MOFNeSHWLQzgwLUYlRlc9+tzLNwxMuL2ilWgdjKOArsUVHo9LEKNfQ3T6zCJU="
+    secure: "sRsKY1YoPDb3b+9hHnBv4tDSdyB/FraYEKI1/+aKmqWxvOI6xYYFFP0Tvn6f4Rgk0wzYmxO/5V+cR+fmKxVhb1pItFXOdVqML0ilOTP5gtlOPUeHu9fytqw3q7GgMV8JR75g60BNVko9vZegtd2LIq6FWzAIvPSUJOAw7qekjGU="
   secret_access_key:
-    secure: "GE6O0gk5VRervntCKAmczfBdSOvbr9bouJ15H2rpcOgHi8KTDEjI/NS69eLiRRSHBCARtcRqN4wfgy+/dn7D1VklY8a1rAKu02wGjw+fq7k7GVSSmynR/aF619R4SIABsaAhNCwswXnLHuLlq8HFk5ulG3z8DUvYBczB45bWZfQ="
+    secure: "ZDjH6Z9CJr2yo7Splm+0xpo30QbO+cpeqxFUn1d9XOyLZQ0dapr6iboxdPlJaCOIhqVUWXS0IJgFwCW+5vWb9Za6tFumP1MtJGiwE6bqr820G8E02umwSvbNijr44h+EyxQcxP71Ljjk22Pfu7SLKWqMJ/iIzcYe6Z6Sz8obSWA="
   bucket: librdkafka-ci-packages
   region: us-west-1
   skip_cleanup: true

CHANGELOG.md

@@ -1,3 +1,89 @@
+# librdkafka v1.9.0
+
+librdkafka v1.9.0 is a feature release:
+
+
+## Enhancements
+
+ * SASL OAUTHBEARER refresh callbacks can now be scheduled for execution
+   on librdkafka's background thread. This solves the problem where an
+   application has a custom SASL OAUTHBEARER refresh callback and thus needs to
+   call `rd_kafka_poll()` (et.al.) at least once to trigger the
+   refresh callback before being able to connect to brokers.
+   With the new `rd_kafka_conf_enable_sasl_queue()` configuration API and
+   `rd_kafka_sasl_background_callbacks_enable()` the refresh callbacks
+   can now be triggered automatically on the librdkafka background thread.
+ * `rd_kafka_queue_get_background()` now creates the background thread
+   if not already created.
+
+
+## Fixes
+
+### General fixes
+
+ * Windows: some applications would crash with an error message like
+   `no OPENSSL_Applink()` written to the console if `ssl.keystore.location`
+   was configured.
+   This regression was introduced in v1.8.0 due to use of vcpkgs and how
+   keystore file was read. #3554.
+
+
+
+# librdkafka v1.8.2
+
+librdkafka v1.8.2 is a maintenance release.
+
+## Enhancements
+
+ * Added `ssl.ca.pem` to add CA certificate by PEM string. (#2380)
+ * Prebuilt binaries for Mac OSX now contain statically linked OpenSSL v1.1.1l.
+   Previously the OpenSSL version was either v1.1.1 or v1.0.2 depending on
+   build type.
+
+## Fixes
+
+ * The `librdkafka.redist` 1.8.0 package had two flaws:
+   - the linux-arm64 .so build was a linux-x64 build.
+   - the included Windows MSVC 140 runtimes for x64 were infact x86.
+   The release script has been updated to verify the architectures of
+   provided artifacts to avoid this happening in the future.
+ * Prebuilt binaries for Mac OSX Sierra (10.12) and older are no longer provided.
+   This affects [confluent-kafka-go](https://github.com/confluentinc/confluent-kafka-go).
+ * Some of the prebuilt binaries for Linux were built on Ubuntu 14.04,
+   these builds are now performed on Ubuntu 16.04 instead.
+   This may affect users on ancient Linux distributions.
+ * It was not possible to configure `ssl.ca.location` on OSX, the property
+   would automatically revert back to `probe` (default value).
+   This regression was introduced in v1.8.0. (#3566)
+ * librdkafka's internal timers would not start if the timeout was set to 0,
+   which would result in some timeout operations not being enforced correctly,
+   e.g., the transactional producer API timeouts.
+   These timers are now started with a timeout of 1 microsecond.
+
+### Transactional producer fixes
+
+ * Upon quick repeated leader changes the transactional producer could receive
+   an `OUT_OF_ORDER_SEQUENCE` error from the broker, which triggered an
+   Epoch bump on the producer resulting in an InitProducerIdRequest being sent
+   to the transaction coordinator in the middle of a transaction.
+   This request would start a new transaction on the coordinator, but the
+   producer would still think (erroneously) it was in current transaction.
+   Any messages produced in the current transaction prior to this event would
+   be silently lost when the application committed the transaction, leading
+   to message loss.
+   This has been fixed by setting the Abortable transaction error state
+   in the producer. #3575.
+ * The transactional producer could stall during a transaction if the transaction
+   coordinator changed while adding offsets to the transaction (send_offsets_to_transaction()).
+   This stall lasted until the coordinator connection went down, the
+   transaction timed out, transaction was aborted, or messages were produced
+   to a new partition, whichever came first. #3571.
+
+
+
+*Note: there was no v1.8.1 librdkafka release*
+
+
 # librdkafka v1.8.0
 
 librdkafka v1.8.0 is a security release: