Merge branch 'develop'

.clang-format

@@ -0,0 +1,92 @@
+---
+Language:        Cpp
+# BasedOnStyle:  LLVM
+AccessModifierOffset: -2
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: false
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: false
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Custom
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IncludeCategories:
+  - Regex:           '^"config'
+    Priority:        -1
+  - Regex:           '^<'
+    Priority:        4
+  - Regex:           '^"'
+    Priority:        5
+IndentCaseLabels: false
+IndentWidth:     8
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+JavaScriptQuotes: Leave
+...

.gitlab-ci.yml

@@ -1,5 +1,6 @@
 stages:
   - build
+  - test
   - upload
 
 build-amd64-debian-jessie:
@@ -72,6 +73,134 @@ build-amd64-ubuntu-cosmic:
   only:
     - tags
 
+build-amd64-fedora-29:
+  stage: build
+  image: fedora:29
+  script:
+    - ./gitlab-build-rpm.sh fedora29
+    - mkdir -p built-packages/amd64/fedora_29/
+    - mv ~/rpmbuild/RPMS/x86_64/*.rpm built-packages/amd64/fedora_29/
+  artifacts:
+    paths:
+      - built-packages/*
+    expire_in: 1 day
+  only:
+    - tags
+
+build-amd64-fedora-28:
+  stage: build
+  image: fedora:28
+  script:
+    - ./gitlab-build-rpm.sh fedora28
+    - mkdir -p built-packages/amd64/fedora_28/
+    - mv ~/rpmbuild/RPMS/x86_64/*.rpm built-packages/amd64/fedora_28/
+  artifacts:
+    paths:
+      - built-packages/*
+    expire_in: 1 day
+  only:
+    - tags
+
+build-amd64-centos7:
+  stage: build
+  image: centos:7
+  script:
+    - ./gitlab-build-rpm.sh centos7
+    - mkdir -p built-packages/amd64/centos_7/
+    - mv ~/rpmbuild/RPMS/x86_64/*.rpm built-packages/amd64/centos_7/
+  artifacts:
+    paths:
+      - built-packages/*
+    expire_in: 1 day
+  only:
+    - tags
+
+build-amd64-centos6:
+  stage: build
+  image: centos:6
+  script:
+    - ./gitlab-build-rpm.sh centos6
+    - mkdir -p built-packages/amd64/centos_6/
+    - mv ~/rpmbuild/RPMS/x86_64/*.rpm built-packages/amd64/centos_6/
+  artifacts:
+    paths:
+      - built-packages/*
+    expire_in: 1 day
+  only:
+    - tags
+
+run-unit-tests-ubuntu-xenial:
+  stage: test
+  image: ubuntu:xenial
+  script:
+    - ./run-tests.sh ubuntu_xenial
+  only:
+    - tags
+
+run-unit-tests-ubuntu-bionic:
+  stage: test
+  image: ubuntu:bionic
+  script:
+    - ./run-tests.sh ubuntu_bionic
+  only:
+    - tags
+
+run-unit-tests-ubuntu-cosmic:
+  stage: test
+  image: ubuntu:cosmic
+  script:
+    - ./run-tests.sh ubuntu_cosmic
+  only:
+    - tags
+
+run-unit-tests-debian-stretch:
+  stage: test
+  image: debian:stretch
+  script:
+    - ./run-tests.sh debian_stretch
+  only:
+    - tags
+
+run-unit-tests-debian-jessie:
+  stage: test
+  image: debian:jessie
+  script:
+    - ./run-tests.sh debian_jessie
+  only:
+    - tags
+
+run-unit-tests-fedora-29:
+  stage: test
+  image: fedora:29
+  script:
+    - ./run-tests-rpm.sh fedora_29
+  only:
+    - tags
+
+run-unit-tests-fedora-28:
+  stage: test
+  image: fedora:28
+  script:
+    - ./run-tests-rpm.sh fedora_28
+  only:
+    - tags
+
+run-unit-tests-centos-6:
+  stage: test
+  image: centos:6
+  script:
+    - ./run-tests-rpm.sh centos_6
+  only:
+    - tags
+
+run-unit-tests-centos-7:
+  stage: test
+  image: centos:7
+  script:
+    - ./run-tests-rpm.sh centos_7
+  only:
+    - tags
+
 upload-packages:
   stage: upload
   image: ubuntu:bionic

AUTHORS

@@ -1,15 +1,17 @@
-The primary authors of libwandio were:
+The original authors of libwandio were:
  * Perry Lorier
  * Shane Alcock
 
 Libwandio is currently maintained by Shane Alcock <[email protected]>.
 
 Thanks to Alistair King for numerous contributions (including the entirety
-of the HTTP reader), bug reports and being kind enough to take on the job of 
+of the HTTP reader), bug reports and being kind enough to take on the job of
 separating libwandio from libtrace.
 
 Thanks to Robert Zeh for adding support for zstd.
 
+Thanks to Sergey Cherepanov for adding support for lz4.
+
 Thanks to Mingwei Zhang for adding ability to resume HTTP reads and fixing
 a ton of lazy error checking issues.
 

ChangeLog

@@ -1,3 +1,17 @@
+Version 4.1.0
+=============
+Thanks to Sergey Cherepanov and Alistair King for contributing to this release
+
+ * Added support for both reading and writing lz4 compressed files.
+ * Added support for using Intel QuickAssist (if present) for reading
+   and writing gzip compressed files.
+ * Added new functionality for interacting with wandio files
+   (wandio_fgets(), wandio_vprintf(), wandio_printf())
+ * Added API function for determining correct compression method
+   based on output filename extension (wandio_detect_compression_type())
+ * Some potential performance improvements by aligning buffers sensibly.
+ * Fixed "inline" errors when building with LLVM.
+
 Version 4.0.0
 =============
 Thanks to Alistair King, Mingwei Zhang and Robert Zeh for their contributions

INSTALL

@@ -7,6 +7,9 @@ Strongly recommended:
  * zlib-dev
  * libbz2-dev
  * liblzma-dev
+ * liblzo2-dev
+ * liblz4-dev
+ * libzstd-dev
 
 ----------------------------------
 

README

@@ -1,7 +1,7 @@
-WANDIO 4.0.0
+WANDIO 4.1.0
 
 ---------------------------------------------------------------------------
-Copyright (c) 2007-2018 The University of Waikato, Hamilton, New Zealand.
+Copyright (c) 2007-2019 The University of Waikato, Hamilton, New Zealand.
 All rights reserved.
 
 This code has been developed by the University of Waikato WAND
@@ -18,16 +18,19 @@ provides transparent compression/decompression for the following formats:
  - lzo (write-only)
  - lzma
  - zstd
+ - lz4
+ - Intel QAT (write-only)
  - http (read-only)
 
 WANDIO also improves IO performance by performing compression/decompression in a
 separate thread (if pthreads are available).
 
+Documentation for WANDIO and its included tools can be found at
+https://github.com/wanduow/wandio/wiki
+
 Bugs should be reported by either emailing [email protected] or filing
 an issue at https://github.com/wanduow/wandio
 
-It is licensed under the Lesser GNU General Public License (LGPL) version 3. Please
-see the included files COPYING and COPYING.LESSER for details of this license.
-
-For further information, please contact the WAND group. See
-http://www.wand.net.nz/ for details.
+It is licensed under the Lesser GNU General Public License (LGPL) version 3.
+Please see the included files COPYING and COPYING.LESSER for details of this
+license.

bintray-upload.sh

@@ -2,14 +2,35 @@
 
 set -x -e -o pipefail
 
-apt-get update && apt-get install -y curl
+apt-get update && apt-get install -y curl util-linux
 
 for path in `find built-packages/amd64/ -maxdepth 1 -type d`; do
     IFS=_ read linux_dist linux_version <<< $(basename "${path}")
     for deb in `find "${path}" -maxdepth 1 -type f`; do
+
+        ext=${deb##*.}
         pkg_filename=$(basename "${deb}")
-        IFS=_ read pkg_name pkg_version pkg_arch <<< $(basename -s ".deb" "${pkg_filename}")
-        curl -T "${deb}" -u${BINTRAY_USERNAME}:${BINTRAY_API_KEY} \
-            "https://api.bintray.com/content/wand/general/${pkg_name}/${pkg_version}/pool/${linux_version}/main/${pkg_filename};deb_distribution=${linux_version};deb_component=main;deb_architecture=${pkg_arch}"
+
+        if [ "$ext" = "deb" ]; then
+            IFS=_ read pkg_name pkg_version pkg_arch <<< $(basename -s ".deb" "${pkg_filename}")
+            curl -T "${deb}" -u${BINTRAY_USERNAME}:${BINTRAY_API_KEY} \
+                "https://api.bintray.com/content/wand/general/${pkg_name}/${pkg_version}/pool/${linux_version}/main/${pkg_filename};deb_distribution=${linux_version};deb_component=main;deb_architecture=${pkg_arch}"
+        fi
+
+        if [ "$ext" = "rpm" ]; then
+
+            rev_filename=`echo ${pkg_filename} | rev`
+
+            if [[ "$1" =~ centos_* ]]; then
+                pkg_name=`echo ${rev_filename} | cut -d '-' -f4- | rev`
+                pkg_version=`echo ${rev_filename} | cut -d '-' -f1-3 | rev | cut -d '.' -f1-3`
+            else
+                pkg_name=`echo ${rev_filename} | cut -d '-' -f3- | rev`
+                pkg_version=`echo ${rev_filename} | cut -d '-' -f1-2 | rev | cut -d '.' -f1-3`
+            fi
+
+            curl -T "${deb}" -u${BINTRAY_USERNAME}:${BINTRAY_API_KEY} \
+                "https://api.bintray.com/content/wand/general-rpm/${pkg_name}/${pkg_version}/${pkg_filename}"
+        fi
     done
 done

bootstrap.sh

@@ -1,21 +1,23 @@
 #! /bin/sh
 
-set -x
+set +x
 # Prefer aclocal 1.9 if we can find it
-aclocal-1.11 -I m4 ||
-	aclocal-1.9 -I m4 ||
+(which aclocal-1.11 &> /dev/null && aclocal-1.11 -I m4) ||
+(which aclocal-1.9 &>/dev/null && aclocal-1.9 -I m4) ||
 	aclocal  -I m4
 
 # Darwin bizarrely uses glibtoolize
-libtoolize --force --copy ||
+(which libtoolize &>/dev/null && libtoolize --force --copy) ||
 	glibtoolize --force --copy
 
-autoheader2.50 || autoheader
+(which autoheader2.50 &>/dev/null && autoheader2.50) || 
+autoheader
 
 # Prefer automake-1.9 if we can find it
-automake-1.11 --add-missing --copy --foreign ||
-	automake-1.10 --add-missing --copy --foreign ||
-	automake-1.9 --add-missing --copy --foreign ||
+(which automake-1.11 &>/dev/null && automake-1.11 --add-missing --copy --foreign) ||
+(which automake-1.10 &>/dev/null && automake-1.10 --add-missing --copy --foreign) ||
+(which automake-1.9  &>/dev/null && automake-1.9  --add-missing --copy --foreign) ||
 	automake --add-missing --copy --foreign
 
-autoconf2.50 || autoconf
+(which autoconf2.50 &>/dev/null && autoconf2.50) || 
+autoconf