Update dependencies (#245)

* update denpendencies to latest

* add libzstd-dev to travis

* use xenial as travis base image

* try to solve zstd compile errors with go get instead of apt

* move go get from before install to install

* add zstd to apt-get again

* disable CGO for travis builds

* enable cgo again for pipeline-accept steps

* explicitly exclude vendor errors from linting

* enable CGO again for pipleine builds

* fix dependencies complaining about being built with newer go versions

* calm linter about unused variable

* re-vendor go modules

* add .c and .h files when present in vendor

* disable go modules for go 1.11

this will suppress errors regarding the x/sys module requireing go 1.12

.gitignore

@@ -36,5 +36,7 @@ vendor/**/*
 !vendor/**/
 !vendor/**/*.go
 !vendor/**/*.s
+!vendor/**/*.h
+!vendor/**/*.c
 !vendor/**/LICENSE
 vendor/**/*_test.go

.travis.yml

@@ -1,6 +1,6 @@
 language: go
 sudo: required
-dist: trusty
+dist: xenial
 
 git:
   depth: 3
@@ -12,7 +12,7 @@ matrix:
       env: ACCEPT_TARGET=pipeline-accept
     - 
       go: "1.11"
-      env: ACCEPT_TARGET=pipeline-accept
+      env: ACCEPT_TARGET=pipeline-accept GOMOD=off
     - 
       go: "1.10"
       env: ACCEPT_TARGET=pipeline-accept
@@ -21,12 +21,8 @@ matrix:
       env: ACCEPT_TARGET=test
 
 #before_install:
-#  - sudo apt-get -qq update
-#  - sudo apt-get install -qqy libpcap-dev libsystemd-journal-dev
-#  - wget http://launchpadlibrarian.net/234454186/librdkafka1_0.8.6-1.1_amd64.deb
-#  - wget http://launchpadlibrarian.net/234454185/librdkafka-dev_0.8.6-1.1_amd64.deb
-#  - sudo dpkg -i librdkafka1_0.8.6-1.1_amd64.deb
-#  - sudo dpkg -i librdkafka-dev_0.8.6-1.1_amd64.deb
+#  - sudo apt-get -qqy update
+#  - sudo apt-get -qqy install libzstd-dev libpcap-dev libsystemd-journal-dev libzstd-dev librdkafka-dev
 
 install:
   - make pipeline-tools

consumer/syslogd.go

@@ -172,7 +172,7 @@ func parseCustomFields(data string, metadata *core.Metadata) {
 		}
 
 		i := startOfValue
-		endOfValue := i + 1
+		var endOfValue int
 		hasQuotes := false
 		for {
 			endOfValue = strings.IndexByte(data[i:], '"')

core/metrics.go

@@ -21,7 +21,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rcrowley/go-metrics"
+	metrics "github.com/rcrowley/go-metrics"
 	"github.com/trivago/tgo/tmath"
 )
 

format/processtsv_test.go

@@ -197,7 +197,7 @@ func TestProcessTSVDirectives(t *testing.T) {
 	expect.NoError(err)
 
 	expect.Equal(
-		"2016-08-19 17:10:53\t\"preyamlpost\"\tFirefox\tWindows NT 10.0\t48.0\t5.0\tWindows\tWindows NT 10.0\trv:48.0\tGecko\t20100101\tFirefox\t48.0",
+		"2016-08-19 17:10:53\t\"preyamlpost\"\tFirefox\tWindows 10\t48.0\t5.0\tWindows\tWindows 10\t\tGecko\t20100101\tFirefox\t48.0",
 		msg.String(),
 	)
 }
@@ -236,7 +236,7 @@ func TestProcessTSVDelimiterAndDirectives(t *testing.T) {
 	expect.NoError(err)
 
 	expect.Equal(
-		`2016-08-19 17:10:53,"preyamlpost",Firefox,Windows NT 10.0,48.0,5.0,Windows,Windows NT 10.0,rv:48.0,Gecko,20100101,Firefox,48.0`,
+		`2016-08-19 17:10:53,"preyamlpost",Firefox,Windows 10,48.0,5.0,Windows,Windows 10,,Gecko,20100101,Firefox,48.0`,
 		msg.String(),
 	)
 }
@@ -276,7 +276,7 @@ func TestProcessTSVQuotedValuesAndDirectives(t *testing.T) {
 	expect.NoError(err)
 
 	expect.Equal(
-		"\"2016-08-19 17:10:53\"\t\"pre\tyaml\tpost\"\tFirefox\tWindows NT 10.0\t48.0\t5.0\tWindows\tWindows NT 10.0\trv:48.0\tGecko\t20100101\tFirefox\t48.0",
+		"\"2016-08-19 17:10:53\"\t\"pre\tyaml\tpost\"\tFirefox\tWindows 10\t48.0\t5.0\tWindows\tWindows 10\t\tGecko\t20100101\tFirefox\t48.0",
 		msg.String(),
 	)
 }
@@ -319,7 +319,7 @@ func TestProcessTSVDelimiterAndQuotedValuesAndDirectives(t *testing.T) {
 	expect.NoError(err)
 
 	expect.Equal(
-		`"2016-08-19 17:10:53" "pre yaml post" Firefox "Windows NT 10.0" 48.0 5.0 Windows "Windows NT 10.0" rv:48.0 Gecko 20100101 Firefox 48.0`,
+		`"2016-08-19 17:10:53" "pre yaml post" Firefox "Windows 10" 48.0 5.0 Windows "Windows 10"  Gecko 20100101 Firefox 48.0`,
 		msg.String(),
 	)
 }

go.mod

@@ -1,53 +1,40 @@
 module github.com/trivago/gollum
 
 require (
-	github.com/MeteoGroup/go-metrics-prometheus v0.0.0-20170102121754-1d412ec2ed4f
-	github.com/Shopify/sarama v1.17.0
+	github.com/CrowdStrike/go-metrics-prometheus v0.0.0-20190206080537-f2bb39ee2c39
+	github.com/Shopify/sarama v1.22.0
 	github.com/abbot/go-http-auth v0.4.0
 	github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0
 	github.com/artyom/fb303 v0.0.0-20130902104109-fa4a241cefb1
 	github.com/artyom/scribe v0.0.0-20130902104122-35c1da66e76d
 	github.com/artyom/thrift v0.0.0-20130902103359-388840a05deb
-	github.com/aws/aws-sdk-go v1.15.22
-	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
+	github.com/aws/aws-sdk-go v1.19.12
 	github.com/bsm/sarama-cluster v2.1.15+incompatible
-	github.com/coreos/go-systemd v0.0.0-20180705093442-88bfeed483d3
-	github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/deathowl/go-metrics-prometheus v0.0.0-20170731141557-091131e49c33
-	github.com/eapache/go-resiliency v1.1.0 // indirect
-	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
-	github.com/eapache/queue v1.1.0 // indirect
+	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
+	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
 	github.com/fsnotify/fsnotify v1.4.7
-	github.com/go-redis/redis v6.14.0+incompatible
-	github.com/golang/protobuf v1.2.0
-	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
-	github.com/gorilla/websocket v1.3.0
-	github.com/mattn/go-colorable v0.0.9 // indirect
-	github.com/mattn/go-isatty v0.0.3 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/go-redis/redis v6.15.2+incompatible
+	github.com/golang/protobuf v1.3.1
+	github.com/gorilla/websocket v1.4.0
+	github.com/mattn/go-colorable v0.1.1 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/miekg/pcap v0.0.0-20170124221734-51d9d986bf8d
-	github.com/mmcloughlin/geohash v0.0.0-20180625052535-3b756d8ac3d9
-	github.com/mssola/user_agent v0.4.1
+	github.com/mmcloughlin/geohash v0.9.0
+	github.com/mssola/user_agent v0.5.0
+	github.com/onsi/ginkgo v1.8.0 // indirect
+	github.com/onsi/gomega v1.5.0 // indirect
 	github.com/oschwald/maxminddb-golang v1.3.0 // indirect
-	github.com/pierrec/lz4 v2.0.3+incompatible // indirect
-	github.com/pkg/errors v0.8.0
-	github.com/prometheus/client_golang v0.8.0
-	github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect
-	github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e // indirect
-	github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 // indirect
+	github.com/pkg/errors v0.8.1
+	github.com/prometheus/client_golang v0.9.2
 	github.com/quipo/statsd v0.0.0-20180118161217-3d6a5565f314
-	github.com/rcrowley/go-metrics v0.0.0-20180503174638-e2704e165165
-	github.com/sirupsen/logrus v1.0.6
+	github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a
+	github.com/sirupsen/logrus v1.4.1
 	github.com/trivago/grok v1.0.0
-	github.com/trivago/tgo v1.0.5
+	github.com/trivago/tgo v1.0.6
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2
-	golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac
-	golang.org/x/net v0.0.0-20180826012351-8a410e7b638d // indirect
-	golang.org/x/sys v0.0.0-20180828065106-d99a578cf41b // indirect
+	golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a
 	gopkg.in/mcuadros/go-syslog.v2 v2.2.1
-	gopkg.in/olivere/elastic.v5 v5.0.74
+	gopkg.in/olivere/elastic.v5 v5.0.80
 	gopkg.in/oschwald/geoip2-golang.v1 v1.2.1
-	gopkg.in/yaml.v2 v2.2.1
+	gopkg.in/yaml.v2 v2.2.2
 )