-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #13 from azarakovskiy/kafka-consumer-built-in
Add proton internal consumer
- Loading branch information
Showing
11 changed files
with
792 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,148 @@ | ||
package cmd | ||
|
||
import ( | ||
"bytes" | ||
"context" | ||
"log" | ||
"os" | ||
"os/signal" | ||
"strconv" | ||
"strings" | ||
|
||
"github.com/Shopify/sarama" | ||
"github.com/beatlabs/proton/v2/internal/consumer" | ||
"github.com/beatlabs/proton/v2/internal/json" | ||
"github.com/beatlabs/proton/v2/internal/output" | ||
"github.com/beatlabs/proton/v2/internal/protoparser" | ||
"github.com/spf13/cobra" | ||
) | ||
|
||
// consumeCmd represents the consume command | ||
var consumeCmd = &cobra.Command{ | ||
Use: "consume", | ||
Short: "consume from given topics", | ||
Run: Run, | ||
} | ||
|
||
// ConsumeCfg is the config for everything this tool needs. | ||
type ConsumeCfg struct { | ||
consumerCfg consumer.Cfg | ||
offsets []string | ||
model string | ||
format string | ||
} | ||
|
||
var consumeCfg = &ConsumeCfg{ | ||
consumerCfg: consumer.Cfg{}, | ||
} | ||
|
||
func init() { | ||
rootCmd.AddCommand(consumeCmd) | ||
|
||
consumeCmd.Flags().StringVarP(&consumeCfg.consumerCfg.URL, "broker", "b", "", "Broker URL to consume from") | ||
if consumeCmd.MarkFlagRequired("broker") != nil { | ||
log.Fatal("you must specify a a broker URL using the `-b <url>` option") | ||
} | ||
|
||
consumeCmd.Flags().StringVarP(&consumeCfg.consumerCfg.Topic, "topic", "t", "", "A topic to consume from") | ||
if consumeCmd.MarkFlagRequired("topic") != nil { | ||
log.Fatal("you must specify a topic to consume using the `-t <topic>` option") | ||
} | ||
|
||
consumeCmd.Flags().StringVarP(&consumeCfg.model, "proto", "", "", "A path to a proto file an URL to it") | ||
if consumeCmd.MarkFlagRequired("proto") != nil { | ||
log.Fatal("you must specify a proto file using the `-m <path>` option") | ||
} | ||
|
||
consumeCmd.Flags().StringVarP(&consumeCfg.format, "format", "f", "%Tf: %s", ` | ||
A Kcat-like format string. Defaults to "%T: %s". | ||
Format string tokens: | ||
%s Message payload | ||
%k Message key | ||
%t Topic | ||
%p Partition | ||
%o Offset | ||
%T Message timestamp (milliseconds since epoch UTC) | ||
%Tf Message time formatted as RFC3339 | ||
\n \r \t Newlines, tab | ||
Example: | ||
-f 'Key: %k, Time: %Tf \nValue: %s'`) | ||
|
||
consumeCmd.Flags().StringSliceVarP(&consumeCfg.offsets, "offsets", "o", []string{}, ` | ||
Offset to start consuming from | ||
s@<value> (timestamp in ms to start at) | ||
e@<value> (timestamp in ms to stop at (not included)) | ||
`) | ||
|
||
consumeCmd.Flags().StringVarP(&consumeCfg.consumerCfg.KeyGrep, "key", "", ".*", "Grep RegExp for a key value") | ||
|
||
consumeCmd.Flags().BoolVarP(&consumeCfg.consumerCfg.Verbose, "verbose", "v", false, "Whether to print out proton's debug messages") | ||
} | ||
|
||
// Run runs this whole thing. | ||
func Run(cmd *cobra.Command, _ []string) { | ||
ctx, cancel := context.WithCancel(cmd.Context()) | ||
defer cancel() | ||
|
||
protoParser, fileName, err := protoparser.New(ctx, consumeCfg.model) | ||
if err != nil { | ||
log.Fatal(err) | ||
} | ||
|
||
consumeCfg.consumerCfg.Start, consumeCfg.consumerCfg.End = parseOffsets(consumeCfg.offsets) | ||
|
||
kafka, err := consumer.NewKafka(ctx, consumeCfg.consumerCfg, | ||
&protoDecoder{json.Converter{ | ||
Parser: protoParser, | ||
Filename: fileName, | ||
}}, output.NewFormatterPrinter(consumeCfg.format, os.Stdout, os.Stderr)) | ||
|
||
if err != nil { | ||
log.Fatal(err) | ||
} | ||
|
||
signals := make(chan os.Signal, 1) | ||
signal.Notify(signals, os.Interrupt) | ||
|
||
errCh := kafka.Run() | ||
|
||
select { | ||
case err := <-errCh: | ||
if err != nil { | ||
log.Fatal(err) | ||
} | ||
case _ = <-signals: | ||
break | ||
} | ||
} | ||
|
||
type protoDecoder struct { | ||
json.Converter | ||
} | ||
|
||
// Decode uses the existing json decoder and adapts it to this consumer. | ||
func (p *protoDecoder) Decode(rawData []byte) (string, error) { | ||
stream, errCh := p.ConvertStream(bytes.NewReader(rawData)) | ||
select { | ||
case msg := <-stream: | ||
return string(msg), nil | ||
case err := <-errCh: | ||
return "", err | ||
} | ||
} | ||
|
||
func parseOffsets(offsets []string) (int64, int64) { | ||
return parseOffset("s@", offsets, sarama.OffsetOldest), parseOffset("e@", offsets, sarama.OffsetNewest) | ||
} | ||
|
||
func parseOffset(prefix string, offsets []string, defaultVal int64) int64 { | ||
for _, offset := range offsets { | ||
if strings.HasPrefix(offset, prefix) { | ||
v, err := strconv.Atoi(offset[len(prefix):]) | ||
if err == nil { | ||
return int64(v) | ||
} | ||
} | ||
} | ||
return defaultVal | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
package cmd | ||
|
||
import ( | ||
"testing" | ||
|
||
"github.com/Shopify/sarama" | ||
"github.com/stretchr/testify/assert" | ||
) | ||
|
||
func TestParseOffsets(t *testing.T) { | ||
tests := []struct { | ||
name string | ||
given []string | ||
startTime, endTime int64 | ||
}{ | ||
{ | ||
name: "no offsets specified", | ||
given: []string{}, | ||
startTime: sarama.OffsetOldest, | ||
endTime: sarama.OffsetNewest, | ||
}, | ||
{ | ||
name: "start offset specified", | ||
given: []string{"s@24"}, | ||
startTime: 24, | ||
endTime: sarama.OffsetNewest, | ||
}, | ||
{ | ||
name: "end offset specified", | ||
given: []string{"e@42"}, | ||
startTime: sarama.OffsetOldest, | ||
endTime: 42, | ||
}, | ||
{ | ||
name: "both offsets specified", | ||
given: []string{"s@24", "e@42"}, | ||
startTime: 24, | ||
endTime: 42, | ||
}, | ||
{ | ||
name: "multiple offsets specified", | ||
given: []string{"s@24", "e@42", "s@123", "e@321"}, | ||
startTime: 24, | ||
endTime: 42, | ||
}, | ||
} | ||
for _, test := range tests { | ||
t.Run(test.name, func(t *testing.T) { | ||
// given | ||
// when | ||
r1, r2 := parseOffsets(test.given) | ||
|
||
// then | ||
assert.Equal(t, test.startTime, r1) | ||
assert.Equal(t, test.endTime, r2) | ||
}) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,14 +1,53 @@ | ||
module github.com/beatlabs/proton/v2 | ||
|
||
go 1.15 | ||
go 1.17 | ||
|
||
require ( | ||
github.com/Shopify/sarama v1.32.0 | ||
github.com/golang/protobuf v1.4.1 | ||
github.com/jhump/protoreflect v1.7.0 | ||
github.com/mitchellh/go-homedir v1.1.0 | ||
github.com/spf13/cobra v1.0.0 | ||
github.com/spf13/viper v1.7.1 | ||
github.com/stretchr/testify v1.6.1 | ||
github.com/stretchr/testify v1.7.0 | ||
google.golang.org/protobuf v1.25.0 | ||
gopkg.in/h2non/gock.v1 v1.0.15 | ||
) | ||
|
||
require ( | ||
github.com/davecgh/go-spew v1.1.1 // indirect | ||
github.com/eapache/go-resiliency v1.2.0 // indirect | ||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect | ||
github.com/eapache/queue v1.1.0 // indirect | ||
github.com/fsnotify/fsnotify v1.4.7 // indirect | ||
github.com/golang/snappy v0.0.4 // indirect | ||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect | ||
github.com/hashicorp/go-uuid v1.0.2 // indirect | ||
github.com/hashicorp/hcl v1.0.0 // indirect | ||
github.com/inconshreveable/mousetrap v1.0.0 // indirect | ||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect | ||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect | ||
github.com/jcmturner/gofork v1.0.0 // indirect | ||
github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect | ||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect | ||
github.com/klauspost/compress v1.14.4 // indirect | ||
github.com/magiconair/properties v1.8.1 // indirect | ||
github.com/mitchellh/mapstructure v1.1.2 // indirect | ||
github.com/pelletier/go-toml v1.2.0 // indirect | ||
github.com/pierrec/lz4 v2.6.1+incompatible // indirect | ||
github.com/pmezard/go-difflib v1.0.0 // indirect | ||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect | ||
github.com/spf13/afero v1.1.2 // indirect | ||
github.com/spf13/cast v1.3.0 // indirect | ||
github.com/spf13/jwalterweatherman v1.0.0 // indirect | ||
github.com/spf13/pflag v1.0.3 // indirect | ||
github.com/subosito/gotenv v1.2.0 // indirect | ||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect | ||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect | ||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect | ||
golang.org/x/text v0.3.7 // indirect | ||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect | ||
gopkg.in/ini.v1 v1.51.0 // indirect | ||
gopkg.in/yaml.v2 v2.2.4 // indirect | ||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect | ||
) |
Oops, something went wrong.