From 58ad61960e3e4e7db1e17908a48b6e8a2fc0f600 Mon Sep 17 00:00:00 2001 From: Conor Mongey Date: Sun, 3 Jun 2018 16:02:43 +0100 Subject: [PATCH] Readme update --- ssl-ffs/Dockerfile => Dockerfile | 0 GNUmakefile | 8 +- README.md | 135 +++++++++++++++--- ...docker-compose.yaml => docker-compose.yaml | 0 examples/main.tf | 14 ++ kafka/resource_kafka_acl_test.go | 12 +- kafka/resource_kafka_topic.go | 8 +- .../broker1-ca1-signed.crt | 0 {ssl-ffs/secrets => secrets}/broker1.csr | 0 .../broker1_keystore_creds | 0 .../secrets => secrets}/broker1_sslkey_creds | 0 .../broker1_truststore_creds | 0 {ssl-ffs/secrets => secrets}/create-certs.sh | 0 .../host.consumer.ssl.config | 0 .../host.producer.ssl.config | 0 .../kafka.broker1.keystore.jks | Bin .../kafka.broker1.truststore.jks | Bin .../kafkacat-ca1-signed.pem | 0 .../kafkacat-raw-private-key.pem | 0 .../secrets => secrets}/kafkacat.client.key | 0 .../secrets => secrets}/kafkacat.client.req | 0 .../secrets => secrets}/snakeoil-ca-1.crt | 0 .../secrets => secrets}/snakeoil-ca-1.key | 0 .../secrets => secrets}/snakeoil-ca-1.srl | 0 24 files changed, 143 insertions(+), 34 deletions(-) rename ssl-ffs/Dockerfile => Dockerfile (100%) rename ssl-ffs/docker-compose.yaml => docker-compose.yaml (100%) rename {ssl-ffs/secrets => secrets}/broker1-ca1-signed.crt (100%) rename {ssl-ffs/secrets => secrets}/broker1.csr (100%) rename {ssl-ffs/secrets => secrets}/broker1_keystore_creds (100%) rename {ssl-ffs/secrets => secrets}/broker1_sslkey_creds (100%) rename {ssl-ffs/secrets => secrets}/broker1_truststore_creds (100%) rename {ssl-ffs/secrets => secrets}/create-certs.sh (100%) rename {ssl-ffs/secrets => secrets}/host.consumer.ssl.config (100%) rename {ssl-ffs/secrets => secrets}/host.producer.ssl.config (100%) rename {ssl-ffs/secrets => secrets}/kafka.broker1.keystore.jks (100%) rename {ssl-ffs/secrets => secrets}/kafka.broker1.truststore.jks (100%) rename {ssl-ffs/secrets => secrets}/kafkacat-ca1-signed.pem (100%) rename {ssl-ffs/secrets => secrets}/kafkacat-raw-private-key.pem (100%) rename {ssl-ffs/secrets => secrets}/kafkacat.client.key (100%) rename {ssl-ffs/secrets => secrets}/kafkacat.client.req (100%) rename {ssl-ffs/secrets => secrets}/snakeoil-ca-1.crt (100%) rename {ssl-ffs/secrets => secrets}/snakeoil-ca-1.key (100%) rename {ssl-ffs/secrets => secrets}/snakeoil-ca-1.srl (100%) diff --git a/ssl-ffs/Dockerfile b/Dockerfile similarity index 100% rename from ssl-ffs/Dockerfile rename to Dockerfile diff --git a/GNUmakefile b/GNUmakefile index 811c08f4..42e9b5e5 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -3,7 +3,7 @@ GOFMT_FILES?=$$(find . -name '*.go' |grep -v vendor) default: build build: - go install + go build . test: go test -i $(TEST) || exit 1 @@ -12,9 +12,9 @@ test: testacc: KAFKA_BOOTSTRAP_SERVER=localhost:9092 \ - KAFKA_CACERT=../ssl-ffs/secrets/snakeoil-ca-1.crt \ - KAFKA_CLIENT_CERT=../ssl-ffs/secrets/kafkacat-ca1-signed.pem \ - KAFKA_CLIENT_KEY=../ssl-ffs/secrets/kafkacat-raw-private-key.pem \ + KAFKA_CACERT=../secrets/snakeoil-ca-1.crt \ + KAFKA_CLIENT_CERT=../secrets/kafkacat-ca1-signed.pem \ + KAFKA_CLIENT_KEY=../secrets/kafkacat-raw-private-key.pem \ KAFKA_SKIP_VERIFY=true \ KAFKA_ENABLE_TLS=true \ TF_ACC=1 go test $(TEST) -v $(TESTARGS) -timeout 120m diff --git a/README.md b/README.md index 220181f2..6ae559cb 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,52 @@ +# `terraform-provider-kafka` +[![CircleCI](https://circleci.com/gh/Mongey/terraform-provider-kafka.svg?style=svg)](https://circleci.com/gh/Mongey/terraform-provider-kafka) + A [Terraform][1] plugin for managing [Apache Kafka][2]. -[![CircleCI](https://circleci.com/gh/Mongey/terraform-provider-kafka.svg?style=svg)](https://circleci.com/gh/Mongey/terraform-provider-kafka) +## Contents -# Requirements -* [Kafka 1.0.0][3] +* [Installation](#installation) + * [Developing](#developing) +* [`kafka` Provider](#provider-configuration) +* [Resources](#resources) + * [`kafka_topic`](#kafka_topic) + * [`kafka_acl`](#kafka_acl) +* [Requirements](#requirements) + +## Installation + +Download and extract the [latest +release](https://github.com/Mongey/terraform-provider-kafka/releases/latest) to +your [terraform plugin directory][third-party-plugins] (typically `~/.terraform.d/plugins/`) + +### Developing -# Example +0. [Install go][install-go] +0. Clone repository to: `$GOPATH/src/github.com/Mongey/terraform-provider-kafka` + ``` bash + mkdir -p $GOPATH/src/github.com/Mongey/terraform-provider-kafka; cd $GOPATH/src/github.com/Mongey/ + git clone https://github.com/Mongey/terraform-provider-kafka.git + ``` +0. Build the provider `make build` +0. Run the tests `make test` +0. Start a TLS enabled kafka-cluster `docker-compose up` +0. Run the acceptance tests `make testacc` +## Provider Configuration + +### Example + +Example provider with SSL client authentication. ```hcl provider "kafka" { bootstrap_servers = ["localhost:9092"] -} - -resource "kafka_topic" "logs" { - name = "systemd_logs" - replication_factor = 2 - partitions = 100 - - config = { - "segment.ms" = "20000" - } + ca_cert_file = "../secrets/snakeoil-ca-1.crt" + client_cert_file = "../secrets/kafkacat-ca1-signed.pem" + client_key_file = "../secrets/kafkacat-raw-private-key.pem" + skip_tls_verify = true } ``` -# Provider Configuration | Property | Description | Default | | ---------------- | ----------------------- | ---------- | | `bootstrap_servers` | A list of host:port addresses that will be used to discover the full set of alive brokers | `Required` | @@ -35,19 +58,91 @@ resource "kafka_topic" "logs" { | `sasl_username` | Username for SASL authentication. | `""` | | `sasl_password` | Password for SASL authentication. | `""` | -# Importing Existing Topics +## Resources +### `kafka_topic` + +A resource for managing Kafka topics. Increases partition count without +destroying the topic. + +#### Example + +```hcl +provider "kafka" { + bootstrap_servers = ["localhost:9092"] +} + +resource "kafka_topic" "logs" { + name = "systemd_logs" + replication_factor = 2 + partitions = 100 + + config = { + "segment.ms" = "20000" + "cleanup.policy" = "compact" + } +} +``` + +#### Properties + +| Property | Description | +| ---------------- | ----------------------- | +| `name` | The name of the topic | +| `paritions` | The number of partitions the topic should have | +| `replication_factor` | The number of replicas the topic should have | +| `config` | A map of string k/v attributes | + + +#### Importing Existing Topics You can import topics with the following ```sh terraform import kafka_topic.logs systemd_logs ``` -# Resources -* kafka_topic -# Planned Resources -* kafka_acl +### `kafka_acl` +A resource for managing Kafka ACLs. + +#### Example + +```hcl +provider "kafka" { + bootstrap_servers = ["localhost:9092"] + ca_cert_file = "../secrets/snakeoil-ca-1.crt" + client_cert_file = "../secrets/kafkacat-ca1-signed.pem" + client_key_file = "../secrets/kafkacat-raw-private-key.pem" + skip_tls_verify = true +} + +resource "kafka_acl" "test" { + resource_name = "syslog" + resource_type = "Topic" + acl_principal = "User:Alice" + acl_host = "*" + acl_operation = "Write" + acl_permission_type = "Deny" +} +``` + +#### Properties + +| Property | Description | Valid values | +| ---------------- | ----------------------- | -------------- | +| `acl_host` | A map of string k/v attributes | `*` | +| `acl_operation` | A map of string k/v attributes | `Unknown`, `Any`, `All`, `Read`, `Write`, `Create`, `Delete`, `Alter`, `Describe`, `ClusterAction`, `DescribeConfigs`, `AlterConfigs`, `IdempotentWrite` | +| `acl_permission_type` | A map of string k/v attributes | `Unknown`, `Any`, `Allow`, `Deny` | +| `acl_principal` | The number of replicas the topic should have | `*` | +| `resource_name` | The name of the resource | `*` | +| `resource_type` | The type of resource | `Unknown`, `Any`, `Topic`, `Group`, `Cluster`, `TransactionalID` | + + +## Requirements +* [Kafka 1.0.0][3] [1]: https://www.terraform.io [2]: https://kafka.apache.org [3]: https://cwiki.apache.org/confluence/display/KAFKA/KIP-117%3A+Add+a+public+AdminClient+API+for+Kafka+admin+operations +[third-party-plugins]: https://www.terraform.io/docs/configuration/providers.html#third-party-plugins +[install-go]: https://golang.org/doc/install#install + diff --git a/ssl-ffs/docker-compose.yaml b/docker-compose.yaml similarity index 100% rename from ssl-ffs/docker-compose.yaml rename to docker-compose.yaml diff --git a/examples/main.tf b/examples/main.tf index 7c6ea9ad..a1c43e1f 100644 --- a/examples/main.tf +++ b/examples/main.tf @@ -1,5 +1,10 @@ provider "kafka" { bootstrap_servers = ["localhost:9092"] + ca_cert_file = "../ssl-ffs/secrets/snakeoil-ca-1.crt" + client_cert_file = "../ssl-ffs/secrets/kafkacat-ca1-signed.pem" + client_key_file = "../ssl-ffs/secrets/kafkacat-raw-private-key.pem" + tls_enabled = true + skip_tls_verify = true } resource "kafka_topic" "syslog" { @@ -12,3 +17,12 @@ resource "kafka_topic" "syslog" { "retention.ms" = "86400000" } } + +resource "kafka_acl" "test" { + resource_name = "syslog" + resource_type = "Topic" + acl_principal = "User:Alice" + acl_host = "*" + acl_operation = "Write" + acl_permission_type = "Deny" +} diff --git a/kafka/resource_kafka_acl_test.go b/kafka/resource_kafka_acl_test.go index 0e53a880..ff684dfc 100644 --- a/kafka/resource_kafka_acl_test.go +++ b/kafka/resource_kafka_acl_test.go @@ -92,10 +92,10 @@ func testResourceACL_updateCheck(s *terraform.State) error { const testResourceACL_initialConfig = ` provider "kafka" { bootstrap_servers = ["localhost:9092"] - ca_cert_file = "../ssl-ffs/secrets/snakeoil-ca-1.crt" + ca_cert_file = "../secrets/snakeoil-ca-1.crt" + client_cert_file = "../secrets/kafkacat-ca1-signed.pem" + client_key_file = "../secrets/kafkacat-raw-private-key.pem" skip_tls_verify = true - client_cert_file = "../ssl-ffs/secrets/kafkacat-ca1-signed.pem" - client_key_file = "../ssl-ffs/secrets/kafkacat-raw-private-key.pem" } resource "kafka_acl" "test" { @@ -111,10 +111,10 @@ resource "kafka_acl" "test" { const testResourceACL_updateConfig = ` provider "kafka" { bootstrap_servers = ["localhost:9092"] - ca_cert_file = "../ssl-ffs/secrets/snakeoil-ca-1.crt" + ca_cert_file = "../secrets/snakeoil-ca-1.crt" + client_cert_file = "../secrets/kafkacat-ca1-signed.pem" + client_key_file = "../secrets/kafkacat-raw-private-key.pem" skip_tls_verify = true - client_cert_file = "../ssl-ffs/secrets/kafkacat-ca1-signed.pem" - client_key_file = "../ssl-ffs/secrets/kafkacat-raw-private-key.pem" } resource "kafka_acl" "test" { diff --git a/kafka/resource_kafka_topic.go b/kafka/resource_kafka_topic.go index f1aec1a4..7897bafb 100644 --- a/kafka/resource_kafka_topic.go +++ b/kafka/resource_kafka_topic.go @@ -24,24 +24,24 @@ func kafkaTopicResource() *schema.Resource { Type: schema.TypeString, Required: true, ForceNew: true, - Description: "The name of the topic", + Description: "The name of the topic.", }, "partitions": { Type: schema.TypeInt, Required: true, - Description: "number of partitions", + Description: "Number of partitions.", }, "replication_factor": { Type: schema.TypeInt, Required: true, ForceNew: true, - Description: "number of replicas", + Description: "Number of replicas.", }, "config": { Type: schema.TypeMap, Optional: true, ForceNew: false, - Description: "A map of string k/v attributes", + Description: "A map of string k/v attributes.", }, }, } diff --git a/ssl-ffs/secrets/broker1-ca1-signed.crt b/secrets/broker1-ca1-signed.crt similarity index 100% rename from ssl-ffs/secrets/broker1-ca1-signed.crt rename to secrets/broker1-ca1-signed.crt diff --git a/ssl-ffs/secrets/broker1.csr b/secrets/broker1.csr similarity index 100% rename from ssl-ffs/secrets/broker1.csr rename to secrets/broker1.csr diff --git a/ssl-ffs/secrets/broker1_keystore_creds b/secrets/broker1_keystore_creds similarity index 100% rename from ssl-ffs/secrets/broker1_keystore_creds rename to secrets/broker1_keystore_creds diff --git a/ssl-ffs/secrets/broker1_sslkey_creds b/secrets/broker1_sslkey_creds similarity index 100% rename from ssl-ffs/secrets/broker1_sslkey_creds rename to secrets/broker1_sslkey_creds diff --git a/ssl-ffs/secrets/broker1_truststore_creds b/secrets/broker1_truststore_creds similarity index 100% rename from ssl-ffs/secrets/broker1_truststore_creds rename to secrets/broker1_truststore_creds diff --git a/ssl-ffs/secrets/create-certs.sh b/secrets/create-certs.sh similarity index 100% rename from ssl-ffs/secrets/create-certs.sh rename to secrets/create-certs.sh diff --git a/ssl-ffs/secrets/host.consumer.ssl.config b/secrets/host.consumer.ssl.config similarity index 100% rename from ssl-ffs/secrets/host.consumer.ssl.config rename to secrets/host.consumer.ssl.config diff --git a/ssl-ffs/secrets/host.producer.ssl.config b/secrets/host.producer.ssl.config similarity index 100% rename from ssl-ffs/secrets/host.producer.ssl.config rename to secrets/host.producer.ssl.config diff --git a/ssl-ffs/secrets/kafka.broker1.keystore.jks b/secrets/kafka.broker1.keystore.jks similarity index 100% rename from ssl-ffs/secrets/kafka.broker1.keystore.jks rename to secrets/kafka.broker1.keystore.jks diff --git a/ssl-ffs/secrets/kafka.broker1.truststore.jks b/secrets/kafka.broker1.truststore.jks similarity index 100% rename from ssl-ffs/secrets/kafka.broker1.truststore.jks rename to secrets/kafka.broker1.truststore.jks diff --git a/ssl-ffs/secrets/kafkacat-ca1-signed.pem b/secrets/kafkacat-ca1-signed.pem similarity index 100% rename from ssl-ffs/secrets/kafkacat-ca1-signed.pem rename to secrets/kafkacat-ca1-signed.pem diff --git a/ssl-ffs/secrets/kafkacat-raw-private-key.pem b/secrets/kafkacat-raw-private-key.pem similarity index 100% rename from ssl-ffs/secrets/kafkacat-raw-private-key.pem rename to secrets/kafkacat-raw-private-key.pem diff --git a/ssl-ffs/secrets/kafkacat.client.key b/secrets/kafkacat.client.key similarity index 100% rename from ssl-ffs/secrets/kafkacat.client.key rename to secrets/kafkacat.client.key diff --git a/ssl-ffs/secrets/kafkacat.client.req b/secrets/kafkacat.client.req similarity index 100% rename from ssl-ffs/secrets/kafkacat.client.req rename to secrets/kafkacat.client.req diff --git a/ssl-ffs/secrets/snakeoil-ca-1.crt b/secrets/snakeoil-ca-1.crt similarity index 100% rename from ssl-ffs/secrets/snakeoil-ca-1.crt rename to secrets/snakeoil-ca-1.crt diff --git a/ssl-ffs/secrets/snakeoil-ca-1.key b/secrets/snakeoil-ca-1.key similarity index 100% rename from ssl-ffs/secrets/snakeoil-ca-1.key rename to secrets/snakeoil-ca-1.key diff --git a/ssl-ffs/secrets/snakeoil-ca-1.srl b/secrets/snakeoil-ca-1.srl similarity index 100% rename from ssl-ffs/secrets/snakeoil-ca-1.srl rename to secrets/snakeoil-ca-1.srl