Merge branch 'release/v0.4.0' into main

Author: github-actions

deploy-versions.json

@@ -1 +1 @@
-[{"text":"latest","key":"latest","tag":"v0.3.3"}]
+[{"text":"latest","key":"latest","tag":"v0.4.0"}]

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zilla-docs",
-  "version": "0.3.3",
+  "version": "0.4.0",
   "description": "The official documentation for the aklivity/zilla open-source project",
   "keywords": [],
   "author": "aklivity.io",

src/.vuepress/sidebar/en.ts

@@ -204,14 +204,18 @@ export const enSidebar = sidebar({
               text: "Overview",
               link: "concepts/kafka-proxies/mqtt-proxy.md",
             },
+            {
+              text: "Create a Simple MQTT Broker",
+              link: "tutorials/mqtt/mqtt-intro.md",
+            },
+            {
+              text: "Running an MQTT Kafka broker",
+              link: "how-tos/mqtt/mqtt.kafka.broker.md",
+            },
             {
               text: "Run the Taxi Demo",
               link: "https://github.com/aklivity/zilla-demos/tree/main/taxi",
             },
-            {
-              text: "Create a Simple MQTT Broker",
-              link: "tutorials/mqtt/mqtt-intro.md",
-            }
           ],
         },
         {

src/concepts/config-intro.md

@@ -98,11 +98,58 @@ See each of the specific `guard` types linked below for more detailed examples.
 
 ## Vaults
 
-Each configured `vault` represents a container for digital keys and certificates based on a specific implementation `type`.
+Each configured [vault](../reference/config/vaults/) represents a container for digital keys and certificates based on a specific implementation `type`.
 
-Vaults can be used by specific protocol bindings, such as `tls`, to negotiate shared encryption keys.
+### Server Encryption, TLS & SSL
 
-See each of the specific `vault` types linked below for more detailed examples.
+Vaults can be used by specific protocol bindings, such as [tls](../reference/config/bindings/binding-tls.md), to negotiate shared encryption keys. It is easy to add the necessary routing logic and encryption keys.
+
+Using a [filesystem](../reference/config/vaults/vault-filesystem.md) vault, you can see how a pkcs12 certificate on the host is configured to be stored securely by the Zilla runtime. This keystore can then be used by the [tls](../reference/config/bindings/binding-tls.md) binding to decrypt incoming traffic.
+
+```yaml{6}
+vaults:
+  your_servers:
+    type: filesystem
+    options:
+      keys:
+        store: your_servers.p12
+        type: pkcs12
+        password: ${{env.KEYSTORE_PASSWORD}}
+```
+
+The [tcp](../reference/config/bindings/binding-tcp.md) binding can be configured for both encrypted and unencrypted traffic on separate ports. Take the SSL example with ports `80` and `443`. The [tls](../reference/config/bindings/binding-tls.md) binding will use the `keys` as the certificate aliases and the Server Name Indication (`sni`) as the SSL server names. These will likely be the same. Since this example is over [http](../reference/config/bindings/binding-http.md) the Application-Layer Protocol Negotiation (ALPN) will need to handle both HTTP/1.1 and HTTP/2, but the [tls](../reference/config/bindings/binding-tls.md) binding can be configured for any of the [alpn](../reference/config/bindings/binding-tls.md#options-alpn) protocols supported by Zilla.
+
+```yaml{20,29}
+bindings:
+  tcp_server:
+    type: tcp
+    kind: server
+    options:
+      host: 0.0.0.0
+      port: 
+        - 80
+        - 443
+    routes:
+        - when: 
+            - port: 80
+          exit: http_server
+        - when:
+            - port: 443
+          exit: tls_server
+  tls_server:
+    type: tls
+    kind: server
+    vault: your_servers
+    options:
+      keys:
+        - your_server.com
+      sni:
+        - your_server.com
+      alpn:
+        - http/1.1
+        - h2
+    exit: http_server
+```
 
 ## Telemetry
 

src/concepts/kafka-proxies/mqtt-proxy.md

@@ -111,4 +111,4 @@ bindings:
 
 ## Try it out
 
-Go check out the [MQTT Kafka Reflect example](https://github.com/aklivity/zilla-examples/tree/main/mqtt.kafka.broker) or the [JWT Auth example](https://github.com/aklivity/zilla-examples/tree/main/mqtt.kafka.broker.jwt) example for a full implementation of an MQTT proxy.
+Go check out the [Running an MQTT Kafka broker](../../how-tos/mqtt/mqtt.kafka.broker.md) or the [JWT Auth example](https://github.com/aklivity/zilla-examples/tree/main/mqtt.kafka.broker.jwt) example for a full implementation of an MQTT proxy.

src/how-tos/install.md

@@ -12,9 +12,7 @@ The output should display the zilla config and `started` to know zilla is ready
 
 ```output:no-line-numbers
 // default Zilla config
-{
-  "name": "default"
-}
+name: default
 
 // Zilla status
 started
@@ -45,7 +43,7 @@ Go to the [Zilla artifacthub](https://artifacthub.io/) page to find out more on
 ### TL;DR
 
 ```bash:no-line-numbers
-helm install zilla . --namespace zilla --create-namespace --wait \
+helm install zilla oci://ghcr.io/aklivity/charts/zilla --namespace zilla --create-namespace --wait \
     --values values.yaml \
     --set-file zilla\\.yaml=zilla.yaml
 ```

src/how-tos/mqtt/mqtt.kafka.broker.md

@@ -0,0 +1,207 @@
+---
+description: In this guide, you create Kafka topics and use Zilla to mediate MQTT broker messages onto those topics.
+---
+
+# Running an MQTT Kafka broker
+
+In this guide, you create Kafka topics and use Zilla to mediate MQTT broker messages onto those topics.
+
+Specifically, you will:
+
+[Verify prerequisites](#prerequisites) to run this guide.
+[Install and run](#tl-dr) Zilla with Kafka or use your own.
+[Create topics](#check-the-kafka-topics) for the MQTT broker messages.
+[Watch Kafka](#listen-for-messages) for new messages on the topics.
+[Pub & Sub](#send-a-greeting) with an MQTT client.
+
+## Tl;Dr
+
+Download and run the Zilla [zilla-examples/mqtt.kafka.broker](https://github.com/aklivity/zilla-examples/tree/main/mqtt.kafka.broker) example using this install script. It will start Zilla and everything you need for this guide.
+
+```bash:no-line-numbers
+wget -qO- https://raw.githubusercontent.com/aklivity/zilla-examples/main/startup.sh | sh -s -- mqtt.kafka.broker
+```
+
+::: note
+Alternatively, download [mqtt.kafka.broker](https://github.com/aklivity/zilla-examples/releases/latest/download/mqtt.kafka.broker.tar.gz) or the [startup.sh](https://github.com/aklivity/zilla-examples/releases/latest/download/startup.sh) script yourself.
+:::
+
+### Prerequisites
+
+Before proceeding, you should have [Compose](https://docs.docker.com/compose/gettingstarted/) or optionally [Helm](https://helm.sh/docs/intro/install/) and [Kubernetes](https://kubernetes.io/docs/tasks/tools/) installed.
+
+::: details Detailed prerequisites
+
+- A connection to the internet
+- Docker version 1.13.0+ or later is installed and running
+- Docker Desktop or Docker Desktop for Windows on WSL 2
+- Container host resources: 1 CPU, 1GB memory
+
+Optional:
+
+- Kafka 3.0+ hosted with the Docker network allowed to communicate
+- Helm 3.0+
+- Kubernetes 1.13.0+
+
+:::
+
+### Check the Kafka topics
+
+Run the docker command under the `Verify the Kafka topics created` section of the script output. Verify these topics are listed. Read more on the data in these topics in [the overview](../../concepts/kafka-proxies/mqtt-proxy.html#step-2-pub-sub-message-reflect-with-kafka).
+
+```output:no-line-numbers
+mqtt-messages
+mqtt-retained
+mqtt-sessions
+```
+
+### Listen for messages
+
+Run the docker command under the `Start a topic consumer to listen for messages` section of the script output. If you didn't use your own Kafka, you can also see all the topics in the [Kafka UI](http://localhost:8080/ui/clusters/local/all-topics).
+
+### Send a greeting
+
+Using [eclipse-mosquitto](https://hub.docker.com/_/eclipse-mosquitto) subscribe to the `zilla` topic.
+
+```bash:no-line-numbers
+docker run -it --rm eclipse-mosquitto \
+mosquitto_sub -V 'mqttv5' --topic 'zilla' \
+--host 'host.docker.internal' --port 7183 --debug
+```
+
+In a separate session, publish a message on the `zilla` topic.
+
+```bash:no-line-numbers
+docker run -it --rm eclipse-mosquitto \
+mosquitto_pub -V 'mqttv5' --topic 'zilla' --message 'Hello, world' \
+--host 'host.docker.internal' --port 7183 --debug --insecure
+```
+
+Send messages with the retained flag.
+
+```bash:no-line-numbers
+docker run -it --rm eclipse-mosquitto \
+mosquitto_pub -V 'mqttv5' --topic 'zilla' --message 'Hello, retained' --retain \
+--host 'host.docker.internal' --port 7183 --debug --insecure
+```
+
+Then restart the `mosquitto_sub` above. The latest retained message is delivered, and the other messages are not.
+
+## Creating this example yourself
+
+### Start a Kafka instance
+
+You can use your own Kafka or set up a local Kafka with [resource.kafka.compose](https://github.com/aklivity/zilla-examples/releases/latest/download/resource.kafka.compose.tar.gz) and follow the setup instructions in the `README.md`.
+
+Export these env variables or overwrite them with your remote Kafka if you skipped the local setup.
+
+```output:no-line-numbers
+export KAFKA_HOST=host.docker.internal
+export KAFKA_PORT=29092
+```
+
+### Bootstrap Kafka
+
+Create these topics in the Kafka environment.
+
+```bash:no-line-numbers
+\
+/bin/kafka-topics.sh --bootstrap-server $KAFKA_HOST:$KAFKA_PORT --create --if-not-exists --topic mqtt-sessions
+/bin/kafka-topics.sh --bootstrap-server $KAFKA_HOST:$KAFKA_PORT --create --if-not-exists --topic mqtt-messages --config cleanup.policy=compact
+/bin/kafka-topics.sh --bootstrap-server $KAFKA_HOST:$KAFKA_PORT --create --if-not-exists --topic mqtt-retained --config cleanup.policy=compact
+```
+
+### Create your config
+
+Create a new file called `zilla.yaml` and append the below yaml to it.
+
+### Entrypoint
+
+This will configure Zilla for accepting all of the `mqtt` traffic.
+
+```yaml{14-16}
+<!-- @include: ./mqtt_kafka_broker_zilla.yaml{-20} -->
+```
+
+::: right
+[More on binding-tcp](../../reference/config/bindings/binding-tcp.md)
+[More on binding-mqtt](../../reference/config/bindings/binding-mqtt.md)
+:::
+
+### Service definition
+
+The service definitions will define how the clients using this service will interact with Kafka through Zilla.
+
+```yaml{7-9}
+<!-- @include: ./mqtt_kafka_broker_zilla.yaml{22-31} -->
+```
+
+::: right
+[More on binding-mqtt-kafka](../../reference/config/bindings/binding-mqtt-kafka.md)
+[More on topic data](../../concepts/kafka-proxies/mqtt-proxy.html#step-2-pub-sub-message-reflect-with-kafka)
+:::
+
+### Add a Kafka sync layer
+
+The Zilla [cache_client and cache_server](../../reference/config/bindings/binding-kafka.html#kind) helps manage the smooth data transfer between the service definition and Kafka.
+
+```yaml{11-12}
+<!-- @include: ./mqtt_kafka_broker_zilla.yaml{33-46} -->
+```
+
+::: right
+[More on binding-kafka cache](../../reference/config/bindings/binding-kafka.md#cache-behavior)
+:::
+
+### Point to a Running Kafka instance
+
+This will define the location and connection for Zilla to communicate with Kafka.
+
+```yaml{10-11}
+<!-- @include: ./mqtt_kafka_broker_zilla.yaml{47-} -->
+```
+
+::: details Full zilla.yaml
+
+```yaml
+<!-- @include: ./mqtt_kafka_broker_zilla.yaml -->
+```
+
+:::
+
+::: right
+[More on binding-kafka client](../../reference/config/bindings/binding-kafka.md#client-behavior)
+:::
+
+### Start Zilla
+
+With your `zilla.yaml` config, follow the [Zilla install instructions](../install.md) using your method of choice. Set the necessary Kafka environment variables.
+
+::: code-tabs#bash
+
+@tab Docker
+
+```bash:no-line-numbers
+--env KAFKA_HOST="$KAFKA_HOST" --env KAFKA_PORT="$KAFKA_PORT"
+```
+
+@tab Helm values.yaml
+
+```yaml:no-line-numbers
+# use the values from $KAFKA_HOST $KAFKA_PORT variables
+env:
+  - name: KAFKA_HOST
+    value: "\"host.docker.internal\""
+  - name: KAFKA_PORT
+    value: "\"29092\""
+```
+
+:::
+
+### Adding TLS
+
+You can add TLS to this broker by adding a vault and tls binding as described in the [Server Encryption](../../concepts/config-intro.md#server-encryption-tls-ssl) section.
+
+## Remove the running containers
+
+Find the path to the `teardown.sh` script(s) in the `use the teardown script(s) to clean up` section of the example output and run it. If you didn't provide an external Kafka endpoint, there will be scripts for both Zilla and the local Kafka installs.

src/how-tos/mqtt/mqtt_kafka_broker_zilla.yaml

@@ -0,0 +1,60 @@
+---
+name: zilla-mqtt-kafka-broker
+bindings:
+
+# Proxy service entrypoint
+  north_tcp_server:
+    type: tcp
+    kind: server
+    options:
+      host: 0.0.0.0
+      port: 
+        - 7183
+    routes:
+        - when: 
+            - port: 7183
+          exit: north_mqtt_server
+  north_mqtt_server:
+    type: mqtt
+    kind: server
+    exit: north_mqtt_kafka_mapping
+
+# MQTT messages to Kafka topics
+  north_mqtt_kafka_mapping:
+    type: mqtt-kafka
+    kind: proxy
+    options:
+      topics:
+        sessions: mqtt-sessions
+        messages: mqtt-messages
+        retained: mqtt-retained
+    exit: north_kafka_cache_client
+
+# Kafka sync layer
+  north_kafka_cache_client:
+    type: kafka
+    kind: cache_client
+    exit: south_kafka_cache_server
+  south_kafka_cache_server:
+    type: kafka
+    kind: cache_server
+    options:
+      bootstrap:
+        - mqtt-messages
+        - mqtt-retained
+    exit: south_kafka_client
+
+# Connect to Kafka
+  south_kafka_client:
+    type: kafka
+    kind: client
+    exit: south_tcp_client
+  south_tcp_client:
+    type: tcp
+    kind: client
+    options:
+      host: ${{env.KAFKA_HOST}}
+      port: ${{env.KAFKA_PORT}}
+    routes:
+      - when:
+          - cidr: 0.0.0.0/0