Merge branch 'release/0.9.67'

.github/workflows/build.yml

@@ -24,7 +24,7 @@ jobs:
         distribution: zulu
         java-version: ${{ matrix.java }}
     - name: Cache Maven packages
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: |
           ~/.m2/repository

.github/workflows/codeql.yml

@@ -62,7 +62,7 @@ jobs:
 
     # Cache downloaded Maven dependencies
     - name: Cache Maven packages
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: |
           ~/.m2/repository

CHANGELOG.md

@@ -2,18 +2,38 @@
 
 ## [Unreleased](https://github.com/aklivity/zilla/tree/HEAD)
 
-[Full Changelog](https://github.com/aklivity/zilla/compare/0.9.65...HEAD)
+[Full Changelog](https://github.com/aklivity/zilla/compare/0.9.66...HEAD)
+
+**Implemented enhancements:**
+
+- Use `model` and `view` when describing the message type [\#750](https://github.com/aklivity/zilla/issues/750)
+- Support obtaining `protobuf` schemas from `schema registry` for `grpc` services [\#697](https://github.com/aklivity/zilla/issues/697)
+- Support idempotent `mqtt` `qos 2` publish to `kafka` [\#677](https://github.com/aklivity/zilla/issues/677)
+- Detect and inspect invalid messages received [\#676](https://github.com/aklivity/zilla/issues/676)
+- Support incremental validation of fragmented messages sent by client [\#671](https://github.com/aklivity/zilla/issues/671)
 
 **Fixed bugs:**
 
-- Schema validation fails before the `${{env.*}}` parameters have been removed [\#583](https://github.com/aklivity/zilla/issues/583)
+- TLSv1.3 client handshake stall [\#791](https://github.com/aklivity/zilla/issues/791)
+- Zilla crashes when it tries to send flush on retain stream [\#770](https://github.com/aklivity/zilla/issues/770)
+- Running emqtt\_bench triggers exception in connection pool [\#716](https://github.com/aklivity/zilla/issues/716)
+- `mqtt-kafka` does not limit client sharding to `mqtt v5` [\#708](https://github.com/aklivity/zilla/issues/708)
+- `tls binding` should handle `null` key returned from `vault` [\#395](https://github.com/aklivity/zilla/issues/395)
 
-**Closed issues:**
+## [0.9.66](https://github.com/aklivity/zilla/tree/0.9.66) (2024-01-24)
+
+[Full Changelog](https://github.com/aklivity/zilla/compare/0.9.65...0.9.66)
+
+**Implemented enhancements:**
 
 - Support `openapi` `http` response validation [\#684](https://github.com/aklivity/zilla/issues/684)
 - Support `protobuf` conversion to and from `json` for `kafka` messages [\#682](https://github.com/aklivity/zilla/issues/682)
 - Support incubator features preview in zilla release docker image [\#670](https://github.com/aklivity/zilla/issues/670)
 
+**Fixed bugs:**
+
+- Schema validation fails before the `${{env.*}}` parameters have been removed [\#583](https://github.com/aklivity/zilla/issues/583)
+
 **Merged pull requests:**
 
 - update license exclude path to include both zpmw files [\#759](https://github.com/aklivity/zilla/pull/759) ([vordimous](https://github.com/vordimous))
@@ -29,6 +49,10 @@
 
 **Implemented enhancements:**
 
+- Support `avro` conversion to and from `json` for `kafka` messages [\#681](https://github.com/aklivity/zilla/issues/681)
+- Support observability of zilla engine internal streams [\#678](https://github.com/aklivity/zilla/issues/678)
+- Simplify configuration of multiple protocols on different tcp ports [\#669](https://github.com/aklivity/zilla/issues/669)
+- Simplify kafka client bootstrap server names and ports config [\#619](https://github.com/aklivity/zilla/issues/619)
 - MQTT publish QoS 2 as Kafka produce with acks in\_sync\_replicas and idempotent `producerId` [\#605](https://github.com/aklivity/zilla/issues/605)
 - Add the option to route by `port` in the `tls` binding [\#564](https://github.com/aklivity/zilla/issues/564)
 - Support outbound message transformation from `protobuf` to `json` [\#458](https://github.com/aklivity/zilla/issues/458)
@@ -59,10 +83,6 @@
 **Closed issues:**
 
 - Prototype composite binding support with nested namespaces [\#685](https://github.com/aklivity/zilla/issues/685)
-- Support `avro` conversion to and from `json` for `kafka` messages [\#681](https://github.com/aklivity/zilla/issues/681)
-- Support observability of zilla engine internal streams [\#678](https://github.com/aklivity/zilla/issues/678)
-- Simplify configuration of multiple protocols on different tcp ports [\#669](https://github.com/aklivity/zilla/issues/669)
-- Simplify kafka client bootstrap server names and ports config [\#619](https://github.com/aklivity/zilla/issues/619)
 - Build has been failed in local [\#229](https://github.com/aklivity/zilla/issues/229)
 
 **Merged pull requests:**

build/flyweight-maven-plugin/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>build</artifactId>
-    <version>0.9.66</version>
+    <version>0.9.67</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

build/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>zilla</artifactId>
-    <version>0.9.66</version>
+    <version>0.9.67</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

cloud/docker-image/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>cloud</artifactId>
-    <version>0.9.66</version>
+    <version>0.9.67</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -249,19 +249,25 @@
     </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
-      <artifactId>validator-avro</artifactId>
+      <artifactId>model-avro</artifactId>
       <version>${project.version}</version>
       <scope>runtime</scope>
     </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
-      <artifactId>validator-core</artifactId>
+      <artifactId>model-core</artifactId>
       <version>${project.version}</version>
       <scope>runtime</scope>
     </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
-      <artifactId>validator-json</artifactId>
+      <artifactId>model-json</artifactId>
+      <version>${project.version}</version>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>model-protobuf</artifactId>
       <version>${project.version}</version>
       <scope>runtime</scope>
     </dependency>

cloud/docker-image/src/main/docker/Dockerfile

@@ -15,14 +15,12 @@
 
 FROM eclipse-temurin:21-jdk AS build
 
-RUN apt update && apt install -y gettext
-
 COPY maven /root/.m2/repository
 
 COPY zpmw zpmw
 COPY zpm.json.template zpm.json.template
 
-RUN cat zpm.json.template | env VERSION=${project.version} envsubst > zpm.json
+RUN cat zpm.json.template | sed "s/\${VERSION}/${project.version}/g" | tee zpm.json
 
 RUN ./zpmw install --debug --exclude-remote-repositories
 RUN ./zpmw clean --keep-image

cloud/docker-image/src/main/docker/assembly.xml

@@ -32,8 +32,8 @@
         <include>io/aklivity/zilla/exporter-*/**</include>
         <include>io/aklivity/zilla/guard-*/**</include>
         <include>io/aklivity/zilla/metrics-*/**</include>
+        <include>io/aklivity/zilla/model-*/**</include>
         <include>io/aklivity/zilla/resolver-*/**</include>
-        <include>io/aklivity/zilla/validator-*/**</include>
         <include>io/aklivity/zilla/vault-*/**</include>
         <include>io/aklivity/zilla/command/**</include>
         <include>io/aklivity/zilla/command-*/**</include>
@@ -63,6 +63,8 @@
         <include>com/fasterxml/jackson/**</include>
         <include>org/yaml/snakeyaml/**</include>
         <include>org/junit/**</include>
+        <include>com/google/**</include>
+        <include>org/checkerframework/**</include>
       </includes>
     </fileSet>
   </fileSets>

cloud/docker-image/src/main/docker/zpm.json.template

@@ -49,10 +49,11 @@
     "io.aklivity.zilla:metrics-stream",
     "io.aklivity.zilla:metrics-http",
     "io.aklivity.zilla:metrics-grpc",
+    "io.aklivity.zilla:model-avro",
+    "io.aklivity.zilla:model-core",
+    "io.aklivity.zilla:model-json",
+    "io.aklivity.zilla:model-protobuf",
     "io.aklivity.zilla:resolver-env",
-    "io.aklivity.zilla:validator-avro",
-    "io.aklivity.zilla:validator-core",
-    "io.aklivity.zilla:validator-json",
     "io.aklivity.zilla:vault-filesystem",
     "org.slf4j:slf4j-simple",
     "org.antlr:antlr4-runtime"

cloud/helm-chart/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>cloud</artifactId>
-    <version>0.9.66</version>
+    <version>0.9.67</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

cloud/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>zilla</artifactId>
-    <version>0.9.66</version>
+    <version>0.9.67</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

conf/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>zilla</artifactId>
-    <version>0.9.66</version>
+    <version>0.9.67</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

incubator/binding-amqp.spec/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>incubator</artifactId>
-    <version>0.9.66</version>
+    <version>0.9.67</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

incubator/binding-amqp/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>incubator</artifactId>
-    <version>0.9.66</version>
+    <version>0.9.67</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

incubator/catalog-inline.spec/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>io.aklivity.zilla</groupId>
         <artifactId>incubator</artifactId>
-        <version>0.9.66</version>
+        <version>0.9.67</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

incubator/catalog-inline/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>io.aklivity.zilla</groupId>
         <artifactId>incubator</artifactId>
-        <version>0.9.66</version>
+        <version>0.9.67</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

incubator/catalog-inline/src/test/java/io/aklivity/zilla/runtime/catalog/inline/internal/InlineIT.java

@@ -20,11 +20,15 @@
 import static org.hamcrest.Matchers.nullValue;
 import static org.junit.Assert.assertEquals;
 
+import org.agrona.DirectBuffer;
+import org.agrona.concurrent.UnsafeBuffer;
 import org.junit.Before;
 import org.junit.Test;
 
 import io.aklivity.zilla.runtime.catalog.inline.config.InlineOptionsConfig;
 import io.aklivity.zilla.runtime.catalog.inline.config.InlineSchemaConfig;
+import io.aklivity.zilla.runtime.engine.catalog.CatalogHandler;
+import io.aklivity.zilla.runtime.engine.model.function.ValueConsumer;
 
 public class InlineIT
 {
@@ -55,4 +59,39 @@ public void shouldResolveSchemaViaSchemaId()
         assertThat(schema, not(nullValue()));
         assertEquals(expected, schema);
     }
+
+    @Test
+    public void shouldResolveSchemaIdAndProcessData()
+    {
+        InlineCatalogHandler catalog = new InlineCatalogHandler(config);
+
+        DirectBuffer data = new UnsafeBuffer();
+
+        String payload =
+                "{" +
+                    "\"id\": \"123\"," +
+                    "\"status\": \"OK\"" +
+                "}";
+        byte[] bytes = payload.getBytes();
+        data.wrap(bytes, 0, bytes.length);
+
+        int valLength = catalog.decode(data, 0, data.capacity(), ValueConsumer.NOP, CatalogHandler.Decoder.IDENTITY);
+
+        assertEquals(data.capacity(), valLength);
+    }
+
+    @Test
+    public void shouldVerifyEncodedData()
+    {
+        InlineCatalogHandler catalog = new InlineCatalogHandler(config);
+
+        DirectBuffer data = new UnsafeBuffer();
+
+        byte[] bytes = {0x06, 0x69, 0x64,
+            0x30, 0x10, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65};
+        data.wrap(bytes, 0, bytes.length);
+
+        assertEquals(13, catalog.encode(1, data, 0, data.capacity(),
+                ValueConsumer.NOP, CatalogHandler.Encoder.IDENTITY));
+    }
 }

incubator/catalog-schema-registry.spec/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>incubator</artifactId>
-    <version>0.9.66</version>
+    <version>0.9.67</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

incubator/catalog-schema-registry.spec/src/main/scripts/io/aklivity/zilla/specs/catalog/schema/registry/config/catalog.yaml

@@ -21,3 +21,4 @@ catalogs:
     options:
       url: http://localhost:8081
       context: default
+      max-age: 30

incubator/catalog-schema-registry.spec/src/main/scripts/io/aklivity/zilla/specs/catalog/schema/registry/schema/schema.registry.schema.patch.json

@@ -39,6 +39,12 @@
                             {
                                 "type": "string",
                                 "default": "default"
+                            },
+                            "max-age":
+                            {
+                                "title": "Max Age",
+                                "type": "number",
+                                "default": 300
                             }
                         },
                         "additionalProperties": false

incubator/catalog-schema-registry/pom.xml

@@ -6,7 +6,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>incubator</artifactId>
-    <version>0.9.66</version>
+    <version>0.9.67</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -75,6 +75,22 @@
         <groupId>com.mycila</groupId>
         <artifactId>license-maven-plugin</artifactId>
       </plugin>
+      <plugin>
+        <groupId>${project.groupId}</groupId>
+        <artifactId>flyweight-maven-plugin</artifactId>
+        <version>${project.version}</version>
+        <configuration>
+          <scopeNames>internal</scopeNames>
+          <packageName>io.aklivity.zilla.runtime.catalog.schema.registry.internal.types</packageName>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>generate</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <artifactId>maven-checkstyle-plugin</artifactId>
       </plugin>
@@ -125,6 +141,9 @@
         <groupId>org.jacoco</groupId>
         <artifactId>jacoco-maven-plugin</artifactId>
         <configuration>
+          <excludes>
+            <exclude>io/aklivity/zilla/runtime/catalog/schema/registry/internal/types/**/*.class</exclude>
+          </excludes>
           <rules>
             <rule>
               <element>BUNDLE</element>