Enable setting compression via the KafkaAttribute

Author: gliljas

README.md

@@ -322,6 +322,8 @@ The settings exposed here are targeted to more advanced users that want to custo
 |LibkafkaDebug|debug|Both
 |MetadataMaxAgeMs|metadata.max.age.ms|Both
 |SocketKeepaliveEnable|socket.keepalive.enable|Both
+|CompressionType|compression.codec|Output
+|CompressionLevel|compression.level|Output
 
 **NOTE:** `MetadataMaxAgeMs` default is `180000` `SocketKeepaliveEnable` default is `true` otherwise, the default value is the same as the [Configuration properties](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md). The reason of the default settings, refer to this [issue](https://github.com/Azure/azure-functions-kafka-extension/issues/187).
 

src/Microsoft.Azure.WebJobs.Extensions.Kafka/Config/MessageCompressionType.cs

@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+namespace Microsoft.Azure.WebJobs.Extensions.Kafka
+{
+    /// <summary>
+    /// Defines the message compression type
+    /// </summary>
+    public enum MessageCompressionType
+    {
+        NotSet = -1,
+        None = 0,
+        Gzip = 1,
+        Snappy = 2,
+        Lz4 = 3,
+        Zstd = 4
+    }
+}

src/Microsoft.Azure.WebJobs.Extensions.Kafka/Output/KafkaAttribute.cs

@@ -15,6 +15,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.Kafka
     [Binding]
     public sealed class KafkaAttribute : Attribute
     {
+        private int? compressionLevel;
         private int? maxMessageBytes;
         private int? batchSize;
         private bool? enableIdempotence;
@@ -146,5 +147,17 @@ public KafkaAttribute()
         /// ssl.key.password in librdkafka
         /// </summary>
         public string SslKeyPassword { get; set; }
+
+        /// <summary>
+        /// Compression level parameter for algorithm selected by configuration property <see cref="CompressionType"/>
+        /// compression.level in librdkafka
+        /// </summary>
+        public int CompressionLevel { get => compressionLevel.GetValueOrDefault(-1); set => compressionLevel = value; }
+
+        /// <summary>
+        /// Compression codec to use for compressing message sets. 
+        /// compression.codec in librdkafka
+        /// </summary>
+        public MessageCompressionType CompressionType { get; set; } = MessageCompressionType.NotSet;
     }
 }

src/Microsoft.Azure.WebJobs.Extensions.Kafka/Output/KafkaProducerFactory.cs

@@ -130,9 +130,15 @@ public ProducerConfig GetProducerConfig(KafkaProducerEntity entity)
                 SslCaLocation = resolvedSslCaLocation,
                 Debug = kafkaOptions?.LibkafkaDebug,
                 MetadataMaxAgeMs = kafkaOptions?.MetadataMaxAgeMs,
-                SocketKeepaliveEnable = kafkaOptions?.SocketKeepaliveEnable
+                SocketKeepaliveEnable = kafkaOptions?.SocketKeepaliveEnable,
+                CompressionLevel = entity.Attribute.CompressionLevel 
             };
 
+            if (entity.Attribute.CompressionType != MessageCompressionType.NotSet)
+            {
+                conf.CompressionType = (CompressionType)entity.Attribute.CompressionType;
+            }
+
             if (entity.Attribute.AuthenticationMode != BrokerAuthenticationMode.NotSet)
             {
                 conf.SaslMechanism = (SaslMechanism)entity.Attribute.AuthenticationMode;

test/Microsoft.Azure.WebJobs.Extensions.Kafka.UnitTests/KafkaProducerFactoryTest.cs

@@ -304,5 +304,30 @@ public void GetProducerConfig_When_Ssl_Locations_Resolve_InAzure_Should_Contain_
             Assert.Equal(sslCa.FullName, config.SslCaLocation);
             Assert.Equal(sslKeyLocation.FullName, config.SslKeyLocation);
         }
+
+        [Theory]
+        [InlineData(MessageCompressionType.NotSet, null)]
+        [InlineData(MessageCompressionType.None, CompressionType.None)]
+        [InlineData(MessageCompressionType.Gzip, CompressionType.Gzip)]
+        [InlineData(MessageCompressionType.Snappy, CompressionType.Snappy)]
+        [InlineData(MessageCompressionType.Lz4, CompressionType.Lz4)]
+        [InlineData(MessageCompressionType.Zstd, CompressionType.Zstd)]
+        public void GetProducerConfig_When_CompressionType_Defined_Should_Set_CompressionType(MessageCompressionType sourceType, CompressionType? targetType)
+        {
+            var attribute = new KafkaAttribute("brokers:9092", "myTopic")
+            {
+                CompressionType = sourceType
+            };
+
+            var entity = new KafkaProducerEntity()
+            {
+                Attribute = attribute,
+                ValueType = typeof(ProtoUser),
+            };
+
+            var factory = new KafkaProducerFactory(emptyConfiguration, new DefaultNameResolver(emptyConfiguration), NullLoggerProvider.Instance);
+            var config = factory.GetProducerConfig(entity);
+            Assert.Equal(targetType, config.CompressionType);
+        }
     }
 }