[TH2-3689] Mangler > Add basic support for repeating groups

README.md

@@ -1,4 +1,4 @@
-# th2-conn-dirty-fix (0.0.7)
+# th2-conn-dirty-fix (0.1.0)
 
 This microservice allows sending and receiving messages via FIX protocol
 
@@ -58,14 +58,29 @@ This microservice allows sending and receiving messages via FIX protocol
 Mangler is configured by specifying a list of transformations which it will try to apply to outgoing messages.   
 Each transformation has a list of conditions which message must meet for transformation actions to be applied.
 
-Condition is basically a field value check:
+Condition can be one of the following:
 
-```yaml
-tag: 35
-matches: (8|D)
-```
+1. field selector:
+
+   ```yaml
+    tag: 35
+    matches: (8|D)
+   ```
+
+   Where `tag` is a field tag to match and `matches` is a regex pattern for the field value.
+
+2. group selector:
 
-Where `tag` is a field tag to match and `matches` is a regex pattern for the field value.
+    ```yaml
+      group: group-name
+      contains:
+        - tag: 100
+          matching: A
+        - tag: 101
+          matching: C
+    ```
+
+   Where `group` is a name of a predefined group and `contains` is a list of field selectors
 
 Conditions are specified in `when` block of transformation definition:
 
@@ -77,22 +92,35 @@ when:
     matches: SENDER(.*)
 ```
 
+Groups are defined in `context.groups` map of mangler configuration
+
+```yaml
+context:
+  groups:
+    group-name:
+      counter: 99
+      delimiter: 100
+      tags: [ 101, 102, 103 ]
+```
+
+Where `counter` is a counter tag, `delimiter` is a delimiter tag and `tags` is a set of tags that could follow delimiter
+
 Actions describe modifications which will be applied to a message. There are 4 types of actions:
 
 * set - sets value of an existing field to the specified value:
 
   ```yaml
   set:
     tag: 1
-    value: new account
+    to: new account
   ```
 
 * add - adds new field before or after an existing field:
 
   ```yaml
   add:
     tag: 15
-    value: USD
+    equal: USD
   after: # or before
     tag: 58
     matches: (.*)
@@ -103,7 +131,7 @@ Actions describe modifications which will be applied to a message. There are 4 t
   ```yaml
   move:
     tag: 49
-    matches: (.*)
+    matching: (.*)
   after: # or before
     tag: 56
     matches: (.*)
@@ -114,20 +142,33 @@ Actions describe modifications which will be applied to a message. There are 4 t
   ```yaml
   replace:
     tag: 64
-    matches: (.*)
+    matching: (.*)
   with:
     tag: 63
-    value: 1
+    equal: 1
   ```
 
 * remove - removes an existing field:
 
   ```yaml
   remove:
     tag: 110
-    matches: (.*)
+    matching: (.*)
   ```
 
+Action scope could be limited to a certain group by specifying group selector in `in` field:
+
+```yaml
+set:
+  tag: 100
+  to: ABC
+in:
+  group: group-name
+  where:
+    - tag: 101
+      matches: C
+```
+
 Actions are specified in `then` block of transformation definition:
 
 ```yaml
@@ -158,6 +199,12 @@ Complete mangler configuration would look something like this:
 
 ```yaml
 mangler:
+  context:
+    groups:
+      NoPartyIDs:
+        counter: 453
+        delimiter: 448
+        tags: [ 447, 452 ]
   rules:
     - name: rule-1
       transform:
@@ -166,13 +213,17 @@ mangler:
               matches: FIXT.1.1
             - tag: 35
               matches: D
+            - group: NoPartyIDs
+              contains:
+                - tag: 448
+                  matching: ABC
           then:
             - set:
                 tag: 1
-                value: new account
+                to: new account
             - add:
                 tag: 15
-                value: USD
+                equal: USD
               after:
                 tag: 58
                 matches: (.*)
@@ -185,13 +236,13 @@ mangler:
           then:
             - replace:
                 tag: 64
-                matches: (.*)
+                matching: (.*)
               with:
                 tag: 63
-                value: 1
+                equal: 1
             - remove:
                 tag: 110
-                matches: (.*)
+                matching: (.*)
           update-checksum: false
 ```
 
@@ -253,24 +304,31 @@ spec:
           reconnectDelay": 5
           disconnectRequestDelay: 5
         mangler:
+          context:
+            groups:
+              NoPartyIDs:
+                counter: 453
+                delimiter: 448
+                tags: [ 447, 452 ]
           rules:
             - name: rule-1
               transform:
                 - when:
                     - { tag: 8, matches: FIXT.1.1 }
                     - { tag: 35, matches: D }
+                    - { group: NoPartyIDs, contains: [ { tag: 448, matching: ABC } ] }
                   then:
-                    - set: { tag: 1, value: new account }
-                    - add: { tag: 15, valueOneOf: ["USD", "EUR"] }
+                    - set: { tag: 1, to: new account }
+                    - add: { tag: 15, equal-one-of: [ "USD", "EUR" ] }
                       after: { tag: 58, matches: (.*) }
                   update-length: false
                 - when:
                     - { tag: 8, matches: FIXT.1.1 }
                     - { tag: 35, matches: 8 }
                   then:
-                    - replace: { tag: 64, matches: (.*) }
-                      with: { tag: 63, value: 1 }
-                    - remove: { tag: 110, matches: (.*) }
+                    - replace: { tag: 64, matching: (.*) }
+                      with: { tag: 63, equal: 1 }
+                    - remove: { tag: 110, matching: (.*) }
                   update-checksum: false
   pins:
   - name: to_data_provider
@@ -323,6 +381,10 @@ spec:
 
 # Changelog
 
+## 0.1.0
+
+* add basic support for repeating groups to mangler
+
 ## 0.0.7
 
 * wait for acceptor logout response on close

gradle.properties

@@ -1,2 +1,2 @@
-release_version=0.0.7
+release_version=0.1.0
 jackson_version=2.11.2

src/main/kotlin/com/exactpro/th2/conn/dirty/fix/FixByteBufUtil.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2022 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 
 package com.exactpro.th2.conn.dirty.fix
 
+import com.exactpro.th2.conn.dirty.fix.FixField.FixGroup
 import com.exactpro.th2.netty.bytebuf.util.EMPTY_STRING
 import com.exactpro.th2.netty.bytebuf.util.endsWith
 import com.exactpro.th2.netty.bytebuf.util.get
@@ -43,6 +44,9 @@ fun ByteBuf.firstField(charset: Charset = UTF_8): FixField? = FixField.atOffset(
 @JvmOverloads
 fun ByteBuf.lastField(charset: Charset = UTF_8): FixField? = FixField.atOffset(this, writerIndex() - 1, charset)
 
+val ByteBuf.fields: Iterable<FixField>
+    get() = Iterable { iterator { forEachField { yield(it) } } }
+
 @JvmOverloads
 inline fun ByteBuf.forEachField(
     charset: Charset = UTF_8,
@@ -271,12 +275,23 @@ fun ByteBuf.updateLength(): ByteBuf {
     return replace(valueIndex, sohIndex, (endIndex - startIndex).toString())
 }
 
+fun Iterable<FixField>.findGroup(counter: Int, delimiter: Int, tags: Iterable<Int>): FixGroup? {
+    val counter = find { it.tag == counter } ?: return null
+    val delimiter = counter.next()?.takeIf { it.tag == delimiter } ?: return null
+    return FixGroup(counter, delimiter, tags)
+}
+
+interface FixElement {
+    fun previous(): FixElement?
+    fun next(): FixElement?
+}
+
 class FixField private constructor(
     private val buffer: ByteBuf,
     private var startIndex: Int,
     private var endIndex: Int,
     private val charset: Charset = UTF_8,
-) {
+) : FixElement {
     private var previous: FixField? = null
     private var next: FixField? = null
 
@@ -412,7 +427,7 @@ class FixField private constructor(
         return "${tag ?: EMPTY_STRING}$SEP_CHAR${value ?: EMPTY_STRING}${SOH_CHAR}"
     }
 
-    fun next(): FixField? = next ?: when (endIndex) {
+    override fun next(): FixField? = next ?: when (endIndex) {
         buffer.writerIndex() -> null
         else -> atOffset(
             buffer,
@@ -426,7 +441,7 @@ class FixField private constructor(
         }
     }
 
-    fun previous(): FixField? = previous ?: when (startIndex) {
+    override fun previous(): FixField? = previous ?: when (startIndex) {
         buffer.readerIndex() -> null
         else -> atOffset(
             buffer,
@@ -493,4 +508,37 @@ class FixField private constructor(
             }
         }
     }
+
+    class FixGroup(
+        val counter: FixField,
+        val delimiter: FixField,
+        val tags: Iterable<Int>,
+    ) : Iterable<FixField>, FixElement {
+        override fun iterator(): Iterator<FixField> = iterator {
+            var field: FixField? = delimiter
+
+            while (field != null) {
+                if (field === delimiter || field.tag in tags) yield(field) else break
+                field = field.next()
+            }
+        }
+
+        override fun next(): FixGroup? = find(delimiter.next()) { next() }
+
+        override fun previous(): FixGroup? = find(delimiter.previous()) { previous() }
+
+        private inline fun find(start: FixField?, next: FixField.() -> FixField?): FixGroup? {
+            val delimiter = delimiter.tag
+            var field = start
+
+            while (field != null) {
+                val tag = field.tag
+                if (tag == delimiter) return FixGroup(counter, field, tags)
+                if (tag !in tags) break
+                field = field.next()
+            }
+
+            return null
+        }
+    }
 }

src/main/kotlin/com/exactpro/th2/conn/dirty/fix/FixProtocolMangler.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2022 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -89,9 +89,7 @@ class FixProtocolMangler(context: IManglerContext) : IMangler {
         if (rules.isEmpty()) return null
 
         val rule = rules.filter { rule ->
-            rule.transform.any { transform ->
-                transform.conditions.all { it.matches(message) }
-            }
+            rule.transform.any { it.conditions.all(message::contains) }
         }.randomOrNull()
 
         if (rule == null) {
@@ -110,11 +108,18 @@ class FixProtocolManglerFactory : IManglerFactory {
     override fun create(context: IManglerContext) = FixProtocolMangler(context)
 }
 
-class FixProtocolManglerSettings(val rules: List<Rule> = emptyList()) : IManglerSettings
+class FixProtocolManglerSettings(
+    val context: Context = Context(),
+    val rules: List<Rule> = emptyList(),
+) : IManglerSettings {
+    init {
+        rules.forEach { it.init(context) }
+    }
+}
 
 private data class ActionRow(
     val corruptionType: String,
     val corruptedTag: Int,
     val corruptedValue: String?,
     val corruptionDescription: String,
-) : IRow
+) : IRow