From 05c1fded542b86b26e9e7261115dd43a7a8eab09 Mon Sep 17 00:00:00 2001
From: Oleg Smirnov <oleg.smirnov@exactprosystems.com>
Date: Thu, 13 Oct 2022 13:41:27 +0400
Subject: [PATCH 01/20] [TH2-4309] Add filtering by timestamp (#31)

* [TH2-4309] Update dependencies with vulnerabilities

* [TH2-4309] Add timestamp filter. Add sync with cradle

* [TH2-4309] Update version and readme

* [TH2-4309] Correct setter in main config
---
 .gitignore                                    |  1 +
 .project                                      | 23 --------
 README.md                                     | 17 +++++-
 build.gradle                                  | 30 ++++++-----
 gradle.properties                             |  2 +-
 .../com/exactpro/th2/readlog/LogData.java     | 26 ++++-----
 .../java/com/exactpro/th2/readlog/Main.java   | 16 +++---
 .../exactpro/th2/readlog/RegexLogParser.java  | 21 ++++++--
 .../th2/readlog/cfg/AliasConfiguration.java   | 16 ++++++
 .../readlog/cfg/LogReaderConfiguration.java   | 16 +++++-
 .../th2/readlog/impl/RegexpContentParser.java |  6 +--
 .../th2/readlog/impl/CradleReaderState.kt     | 53 +++++++++++++++++++
 .../exactpro/th2/readlog/TestLogParser.java   | 53 +++++++++++++++----
 .../cfg/TestLogReaderConfiguration.java       | 18 +++++--
 src/test/resources/log4j.properties           | 25 ---------
 src/test/resources/log4j2.properties          | 20 +++++++
 src/test/resources/test_cfg.json              |  3 +-
 17 files changed, 238 insertions(+), 108 deletions(-)
 delete mode 100644 .project
 create mode 100644 src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
 delete mode 100644 src/test/resources/log4j.properties
 create mode 100644 src/test/resources/log4j2.properties

diff --git a/.gitignore b/.gitignore
index 4420ec0..1c91b7c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+/out/
 /build/
 /.idea/
 /.gradle/
diff --git a/.project b/.project
deleted file mode 100644
index b28f978..0000000
--- a/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
-	<name>read-log</name>
-	<comment></comment>
-	<projects>
-	</projects>
-	<buildSpec>
-		<buildCommand>
-			<name>org.eclipse.jdt.core.javabuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-		<buildCommand>
-			<name>org.eclipse.m2e.core.maven2Builder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-	</buildSpec>
-	<natures>
-		<nature>org.eclipse.jdt.core.javanature</nature>
-		<nature>org.eclipse.m2e.core.maven2Nature</nature>
-	</natures>
-</projectDescription>
diff --git a/README.md b/README.md
index 0432076..414538e 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Log Reader User Manual 3.4.0
+# Log Reader User Manual 3.5.0
 
 ## Document Information
 
@@ -27,6 +27,7 @@ spec:
   type: th2-read
   custom-config:
       logDirectory: "log/dir"
+      syncWithCradle: true
       aliases:
         A:
           regexp: ".*"
@@ -44,6 +45,7 @@ spec:
           timestampRegexp: "^202.+?(?= QUICK)"
           timestampFormat: "yyyy-MM-dd HH:mm:ss"
           timestampZone: UTC
+          skipBefore: "2022-10-31T12:00:00Z"
         D:
           regexp: ".*"
           pathFilter: "fileC.*\\.log"
@@ -88,6 +90,7 @@ spec:
 ##### Reader configuration
 
 + logDirectory - the directory to watch files
++ syncWithCradle - enables synchronization with Cradle for timestamps and sequences that correspond to the alias
 + aliases - the mapping between alias and files that correspond to that alias
     + pathFilter - filter for files that correspond to that alias
     + regexp - the regular expression to extract data from the source lines
@@ -104,6 +107,9 @@ spec:
     + timestampFormat - the format for the timestamp extract from the log's line. **Works only with specified _timestampRegexp_ parameter**.
       If _timestampFormat_ is specified the timestamp extract with _timestampRegexp_ will be parsed using this format and used as a message's timestamp.
     + timestampZone - the zone which should be used to process the timestamp from the log file
+    + skipBefore - the parameter defines the minimum timestamp in UTC (ISO format) for log messages.
+      If log message has timestamp less than the specified one it will be dropped.
+      **NOTE: the parameter only works if 'timestampRegexp' and 'timestampFormat' are specified**
     + joinGroups - enables joining groups into a message in CSV format. Can be used to extract generic data from the log. Disabled by default.
     + groupsJoinDelimiter - the delimiter that will be used to join groups from the _regexp_ parameter. **Works only if _joinGroups_ is enabled**. The default value is `,`.
     + headersFormat - the headers' definition. The reader uses the keys as headers. The value to the key will be converted to a value for each match in the current line.
@@ -170,6 +176,15 @@ Output: 8=FIXT.1.1\u00019=66\u000135=A\u000134=1\u000149=NFT2_FIX1\u000156=FGW\u
 
 ## Changes
 
+### 3.5.0
+
++ Update dependencies with vulnerabilities
+  + log4j 1.2 is excluded
+  + kotlin updated to 1.6.21
++ Parameter `skipBefore` for filtering log messages by timestamp from the file
++ Parameter `syncWithCradle` for timestamp and sequence synchronization with Cradle.
+  Enabled by default.
+
 ### 3.4.0
 
 + Add parameter `timestampZone` which should be used to process the timestamp from the log file
diff --git a/build.gradle b/build.gradle
index 4229225..92e6cf0 100644
--- a/build.gradle
+++ b/build.gradle
@@ -18,7 +18,7 @@ plugins {
     id 'java-library'
     id 'application'
     id 'com.palantir.docker' version '0.25.0'
-    id 'org.jetbrains.kotlin.jvm' version '1.4.32'
+    id 'org.jetbrains.kotlin.jvm' version '1.6.21'
 }
 
 group 'com.exactpro.th2'
@@ -43,26 +43,22 @@ repositories {
 }
 
 dependencies {
-    api platform('com.exactpro.th2:bom:3.0.0')
-    api 'com.exactpro.th2:common:3.37.1'
-    api 'com.exactpro.th2:read-file-common-core:1.2.2'
+    api platform('com.exactpro.th2:bom:4.0.1')
+    api 'com.exactpro.th2:common:3.41.0'
+    api 'com.exactpro.th2:read-file-common-core:1.5.0-dev-SNAPSHOT'
 
-    implementation('com.opencsv:opencsv:5.3') {
+    implementation('com.opencsv:opencsv:5.7.0') {
         because("we need to write a correct CSV in case of free pattern is used")
     }
 
-    implementation 'javax.annotation:javax.annotation-api:1.3.2'
-    implementation "org.jetbrains.kotlin:kotlin-stdlib"
-
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
     implementation 'com.fasterxml.jackson.module:jackson-module-kotlin'
 
-    implementation "org.slf4j:slf4j-log4j12"
     implementation "org.slf4j:slf4j-api"
 
-    testImplementation 'org.junit.jupiter:junit-jupiter:5.6.2'
-    testImplementation 'org.mockito:mockito-core:3.6.0'
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.9.0'
+    testImplementation 'org.mockito:mockito-core:3.12.4'
 }
 
 test {
@@ -85,11 +81,17 @@ jar {
 }
 
 description = 'DataReaderClient'
-sourceCompatibility = '11'
+sourceCompatibility = JavaVersion.VERSION_11
 applicationName = 'service'
 
+tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile) {
+    kotlinOptions {
+        jvmTarget = "11"
+    }
+}
+
 distTar {
-    archiveName "${applicationName}.tar"
+    archiveFileName.set("${applicationName}.tar")
 }
 
 dockerPrepare {
@@ -101,5 +103,5 @@ docker {
 }
 
 application {
-    mainClassName = "com.exactpro.th2.readlog.Main"
+    mainClass.set("com.exactpro.th2.readlog.Main")
 }
diff --git a/gradle.properties b/gradle.properties
index 3ca4a3b..ff6d380 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1 +1 @@
-release_version=3.4.0
\ No newline at end of file
+release_version=3.5.0
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/LogData.java b/src/main/java/com/exactpro/th2/readlog/LogData.java
index ebe362f..839d7e4 100644
--- a/src/main/java/com/exactpro/th2/readlog/LogData.java
+++ b/src/main/java/com/exactpro/th2/readlog/LogData.java
@@ -15,18 +15,26 @@
  */
 package com.exactpro.th2.readlog;
 
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-final public class LogData {
+public final class LogData {
+    public static LogData EMPTY = new LogData(List.of());
     private List<String> body;
     private String rawTimestamp;
-    private LocalDateTime parsedTimestamp;
+    private Instant parsedTimestamp;
 
-    private ZoneId timestampZone;
+    public LogData() {
+        this(null);
+    }
+
+    private LogData(List<String> body) {
+        this.body = body;
+    }
 
     public void addBody(String item) {
         initIfNeeded();
@@ -45,22 +53,14 @@ public void setRawTimestamp(String rawTimestamp) {
         this.rawTimestamp = rawTimestamp;
     }
 
-    public LocalDateTime getParsedTimestamp() {
+    public Instant getParsedTimestamp() {
         return parsedTimestamp;
     }
 
-    public void setParsedTimestamp(LocalDateTime localDateTime) {
+    public void setParsedTimestamp(Instant localDateTime) {
         this.parsedTimestamp = localDateTime;
     }
 
-    public ZoneId getTimestampZone() {
-        return timestampZone;
-    }
-
-    public void setTimestampZone(ZoneId timestampZone) {
-        this.timestampZone = timestampZone;
-    }
-
     private void initIfNeeded() {
         if (body == null) {
             body = new ArrayList<>();
diff --git a/src/main/java/com/exactpro/th2/readlog/Main.java b/src/main/java/com/exactpro/th2/readlog/Main.java
index b027a17..af4ed4e 100644
--- a/src/main/java/com/exactpro/th2/readlog/Main.java
+++ b/src/main/java/com/exactpro/th2/readlog/Main.java
@@ -47,9 +47,11 @@
 import com.exactpro.th2.read.file.common.MovedFileTracker;
 import com.exactpro.th2.read.file.common.StreamId;
 import com.exactpro.th2.read.file.common.impl.DefaultFileReader;
+import com.exactpro.th2.read.file.common.impl.DefaultFileReader.Builder;
 import com.exactpro.th2.read.file.common.impl.RecoverableBufferedReaderWrapper;
 import com.exactpro.th2.read.file.common.state.impl.InMemoryReaderState;
 import com.exactpro.th2.readlog.cfg.LogReaderConfiguration;
+import com.exactpro.th2.readlog.impl.CradleReaderState;
 import com.exactpro.th2.readlog.impl.RegexpContentParser;
 import kotlin.Unit;
 import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -69,7 +71,7 @@ public static void main(String[] args) {
         var condition = lock.newCondition();
         configureShutdownHook(toDispose, lock, condition);
 
-        CommonMetrics.setLiveness(true);
+        CommonMetrics.LIVENESS_MONITOR.enable();
         CommonFactory commonFactory = CommonFactory.createFromArguments(args);
         toDispose.add(commonFactory);
 
@@ -108,13 +110,15 @@ public static void main(String[] args) {
             eventBatchRouter.sendAll(EventBatch.newBuilder().addEvents(protoEvent).build());
             EventID rootId = protoEvent.getId();
 
-            CommonMetrics.setReadiness(true);
-            AbstractFileReader<LineNumberReader> reader = new DefaultFileReader.Builder<>(
+            CommonMetrics.READINESS_MONITOR.enable();
+            AbstractFileReader<LineNumberReader> reader = new Builder<>(
                     configuration.getCommon(),
                     directoryChecker,
                     new RegexpContentParser(logParser),
                     new MovedFileTracker(configuration.getLogDirectory()),
-                    new InMemoryReaderState(),
+                    configuration.isSyncWithCradle()
+                            ? new CradleReaderState(commonFactory.getCradleManager().getStorage())
+                            : new InMemoryReaderState(),
                     Main::createSource
             )
                     .readFileImmediately()
@@ -201,7 +205,7 @@ private static FileSourceWrapper<LineNumberReader> createSource(StreamId streamI
     private static void configureShutdownHook(Deque<AutoCloseable> resources, ReentrantLock lock, Condition condition) {
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             LOGGER.info("Shutdown start");
-            CommonMetrics.setReadiness(false);
+            CommonMetrics.READINESS_MONITOR.disable();
             try {
                 lock.lock();
                 condition.signalAll();
@@ -216,7 +220,7 @@ private static void configureShutdownHook(Deque<AutoCloseable> resources, Reentr
                 }
             });
 
-            CommonMetrics.setLiveness(false);
+            CommonMetrics.LIVENESS_MONITOR.disable();
             LOGGER.info("Shutdown end");
         }, "Shutdown hook"));
     }
diff --git a/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java b/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java
index 0a077b2..85b870f 100644
--- a/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java
+++ b/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java
@@ -17,7 +17,10 @@
 
 import java.io.StringWriter;
 import java.time.DateTimeException;
+import java.time.Instant;
 import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.List;
@@ -96,17 +99,27 @@ public LogData parse(StreamId streamId, String raw) {
         // DateTime from log
         DateTimeFormatter timestampFormat = configuration.getTimestampFormat();
         if (timestampFormat != null) {
-            parseTimestamp(timestampFormat, resultData);
-            resultData.setTimestampZone(configuration.getTimestampZone());
+            ZoneOffset offset = Objects.requireNonNullElse(configuration.getTimestampZone(), ZoneId.systemDefault())
+                    .getRules().getOffset(Instant.now());
+            parseTimestamp(timestampFormat, resultData, offset);
+        }
+
+        if (resultData.getParsedTimestamp() != null && configuration.getSkipBefore() != null) {
+            if (resultData.getParsedTimestamp().isBefore(configuration.getSkipBefore())) {
+                logger.trace("Content dropped because of 'skipBefore' condition. Log timestamp: {}, Skip before: {}",
+                        resultData.getParsedTimestamp(), configuration.getSkipBefore()
+                );
+                return LogData.EMPTY;
+            }
         }
 
         return resultData;
     }
 
-    private void parseTimestamp(DateTimeFormatter formatter, LogData data) {
+    private void parseTimestamp(DateTimeFormatter formatter, LogData data, ZoneOffset offset) {
         String rawTimestamp = data.getRawTimestamp();
         try {
-            LocalDateTime dateTime = LocalDateTime.parse(rawTimestamp, formatter);
+            Instant dateTime = LocalDateTime.parse(rawTimestamp, formatter).toInstant(offset);
             data.setParsedTimestamp(dateTime);
             logger.trace("ParsedTimestamp: {}", dateTime);
         } catch (DateTimeException e) {
diff --git a/src/main/java/com/exactpro/th2/readlog/cfg/AliasConfiguration.java b/src/main/java/com/exactpro/th2/readlog/cfg/AliasConfiguration.java
index a2212cc..98b27ed 100644
--- a/src/main/java/com/exactpro/th2/readlog/cfg/AliasConfiguration.java
+++ b/src/main/java/com/exactpro/th2/readlog/cfg/AliasConfiguration.java
@@ -19,6 +19,7 @@
 import com.exactpro.th2.common.grpc.Direction;
 import com.fasterxml.jackson.annotation.JsonPropertyDescription;
 
+import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
@@ -60,6 +61,12 @@ public class AliasConfiguration {
             + "It not set the time zone from the local machine will be taken")
     private ZoneId timestampZone;
 
+    @JsonPropertyDescription(
+            "The date time in ISO format that will be used to filter log messages if 'timestampRegexp' and 'timestampFormat' are set."
+                    + "Otherwise, the filtering won't be applied"
+    )
+    private Instant skipBefore;
+
     @JsonCreator
     public AliasConfiguration(
             @JsonProperty(value = "regexp", required = true) String regexp,
@@ -149,4 +156,13 @@ public ZoneId getTimestampZone() {
     public void setTimestampZone(ZoneOffset timestampZone) {
         this.timestampZone = timestampZone;
     }
+
+    @Nullable
+    public Instant getSkipBefore() {
+        return skipBefore;
+    }
+
+    public void setSkipBefore(@Nullable Instant skipBefore) {
+        this.skipBefore = skipBefore;
+    }
 }
diff --git a/src/main/java/com/exactpro/th2/readlog/cfg/LogReaderConfiguration.java b/src/main/java/com/exactpro/th2/readlog/cfg/LogReaderConfiguration.java
index 6fe6410..7ddf5be 100644
--- a/src/main/java/com/exactpro/th2/readlog/cfg/LogReaderConfiguration.java
+++ b/src/main/java/com/exactpro/th2/readlog/cfg/LogReaderConfiguration.java
@@ -28,11 +28,14 @@
 import com.fasterxml.jackson.annotation.JsonPropertyDescription;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.fasterxml.jackson.module.kotlin.KotlinFeature;
 import com.fasterxml.jackson.module.kotlin.KotlinModule;
 
 public class LogReaderConfiguration {
     public static final ObjectMapper MAPPER = new ObjectMapper()
-            .registerModule(new KotlinModule())
+            .registerModule(new KotlinModule.Builder()
+                    .enable(KotlinFeature.NullIsSameAsDefault)
+                    .build())
             .registerModule(new JavaTimeModule());
 
     @JsonProperty(required = true)
@@ -46,6 +49,9 @@ public class LogReaderConfiguration {
 
     private Duration pullingInterval = Duration.ofSeconds(5);
 
+    @JsonPropertyDescription("Enables synchronization information about last timestamp and sequence for stream with Cradle")
+    private boolean syncWithCradle = true;
+
     @JsonCreator
     public LogReaderConfiguration(@JsonProperty("logDirectory") Path logDirectory) {
         this.logDirectory = Objects.requireNonNull(logDirectory, "'Log directory' parameter");
@@ -78,4 +84,12 @@ public Duration getPullingInterval() {
     public void setPullingInterval(Duration pullingInterval) {
         this.pullingInterval = pullingInterval;
     }
+
+    public boolean isSyncWithCradle() {
+        return syncWithCradle;
+    }
+
+    public void setSyncWithCradle(boolean syncWithCradle) {
+        this.syncWithCradle = syncWithCradle;
+    }
 }
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java b/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java
index a5f4a9e..e3dfa86 100644
--- a/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java
+++ b/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java
@@ -60,11 +60,7 @@ protected List<RawMessage.Builder> lineToMessages(@Nonnull StreamId streamId, @N
 
     private void setupMetadata(RawMessageMetadata.Builder builder, LogData logData) {
         if (logData.getParsedTimestamp() != null) {
-            ZoneOffset currentOffsetForMyZone = Objects.requireNonNullElse(
-                    logData.getTimestampZone(),
-                    ZoneId.systemDefault()
-            ).getRules().getOffset(Instant.now());
-            builder.setTimestamp(MessageUtils.toTimestamp(logData.getParsedTimestamp(),currentOffsetForMyZone));
+            builder.setTimestamp(MessageUtils.toTimestamp(logData.getParsedTimestamp()));
         }
         if (logData.getRawTimestamp() != null) {
             builder.putProperties("logTimestamp", logData.getRawTimestamp());
diff --git a/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt b/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
new file mode 100644
index 0000000..c2069ba
--- /dev/null
+++ b/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2022. Exactpro (Exactpro Systems Limited)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.exactpro.th2.readlog.impl
+
+import com.exactpro.cradle.CradleStorage
+import com.exactpro.cradle.messages.StoredMessageId
+import com.exactpro.th2.common.util.toCradleDirection
+import com.exactpro.th2.read.file.common.StreamId
+import com.exactpro.th2.read.file.common.state.ReaderState
+import com.exactpro.th2.read.file.common.state.StreamData
+import com.exactpro.th2.read.file.common.state.impl.InMemoryReaderState
+
+class CradleReaderState private constructor(
+    private val cradleStorage: CradleStorage,
+    private val delegate: ReaderState,
+): ReaderState by delegate {
+    constructor(cradleStorage: CradleStorage) : this(cradleStorage, InMemoryReaderState())
+
+    override fun get(streamId: StreamId): StreamData? {
+        return delegate[streamId] ?: cradleStorage.getLastMessageIndex(
+            streamId.sessionAlias,
+            streamId.direction.toCradleDirection(),
+        ).let { lastSequence ->
+            when {
+                lastSequence < 0 -> null
+                else -> cradleStorage.getMessage(StoredMessageId(
+                    streamId.sessionAlias,
+                    streamId.direction.toCradleDirection(),
+                    lastSequence
+                )).run {
+                    StreamData(
+                        timestamp,
+                        index,
+                    )
+                }
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/test/java/com/exactpro/th2/readlog/TestLogParser.java b/src/test/java/com/exactpro/th2/readlog/TestLogParser.java
index dc8db2d..652dda3 100644
--- a/src/test/java/com/exactpro/th2/readlog/TestLogParser.java
+++ b/src/test/java/com/exactpro/th2/readlog/TestLogParser.java
@@ -19,6 +19,9 @@
 import com.exactpro.th2.common.grpc.Direction;
 import com.exactpro.th2.read.file.common.StreamId;
 import com.exactpro.th2.readlog.cfg.AliasConfiguration;
+
+import java.time.Instant;
+import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -27,11 +30,16 @@
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
 
 import static org.junit.jupiter.api.Assertions.assertAll;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.params.provider.Arguments.arguments;
 
 public class TestLogParser {
 
@@ -40,6 +48,37 @@ public class TestLogParser {
     static final String TEST_MESSAGE_ALIAS_WRONG_TIMESTAMP_PATTERN = "tma_wrong_pattern";
     static final String RAW_LOG = "2021-03-23 13:21:37.991337479 QUICK.TEST INFO quicktest (Test.cpp:99) - incoming fix message fix NewOrderSingle={ AuthenticationBlock={ UserID=\"qwrqwrq\" SessionKey=123456 } Header={ MsgTime=2021-Mar-21 21:21:21.210000000 CreationTime=2021-Mar-21 21:21:21.210000000 } NewOrder={ InstrumentBlock={ InstrSymbol=\"TEST_SYMBOL\" SecurityID=\"212121\" SecurityIDSource=TestSource SecurityExchange=\"test\" }}}";
 
+    @ParameterizedTest(name = "Has message: {1}, Skips before: {0}")
+    @MethodSource("skipMessageParams")
+    void skipMessageByTimestamp(Instant skipBefore, boolean hasMessages) {
+        AliasConfiguration cfg = new AliasConfiguration(
+                ".+",
+                ".*",
+                Map.of(),
+                "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3,9}",
+                "yyyy-MM-dd HH:mm:ss.SSSSSSSSS"
+        );
+        cfg.setGroups(List.of(0));
+        cfg.setTimestampZone(ZoneOffset.UTC);
+        cfg.setSkipBefore(skipBefore);
+        RegexLogParser parser = new RegexLogParser(Map.of(
+                "test", cfg
+        ));
+
+        LogData data = parser.parse(new StreamId("test", Direction.FIRST), "2021-03-23 13:21:37.991337479 some data");
+        assertEquals(hasMessages, !data.getBody().isEmpty(), () -> "unexpected data " + data.getBody());
+        if (hasMessages) {
+            assertEquals(List.of("2021-03-23 13:21:37.991337479 some data"), data.getBody());
+        }
+    }
+
+    static List<Arguments> skipMessageParams() {
+        return List.of(
+                arguments(Instant.parse("2021-03-23T13:21:38Z"), false),
+                arguments(Instant.parse("2021-03-23T13:21:36Z"), true)
+        );
+    }
+
     @Test
     void parser() {
         RegexLogParser logParser = new RegexLogParser(getConfiguration());
@@ -47,13 +86,7 @@ void parser() {
         assertEquals(1, data.getBody().size());
         assertEquals("NewOrderSingle={ AuthenticationBlock={ UserID=\"qwrqwrq\" SessionKey=123456 } Header={ MsgTime=2021-Mar-21 21:21:21.210000000 CreationTime=2021-Mar-21 21:21:21.210000000 } NewOrder={ InstrumentBlock={ InstrSymbol=\"TEST_SYMBOL\" SecurityID=\"212121\" SecurityIDSource=TestSource SecurityExchange=\"test\" }}}", data.getBody().get(0));
         assertEquals("2021-03-23 13:21:37.991337479", data.getRawTimestamp());
-        assertEquals(2021, data.getParsedTimestamp().getYear());
-        assertEquals(23, data.getParsedTimestamp().getDayOfMonth());
-        assertEquals(3, data.getParsedTimestamp().getMonthValue());
-        assertEquals(13, data.getParsedTimestamp().getHour());
-        assertEquals(21, data.getParsedTimestamp().getMinute());
-        assertEquals(37, data.getParsedTimestamp().getSecond());
-        assertEquals(991337479, data.getParsedTimestamp().getNano());
+        assertEquals(Instant.parse("2021-03-23T13:21:37.991337479Z"), data.getParsedTimestamp());
     }
 
     @Test
@@ -98,9 +131,11 @@ private Map<String, AliasConfiguration> getConfiguration() {
         String timestampFormat = "yyyy-MM-dd HH:mm:ss.SSSSSSSSS";
 
         Map<String, AliasConfiguration> result = new HashMap<>();
-        result.put(TEST_MESSAGE_ALIAS, new AliasConfiguration(regexp, "",
+        AliasConfiguration cfg = new AliasConfiguration(regexp, "",
                 Map.of(Direction.FIRST, "incoming", Direction.SECOND, "outgoing"),
-                timextampRegexp, timestampFormat));
+                timextampRegexp, timestampFormat);
+        cfg.setTimestampZone(ZoneOffset.UTC);
+        result.put(TEST_MESSAGE_ALIAS, cfg);
         result.put(TEST_MESSAGE_ALIAS_WRONG_TIMESTAMP_FORMAT, new AliasConfiguration(regexp, "", Collections.emptyMap(), timextampRegexp, "123"));
         result.put(TEST_MESSAGE_ALIAS_WRONG_TIMESTAMP_PATTERN, new AliasConfiguration(regexp, "", Collections.emptyMap(), "3012.*", timestampFormat));
 
diff --git a/src/test/java/com/exactpro/th2/readlog/cfg/TestLogReaderConfiguration.java b/src/test/java/com/exactpro/th2/readlog/cfg/TestLogReaderConfiguration.java
index f0628dc..6a36e73 100644
--- a/src/test/java/com/exactpro/th2/readlog/cfg/TestLogReaderConfiguration.java
+++ b/src/test/java/com/exactpro/th2/readlog/cfg/TestLogReaderConfiguration.java
@@ -19,6 +19,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.time.Duration;
+import java.time.Instant;
 import java.time.format.DateTimeFormatter;
 import java.util.Map;
 import java.util.Objects;
@@ -38,11 +39,18 @@ void deserializes() throws IOException {
             LogReaderConfiguration cfg = LogReaderConfiguration.MAPPER.readValue(input, LogReaderConfiguration.class);
             assertEquals(Duration.ofSeconds(5), cfg.getPullingInterval());
             assertEquals(Set.of("A", "B"), cfg.getAliases().keySet());
-            assertEquals(Set.of(Direction.FIRST, Direction.SECOND), cfg.getAliases().get("A").getDirectionToPattern().keySet());
-            assertEquals(Set.of(Direction.FIRST), cfg.getAliases().get("B").getDirectionToPattern().keySet());
-            assertEquals(".*", Objects.requireNonNull(cfg.getAliases().get("A").getRegexp()).pattern());
-            assertEquals("202.*$", Objects.requireNonNull(cfg.getAliases().get("B").getTimestampRegexp()).pattern());
-            assertEquals(DateTimeFormatter.ofPattern("yyyy.MM.dd").getResolverFields(), Objects.requireNonNull(cfg.getAliases().get("B").getTimestampFormat()).getResolverFields());
+            AliasConfiguration aliasA = cfg.getAliases().get("A");
+            AliasConfiguration aliasB = cfg.getAliases().get("B");
+            assertEquals(Set.of(Direction.FIRST, Direction.SECOND), aliasA.getDirectionToPattern().keySet());
+            assertEquals(Set.of(Direction.FIRST), aliasB.getDirectionToPattern().keySet());
+            assertEquals(".*", Objects.requireNonNull(aliasA.getRegexp()).pattern());
+            assertEquals("202.*$", Objects.requireNonNull(aliasB.getTimestampRegexp()).pattern());
+            assertEquals(DateTimeFormatter.ofPattern("yyyy.MM.dd").getResolverFields(), Objects.requireNonNull(aliasB.getTimestampFormat()).getResolverFields());
+            assertEquals(
+                    Instant.parse("2022-10-31T10:35:00Z"),
+                    aliasB.getSkipBefore(),
+                    "unexpected 'skipBefore' value"
+            );
         }
     }
 
diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties
deleted file mode 100644
index 492a8b8..0000000
--- a/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,25 +0,0 @@
-################################################################################
-# Copyright 2020-2020 Exactpro (Exactpro Systems Limited)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-log4j.rootLogger=TRACE,FILE, CON
-log4j.appender.FILE=org.apache.log4j.FileAppender
-log4j.appender.FILE.File=build/logs/test_build.log
-log4j.appender.FILE.Append=false
-log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.FILE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-6p [%-15t] %c - %m%n
-
-log4j.appender.CON=org.apache.log4j.ConsoleAppender
-log4j.appender.CON.layout=org.apache.log4j.PatternLayout
-log4j.appender.CON.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-6p [%-15t] %c - %m%n
\ No newline at end of file
diff --git a/src/test/resources/log4j2.properties b/src/test/resources/log4j2.properties
new file mode 100644
index 0000000..e1f6b85
--- /dev/null
+++ b/src/test/resources/log4j2.properties
@@ -0,0 +1,20 @@
+name=Th2Logger
+# Console appender configuration
+appender.console.type=Console
+appender.console.name=consoleLogger
+appender.console.layout.type=PatternLayout
+appender.console.layout.pattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-6p [%-15t] %c - %m%n
+
+#logger.th2.name=com.exactpro.th2
+#logger.th2.level=DEBUG
+#logger.evolution.name=com.exactpro.evolution
+#logger.evolution.level=DEBUG
+#logger.cradle.name=com.exactpro.cradle
+#logger.cradle.level=DEBUG
+#logger.sf.name=com.exactpro.sf
+#logger.sf.level=DEBUG
+
+# Root logger level
+rootLogger.level=TRACE
+# Root logger referring to console appender
+rootLogger.appenderRef.stdout.ref=consoleLogger
\ No newline at end of file
diff --git a/src/test/resources/test_cfg.json b/src/test/resources/test_cfg.json
index d49ad34..99d767b 100644
--- a/src/test/resources/test_cfg.json
+++ b/src/test/resources/test_cfg.json
@@ -13,7 +13,8 @@
       "regexp": ".*",
       "pathFilter": "fileB.*\\.log",
       "timestampRegexp": "202.*$",
-      "timestampFormat": "yyyy.MM.dd"
+      "timestampFormat": "yyyy.MM.dd",
+      "skipBefore": "2022-10-31T10:35:00Z"
     }
   },
   "common": {

From 22817c37a431f9424a9b131c71fcdc3857031bd8 Mon Sep 17 00:00:00 2001
From: Revaz Chikovani <revaz.chikovani@exactprosystems.com>
Date: Wed, 16 Nov 2022 13:54:11 +0400
Subject: [PATCH 02/20] Add new class for log reader creation (#32)

* add new class for log reader creation

* add use of LogFileReader to Main
---
 .github/workflows/ci-unwelcome-words.yml      |  2 +-
 .github/workflows/dev-docker-publish.yml      | 25 +++---
 .github/workflows/docker-publish.yml          | 22 ++---
 .../java/com/exactpro/th2/readlog/Main.java   | 80 +++++------------
 .../th2/readlog/impl/LogFileReader.java       | 87 +++++++++++++++++++
 .../th2/readlog/impl/lambdas/ForOnError.java  | 24 +++++
 .../impl/lambdas/ForOnSourceCorrupted.java    | 26 ++++++
 .../readlog/impl/lambdas/ForOnStreamData.java | 27 ++++++
 8 files changed, 211 insertions(+), 82 deletions(-)
 create mode 100644 src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
 create mode 100644 src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnError.java
 create mode 100644 src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnSourceCorrupted.java
 create mode 100644 src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnStreamData.java

diff --git a/.github/workflows/ci-unwelcome-words.yml b/.github/workflows/ci-unwelcome-words.yml
index cd7adcf..a7e7f3a 100644
--- a/.github/workflows/ci-unwelcome-words.yml
+++ b/.github/workflows/ci-unwelcome-words.yml
@@ -7,7 +7,7 @@ jobs:
   test:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         ref: ${{ github.sha }}
     - name: Checkout tool
diff --git a/.github/workflows/dev-docker-publish.yml b/.github/workflows/dev-docker-publish.yml
index 52d8908..f7b18a9 100644
--- a/.github/workflows/dev-docker-publish.yml
+++ b/.github/workflows/dev-docker-publish.yml
@@ -10,35 +10,38 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       # Prepare custom build version
       - name: Get branch name
         id: branch
-        run: echo ::set-output name=branch_name::${GITHUB_REF#refs/*/}
+        run: echo "branch_name=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
+      - name: Get SHA of the commit
+        id: sha
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: Get release_version
         id: ver
-        uses: christian-draeger/read-properties@1.0.1
+        uses: christian-draeger/read-properties@1.1.1
         with:
           path: gradle.properties
-          property: release_version
+          properties: release_version
       - name: Build custom release version
         id: release_ver
-        run: echo ::set-output name=value::"${{ steps.ver.outputs.value }}-${{ steps.branch.outputs.branch_name }}-${{ github.run_id }}"
+        run: echo value="${{ steps.ver.outputs.release_version }}-${{ steps.branch.outputs.branch_name }}-${{ github.run_id }}-${{ steps.sha.outputs.sha_short }}" >> $GITHUB_OUTPUT
       - name: Show custom release version
         run: echo ${{ steps.release_ver.outputs.value }}
       # Build and publish image
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - uses: docker/login-action@v1
+        uses: docker/setup-buildx-action@v2
+      - uses: docker/login-action@v2
         with:
           registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.CR_PAT }}
-      - run: echo "::set-output name=REPOSITORY_NAME::$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')"
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - run: echo "REPOSITORY_NAME=$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" >> $GITHUB_OUTPUT
         id: meta
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           push: true
           tags: ghcr.io/${{ github.repository }}:${{ steps.release_ver.outputs.value }}
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
index 9f51020..e529cdb 100644
--- a/.github/workflows/docker-publish.yml
+++ b/.github/workflows/docker-publish.yml
@@ -12,26 +12,26 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - uses: docker/login-action@v1
+        uses: docker/setup-buildx-action@v2
+      - uses: docker/login-action@v2
         with:
           registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.CR_PAT }}
-      - run: echo "::set-output name=REPOSITORY_NAME::$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')"
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - run: echo "REPOSITORY_NAME=$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" >> $GITHUB_OUTPUT
         id: meta
       - name: Read version from gradle.properties
         id: read_property
-        uses: christian-draeger/read-properties@1.0.1
+        uses: christian-draeger/read-properties@1.1.1
         with:
           path: ./gradle.properties
-          property: release_version
+          properties: release_version
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           push: true
-          tags: ghcr.io/${{ github.repository }}:${{ steps.read_property.outputs.value }}
-          labels: com.exactpro.th2.${{ steps.meta.outputs.REPOSITORY_NAME }}=${{ steps.read_property.outputs.value }}
\ No newline at end of file
+          tags: ghcr.io/${{ github.repository }}:${{ steps.read_property.outputs.release_version }}
+          labels: com.exactpro.th2.${{ steps.meta.outputs.REPOSITORY_NAME }}=${{ steps.read_property.outputs.release_version }}
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/Main.java b/src/main/java/com/exactpro/th2/readlog/Main.java
index af4ed4e..e2a27e8 100644
--- a/src/main/java/com/exactpro/th2/readlog/Main.java
+++ b/src/main/java/com/exactpro/th2/readlog/Main.java
@@ -16,22 +16,6 @@
 
 package com.exactpro.th2.readlog;
 
-import java.io.IOException;
-import java.io.LineNumberReader;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayDeque;
-import java.util.Comparator;
-import java.util.Deque;
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.stream.Collectors;
-
 import com.exactpro.th2.common.event.Event;
 import com.exactpro.th2.common.event.EventUtils;
 import com.exactpro.th2.common.grpc.EventBatch;
@@ -42,24 +26,28 @@
 import com.exactpro.th2.common.schema.factory.CommonFactory;
 import com.exactpro.th2.common.schema.message.MessageRouter;
 import com.exactpro.th2.read.file.common.AbstractFileReader;
-import com.exactpro.th2.read.file.common.DirectoryChecker;
-import com.exactpro.th2.read.file.common.FileSourceWrapper;
-import com.exactpro.th2.read.file.common.MovedFileTracker;
 import com.exactpro.th2.read.file.common.StreamId;
-import com.exactpro.th2.read.file.common.impl.DefaultFileReader;
-import com.exactpro.th2.read.file.common.impl.DefaultFileReader.Builder;
-import com.exactpro.th2.read.file.common.impl.RecoverableBufferedReaderWrapper;
 import com.exactpro.th2.read.file.common.state.impl.InMemoryReaderState;
 import com.exactpro.th2.readlog.cfg.LogReaderConfiguration;
 import com.exactpro.th2.readlog.impl.CradleReaderState;
-import com.exactpro.th2.readlog.impl.RegexpContentParser;
+import com.exactpro.th2.readlog.impl.LogFileReader;
 import kotlin.Unit;
-import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static java.util.Comparator.comparing;
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.nio.file.Path;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
 
 public class Main {
 
@@ -84,19 +72,6 @@ public static void main(String[] args) {
                 throw new IllegalArgumentException("Alias " + alias + " has parameter joinGroups = true but does not have any headers defined");
             }
         });
-        Comparator<Path> pathComparator = comparing(it -> it.getFileName().toString(), String.CASE_INSENSITIVE_ORDER);
-        var directoryChecker = new DirectoryChecker(
-                configuration.getLogDirectory(),
-                (Path path) -> configuration.getAliases().entrySet().stream()
-                        .filter(entry -> entry.getValue().getPathFilter().matcher(path.getFileName().toString()).matches())
-                        .flatMap(entry -> entry.getValue().getDirectionToPattern()
-                                .keySet().stream()
-                                .map(direction -> new StreamId(entry.getKey(), direction))
-                        ).collect(Collectors.toSet()),
-                files -> files.sort(pathComparator),
-                path -> true
-        );
-        RegexLogParser logParser = new RegexLogParser(configuration.getAliases());
 
         if (configuration.getPullingInterval().isNegative()) {
             throw new IllegalArgumentException("Pulling interval " + configuration.getPullingInterval() + " must not be negative");
@@ -111,22 +86,17 @@ public static void main(String[] args) {
             EventID rootId = protoEvent.getId();
 
             CommonMetrics.READINESS_MONITOR.enable();
-            AbstractFileReader<LineNumberReader> reader = new Builder<>(
-                    configuration.getCommon(),
-                    directoryChecker,
-                    new RegexpContentParser(logParser),
-                    new MovedFileTracker(configuration.getLogDirectory()),
+
+            AbstractFileReader<LineNumberReader> reader
+                    = LogFileReader.getLogFileReader(
+                    configuration,
                     configuration.isSyncWithCradle()
                             ? new CradleReaderState(commonFactory.getCradleManager().getStorage())
                             : new InMemoryReaderState(),
-                    Main::createSource
-            )
-                    .readFileImmediately()
-                    .acceptNewerFiles()
-                    .onStreamData((streamId, builders) -> publishMessages(rawMessageBatchRouter, streamId, builders))
-                    .onError((streamId, message, ex) -> publishErrorEvent(eventBatchRouter, streamId, message, ex, rootId))
-                    .onSourceCorrupted((streamId, path, e) -> publishSourceCorruptedEvent(eventBatchRouter, path, streamId, e, rootId))
-                    .build();
+                    (streamId, builders) -> publishMessages(rawMessageBatchRouter, streamId, builders),
+                    (streamId, message, ex) -> publishErrorEvent(eventBatchRouter, streamId, message, ex, rootId),
+                    (streamId, path, e) -> publishSourceCorruptedEvent(eventBatchRouter, path, streamId, e, rootId)
+                    );
 
             toDispose.add(reader);
 
@@ -194,14 +164,6 @@ private static Unit publishMessages(MessageRouter<RawMessageBatch> rawMessageBat
         return Unit.INSTANCE;
     }
 
-    private static FileSourceWrapper<LineNumberReader> createSource(StreamId streamId, Path path) {
-        try {
-            return new RecoverableBufferedReaderWrapper(new LineNumberReader(Files.newBufferedReader(path)));
-        } catch (IOException e) {
-            return ExceptionUtils.rethrow(e);
-        }
-    }
-
     private static void configureShutdownHook(Deque<AutoCloseable> resources, ReentrantLock lock, Condition condition) {
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             LOGGER.info("Shutdown start");
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java b/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
new file mode 100644
index 0000000..20ee385
--- /dev/null
+++ b/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
@@ -0,0 +1,87 @@
+/*******************************************************************************
+ * Copyright 2022 Exactpro (Exactpro Systems Limited)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package com.exactpro.th2.readlog.impl;
+
+import com.exactpro.th2.read.file.common.*;
+import com.exactpro.th2.read.file.common.impl.DefaultFileReader.Builder;
+import com.exactpro.th2.read.file.common.impl.RecoverableBufferedReaderWrapper;
+import com.exactpro.th2.read.file.common.state.ReaderState;
+import com.exactpro.th2.readlog.RegexLogParser;
+import com.exactpro.th2.readlog.cfg.LogReaderConfiguration;
+import com.exactpro.th2.readlog.impl.lambdas.ForOnError;
+import com.exactpro.th2.readlog.impl.lambdas.ForOnSourceCorrupted;
+import com.exactpro.th2.readlog.impl.lambdas.ForOnStreamData;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.stream.Collectors;
+
+import static java.util.Comparator.comparing;
+
+public class LogFileReader {
+
+    public static AbstractFileReader<LineNumberReader> getLogFileReader(
+            LogReaderConfiguration configuration,
+            ReaderState readerState,
+            ForOnStreamData forStream,
+            ForOnError forError,
+            ForOnSourceCorrupted forCorrupted
+    ){
+            return new Builder<>(
+                    configuration.getCommon(),
+                    getDirectoryChecker(configuration),
+                    new RegexpContentParser(new RegexLogParser(configuration.getAliases())),
+                    new MovedFileTracker(configuration.getLogDirectory()),
+                    readerState,
+                    LogFileReader::createSource
+            )
+                    .readFileImmediately()
+                    .acceptNewerFiles()
+                    .onStreamData(forStream::action)
+                    .onError(forError::action)
+                    .onSourceCorrupted(forCorrupted::action)
+                    .build();
+
+    }
+
+    private static DirectoryChecker getDirectoryChecker(LogReaderConfiguration configuration) {
+        Comparator<Path> pathComparator = comparing(it -> it.getFileName().toString(), String.CASE_INSENSITIVE_ORDER);
+        return new DirectoryChecker(
+                configuration.getLogDirectory(),
+                (Path path) -> configuration.getAliases().entrySet().stream()
+                        .filter(entry -> entry.getValue().getPathFilter().matcher(path.getFileName().toString()).matches())
+                        .flatMap(entry -> entry.getValue().getDirectionToPattern()
+                                .keySet().stream()
+                                .map(direction -> new StreamId(entry.getKey(), direction))
+                        ).collect(Collectors.toSet()),
+                files -> files.sort(pathComparator),
+                path -> true
+        );
+    }
+
+    private static FileSourceWrapper<LineNumberReader> createSource(StreamId streamId, Path path) {
+        try {
+            return new RecoverableBufferedReaderWrapper(new LineNumberReader(Files.newBufferedReader(path)));
+        } catch (IOException e) {
+            return ExceptionUtils.rethrow(e);
+        }
+    }
+}
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnError.java b/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnError.java
new file mode 100644
index 0000000..dcdee15
--- /dev/null
+++ b/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnError.java
@@ -0,0 +1,24 @@
+/*******************************************************************************
+ * Copyright 2022 Exactpro (Exactpro Systems Limited)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package com.exactpro.th2.readlog.impl.lambdas;
+
+import com.exactpro.th2.read.file.common.StreamId;
+import kotlin.Unit;
+
+public interface ForOnError {
+    Unit action(StreamId id, String message, Exception ex);
+}
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnSourceCorrupted.java b/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnSourceCorrupted.java
new file mode 100644
index 0000000..2d6344b
--- /dev/null
+++ b/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnSourceCorrupted.java
@@ -0,0 +1,26 @@
+/*******************************************************************************
+ * Copyright 2022 Exactpro (Exactpro Systems Limited)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package com.exactpro.th2.readlog.impl.lambdas;
+
+import com.exactpro.th2.read.file.common.StreamId;
+import kotlin.Unit;
+
+import java.nio.file.Path;
+
+public interface ForOnSourceCorrupted {
+    Unit action(StreamId id, Path path, Exception ex);
+}
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnStreamData.java b/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnStreamData.java
new file mode 100644
index 0000000..c35a662
--- /dev/null
+++ b/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnStreamData.java
@@ -0,0 +1,27 @@
+/*******************************************************************************
+ * Copyright 2022 Exactpro (Exactpro Systems Limited)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package com.exactpro.th2.readlog.impl.lambdas;
+
+import com.exactpro.th2.common.grpc.RawMessage;
+import com.exactpro.th2.read.file.common.StreamId;
+import kotlin.Unit;
+
+import java.util.List;
+
+public interface ForOnStreamData {
+    Unit action(StreamId id, List<RawMessage.Builder> builder);
+}

From 04180d96ed23498001e67c357c71bc189dc40031 Mon Sep 17 00:00:00 2001
From: Oleg Smirnov <oleg.smirnov@exactprosystems.com>
Date: Wed, 23 Nov 2022 15:07:00 +0400
Subject: [PATCH 03/20] Rebuild with never common read core (#34)


From 618fc556d9a1c14c8e138c5396839a2153ff220e Mon Sep 17 00:00:00 2001
From: Oleg Smirnov <oleg.smirnov@exactprosystems.com>
Date: Wed, 7 Dec 2022 14:00:20 +0400
Subject: [PATCH 04/20] Rebuild with newer core version. Correct imports (#35)

---
 .../java/com/exactpro/th2/readlog/impl/LogFileReader.java   | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java b/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
index 20ee385..124cbb5 100644
--- a/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
+++ b/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
@@ -16,7 +16,11 @@
 
 package com.exactpro.th2.readlog.impl;
 
-import com.exactpro.th2.read.file.common.*;
+import com.exactpro.th2.read.file.common.AbstractFileReader;
+import com.exactpro.th2.read.file.common.DirectoryChecker;
+import com.exactpro.th2.read.file.common.FileSourceWrapper;
+import com.exactpro.th2.read.file.common.MovedFileTracker;
+import com.exactpro.th2.read.file.common.StreamId;
 import com.exactpro.th2.read.file.common.impl.DefaultFileReader.Builder;
 import com.exactpro.th2.read.file.common.impl.RecoverableBufferedReaderWrapper;
 import com.exactpro.th2.read.file.common.state.ReaderState;

From ad1bc7a0f08178bf7fb1cc63dde14ba277f6d8a7 Mon Sep 17 00:00:00 2001
From: Oleg Smirnov <oleg.smirnov@exactprosystems.com>
Date: Fri, 9 Dec 2022 16:01:54 +0400
Subject: [PATCH 05/20] Rebuild with newer core. Possible fix for handing
 during checking a file (#36)

---
 .../kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt b/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
index c2069ba..fe81a66 100644
--- a/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
+++ b/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
@@ -18,11 +18,13 @@ package com.exactpro.th2.readlog.impl
 
 import com.exactpro.cradle.CradleStorage
 import com.exactpro.cradle.messages.StoredMessageId
+import com.exactpro.th2.common.grpc.RawMessage
 import com.exactpro.th2.common.util.toCradleDirection
 import com.exactpro.th2.read.file.common.StreamId
 import com.exactpro.th2.read.file.common.state.ReaderState
 import com.exactpro.th2.read.file.common.state.StreamData
 import com.exactpro.th2.read.file.common.state.impl.InMemoryReaderState
+import com.google.protobuf.ByteString
 
 class CradleReaderState private constructor(
     private val cradleStorage: CradleStorage,
@@ -45,6 +47,7 @@ class CradleReaderState private constructor(
                     StreamData(
                         timestamp,
                         index,
+                        content?.let(RawMessage::parseFrom)?.body ?: ByteString.EMPTY, // TODO: change when migrate to cradle storing only body in the content
                     )
                 }
             }

From c88bef87ad0103aeb56947d1672b580ad7a20525 Mon Sep 17 00:00:00 2001
From: Nikita Shaposhnikov <topru123@gmail.com>
Date: Thu, 14 Apr 2022 11:28:15 +0400
Subject: [PATCH 06/20] book and pages migration

---
 README.md                                              | 10 +++++++++-
 build.gradle                                           |  6 +++---
 gradle.properties                                      |  2 +-
 src/main/java/com/exactpro/th2/readlog/Main.java       |  3 ++-
 .../exactpro/th2/readlog/impl/RegexpContentParser.java |  2 +-
 5 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 414538e..e271382 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Log Reader User Manual 3.5.0
+# Log Reader User Manual 4.0.0
 
 ## Document Information
 
@@ -176,6 +176,14 @@ Output: 8=FIXT.1.1\u00019=66\u000135=A\u000134=1\u000149=NFT2_FIX1\u000156=FGW\u
 
 ## Changes
 
+### 4.0.0
+
++ Updated common to 4.0.0
+    + Migration to books-pages
++ Updated bom to 3.1.0
++ Updated read-file-common-core to 2.0.0
+
+
 ### 3.5.0
 
 + Update dependencies with vulnerabilities
diff --git a/build.gradle b/build.gradle
index 92e6cf0..1f44b3c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -43,9 +43,9 @@ repositories {
 }
 
 dependencies {
-    api platform('com.exactpro.th2:bom:4.0.1')
-    api 'com.exactpro.th2:common:3.41.0'
-    api 'com.exactpro.th2:read-file-common-core:1.5.0-dev-SNAPSHOT'
+    api platform('com.exactpro.th2:bom:4.0.2')
+    api 'com.exactpro.th2:common:5.0.0-dev-version-5-3838510969-SNAPSHOT'
+    api 'com.exactpro.th2:read-file-common-core:2.0.0-th2-4665-SNAPSHOT'
 
     implementation('com.opencsv:opencsv:5.7.0') {
         because("we need to write a correct CSV in case of free pattern is used")
diff --git a/gradle.properties b/gradle.properties
index ff6d380..bf5221a 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1 +1 @@
-release_version=3.5.0
\ No newline at end of file
+release_version=4.0.0
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/Main.java b/src/main/java/com/exactpro/th2/readlog/Main.java
index e2a27e8..b75b504 100644
--- a/src/main/java/com/exactpro/th2/readlog/Main.java
+++ b/src/main/java/com/exactpro/th2/readlog/Main.java
@@ -61,6 +61,7 @@ public static void main(String[] args) {
 
         CommonMetrics.LIVENESS_MONITOR.enable();
         CommonFactory commonFactory = CommonFactory.createFromArguments(args);
+        var boxBookName = commonFactory.getBoxConfiguration().getBookName();
         toDispose.add(commonFactory);
 
         MessageRouter<RawMessageBatch> rawMessageBatchRouter = commonFactory.getMessageRouterRawBatch();
@@ -81,7 +82,7 @@ public static void main(String[] args) {
             Event rootEvent = Event.start().endTimestamp()
                     .name("Log reader for " + String.join(",", configuration.getAliases().keySet()))
                     .type("Microservice");
-            var protoEvent = rootEvent.toProto(null);
+            var protoEvent = rootEvent.toProto(boxBookName);
             eventBatchRouter.sendAll(EventBatch.newBuilder().addEvents(protoEvent).build());
             EventID rootId = protoEvent.getId();
 
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java b/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java
index e3dfa86..60c655b 100644
--- a/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java
+++ b/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java
@@ -60,7 +60,7 @@ protected List<RawMessage.Builder> lineToMessages(@Nonnull StreamId streamId, @N
 
     private void setupMetadata(RawMessageMetadata.Builder builder, LogData logData) {
         if (logData.getParsedTimestamp() != null) {
-            builder.setTimestamp(MessageUtils.toTimestamp(logData.getParsedTimestamp()));
+            builder.getIdBuilder().setTimestamp(MessageUtils.toTimestamp(logData.getParsedTimestamp()));
         }
         if (logData.getRawTimestamp() != null) {
             builder.putProperties("logTimestamp", logData.getRawTimestamp());

From a387c06fcd69781c8cfeb5d965f95125871418a7 Mon Sep 17 00:00:00 2001
From: Oleg <oleg.smirnov@exactprosystems.com>
Date: Tue, 17 Jan 2023 13:24:44 +0400
Subject: [PATCH 07/20] [TH2-2150] Update common and file reader core versions

---
 .../java/com/exactpro/th2/readlog/Main.java   | 11 +++--
 .../th2/readlog/impl/LogFileReader.java       |  4 ++
 .../th2/readlog/impl/CradleReaderState.kt     | 42 ++++++++++---------
 3 files changed, 35 insertions(+), 22 deletions(-)

diff --git a/src/main/java/com/exactpro/th2/readlog/Main.java b/src/main/java/com/exactpro/th2/readlog/Main.java
index b75b504..ba95d7c 100644
--- a/src/main/java/com/exactpro/th2/readlog/Main.java
+++ b/src/main/java/com/exactpro/th2/readlog/Main.java
@@ -17,6 +17,7 @@
 package com.exactpro.th2.readlog;
 
 import com.exactpro.th2.common.event.Event;
+import com.exactpro.th2.common.event.Event.Status;
 import com.exactpro.th2.common.event.EventUtils;
 import com.exactpro.th2.common.grpc.EventBatch;
 import com.exactpro.th2.common.grpc.EventID;
@@ -81,8 +82,10 @@ public static void main(String[] args) {
         try {
             Event rootEvent = Event.start().endTimestamp()
                     .name("Log reader for " + String.join(",", configuration.getAliases().keySet()))
-                    .type("Microservice");
-            var protoEvent = rootEvent.toProto(boxBookName);
+                    .type("ReadLog")
+                    .status(Status.PASSED);
+            EventID componentRootEvent = commonFactory.getRootEventId();
+            var protoEvent = rootEvent.toProto(componentRootEvent);
             eventBatchRouter.sendAll(EventBatch.newBuilder().addEvents(protoEvent).build());
             EventID rootId = protoEvent.getId();
 
@@ -92,8 +95,10 @@ public static void main(String[] args) {
                     = LogFileReader.getLogFileReader(
                     configuration,
                     configuration.isSyncWithCradle()
-                            ? new CradleReaderState(commonFactory.getCradleManager().getStorage())
+                            ? new CradleReaderState(commonFactory.getCradleManager().getStorage(),
+                            streamId -> commonFactory.newMessageIDBuilder().getBookName())
                             : new InMemoryReaderState(),
+                    streamId -> commonFactory.newMessageIDBuilder().build(),
                     (streamId, builders) -> publishMessages(rawMessageBatchRouter, streamId, builders),
                     (streamId, message, ex) -> publishErrorEvent(eventBatchRouter, streamId, message, ex, rootId),
                     (streamId, path, e) -> publishSourceCorruptedEvent(eventBatchRouter, path, streamId, e, rootId)
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java b/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
index 124cbb5..04c997b 100644
--- a/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
+++ b/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
@@ -16,6 +16,7 @@
 
 package com.exactpro.th2.readlog.impl;
 
+import com.exactpro.th2.common.grpc.MessageID;
 import com.exactpro.th2.read.file.common.AbstractFileReader;
 import com.exactpro.th2.read.file.common.DirectoryChecker;
 import com.exactpro.th2.read.file.common.FileSourceWrapper;
@@ -36,6 +37,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Comparator;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static java.util.Comparator.comparing;
@@ -45,6 +47,7 @@ public class LogFileReader {
     public static AbstractFileReader<LineNumberReader> getLogFileReader(
             LogReaderConfiguration configuration,
             ReaderState readerState,
+            Function<StreamId, MessageID> initialMessageId,
             ForOnStreamData forStream,
             ForOnError forError,
             ForOnSourceCorrupted forCorrupted
@@ -55,6 +58,7 @@ public static AbstractFileReader<LineNumberReader> getLogFileReader(
                     new RegexpContentParser(new RegexLogParser(configuration.getAliases())),
                     new MovedFileTracker(configuration.getLogDirectory()),
                     readerState,
+                    initialMessageId::apply,
                     LogFileReader::createSource
             )
                     .readFileImmediately()
diff --git a/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt b/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
index fe81a66..7b13472 100644
--- a/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
+++ b/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
@@ -16,7 +16,10 @@
 
 package com.exactpro.th2.readlog.impl
 
+import com.exactpro.cradle.BookId
 import com.exactpro.cradle.CradleStorage
+import com.exactpro.cradle.Order
+import com.exactpro.cradle.messages.MessageFilter
 import com.exactpro.cradle.messages.StoredMessageId
 import com.exactpro.th2.common.grpc.RawMessage
 import com.exactpro.th2.common.util.toCradleDirection
@@ -25,32 +28,33 @@ import com.exactpro.th2.read.file.common.state.ReaderState
 import com.exactpro.th2.read.file.common.state.StreamData
 import com.exactpro.th2.read.file.common.state.impl.InMemoryReaderState
 import com.google.protobuf.ByteString
+import com.google.protobuf.UnsafeByteOperations
+import java.time.Instant
 
 class CradleReaderState private constructor(
     private val cradleStorage: CradleStorage,
     private val delegate: ReaderState,
+    private val bookSupplier: (StreamId) -> String,
 ): ReaderState by delegate {
-    constructor(cradleStorage: CradleStorage) : this(cradleStorage, InMemoryReaderState())
+    constructor(cradleStorage: CradleStorage, bookSupplier: (StreamId) -> String)
+            : this(cradleStorage, InMemoryReaderState(), bookSupplier)
 
     override fun get(streamId: StreamId): StreamData? {
-        return delegate[streamId] ?: cradleStorage.getLastMessageIndex(
-            streamId.sessionAlias,
-            streamId.direction.toCradleDirection(),
-        ).let { lastSequence ->
-            when {
-                lastSequence < 0 -> null
-                else -> cradleStorage.getMessage(StoredMessageId(
-                    streamId.sessionAlias,
-                    streamId.direction.toCradleDirection(),
-                    lastSequence
-                )).run {
-                    StreamData(
-                        timestamp,
-                        index,
-                        content?.let(RawMessage::parseFrom)?.body ?: ByteString.EMPTY, // TODO: change when migrate to cradle storing only body in the content
-                    )
-                }
-            }
+        return delegate[streamId] ?: cradleStorage.getMessages(
+            MessageFilter.builder()
+                .sessionAlias(streamId.sessionAlias)
+                .direction(streamId.direction.toCradleDirection())
+                .bookId(BookId(bookSupplier(streamId)))
+                .timestampTo().isLessThanOrEqualTo(Instant.now())
+                .limit(1)
+                .order(Order.REVERSE)
+                .build()
+        ).asSequence().firstOrNull()?.run {
+            StreamData(
+                timestamp,
+                sequence,
+                content?.let(UnsafeByteOperations::unsafeWrap) ?: ByteString.EMPTY,
+            )
         }
     }
 }
\ No newline at end of file

From edaf45e6f28bd6682c397e01d27fde19132b62ad Mon Sep 17 00:00:00 2001
From: Oleg <oleg.smirnov@exactprosystems.com>
Date: Tue, 17 Jan 2023 13:27:37 +0400
Subject: [PATCH 08/20] [TH2-2150] Update readme

---
 README.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/README.md b/README.md
index e271382..9069816 100644
--- a/README.md
+++ b/README.md
@@ -178,9 +178,8 @@ Output: 8=FIXT.1.1\u00019=66\u000135=A\u000134=1\u000149=NFT2_FIX1\u000156=FGW\u
 
 ### 4.0.0
 
-+ Updated common to 4.0.0
++ Updated common to 5.0.0
     + Migration to books-pages
-+ Updated bom to 3.1.0
 + Updated read-file-common-core to 2.0.0
 
 

From 59adaf1ab988f4f10fe3e7f6b7d9083cbabbec60 Mon Sep 17 00:00:00 2001
From: Denis Plotnikov <denis.plotnikov@exactprosystems.com>
Date: Fri, 3 Feb 2023 19:26:14 +0400
Subject: [PATCH 09/20] update common

---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 49a4bf6..8e6e9e4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -50,7 +50,7 @@ repositories {
 
 dependencies {
     api platform('com.exactpro.th2:bom:4.1.0')
-    api 'com.exactpro.th2:common:5.0.0-dev-version-5-3838510969-SNAPSHOT'
+    api 'com.exactpro.th2:common:5.1.0-dev-version-5-4085018593-8adee33-SNAPSHOT'
     api 'com.exactpro.th2:read-file-common-core:2.0.0-th2-4665-SNAPSHOT'
 
     implementation('com.opencsv:opencsv:5.7.0') {

From 76199907f8d40163e074713c5ea94f8dee3f2991 Mon Sep 17 00:00:00 2001
From: Fiodar Rekish <38082705+Xanclry@users.noreply.github.com>
Date: Tue, 7 Mar 2023 14:37:58 +0400
Subject: [PATCH 10/20] version bump (#44)

---
 README.md         | 6 +++++-
 gradle.properties | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index d993643..1f55690 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Log Reader User Manual 4.0.0
+# Log Reader User Manual 4.0.1
 
 ## Document Information
 
@@ -176,6 +176,10 @@ Output: 8=FIXT.1.1\u00019=66\u000135=A\u000134=1\u000149=NFT2_FIX1\u000156=FGW\u
 
 ## Changes
 
+### 4.0.1
+
++ Added dev-release GitHub workflow
+
 ### 4.0.0
 
 + Updated common to 5.0.0
diff --git a/gradle.properties b/gradle.properties
index 095a815..b5146ef 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,2 +1,2 @@
-release_version=4.0.0
+release_version=4.0.1
 vcs_url=https://github.com/th2-net/th2-read-log

From 62cfadf0e33f419166187221f558e47f78218526 Mon Sep 17 00:00:00 2001
From: Fiodar Rekish <38082705+Xanclry@users.noreply.github.com>
Date: Tue, 7 Mar 2023 17:45:53 +0400
Subject: [PATCH 11/20] set exact version of read-file-common-core (#45)

---
 build.gradle      | 2 +-
 gradle.properties | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index 892d94f..18f0a59 100644
--- a/build.gradle
+++ b/build.gradle
@@ -51,7 +51,7 @@ repositories {
 dependencies {
     api platform('com.exactpro.th2:bom:4.2.0')
     api 'com.exactpro.th2:common:5.2.0-dev'
-    api 'com.exactpro.th2:read-file-common-core:2.0.0-dev-version-2+'
+    api 'com.exactpro.th2:read-file-common-core:2.0.0-dev-version-2-4315008828-1ed0f94-SNAPSHOT'
 
     implementation('com.opencsv:opencsv:5.7.0') {
         because("we need to write a correct CSV in case of free pattern is used")
diff --git a/gradle.properties b/gradle.properties
index b5146ef..524ce60 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,2 +1,2 @@
 release_version=4.0.1
-vcs_url=https://github.com/th2-net/th2-read-log
+vcs_url=https://github.com/th2-net/th2-read-log
\ No newline at end of file

From a637088dc26221344e0895a6b557e688e5f08c30 Mon Sep 17 00:00:00 2001
From: Oleg <oleg.smirnov@exactprosystems.com>
Date: Tue, 11 Apr 2023 16:13:15 +0400
Subject: [PATCH 12/20] Use latest release for common lib

---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 18f0a59..2a0061e 100644
--- a/build.gradle
+++ b/build.gradle
@@ -50,7 +50,7 @@ repositories {
 
 dependencies {
     api platform('com.exactpro.th2:bom:4.2.0')
-    api 'com.exactpro.th2:common:5.2.0-dev'
+    api 'com.exactpro.th2:common:5.2.0'
     api 'com.exactpro.th2:read-file-common-core:2.0.0-dev-version-2-4315008828-1ed0f94-SNAPSHOT'
 
     implementation('com.opencsv:opencsv:5.7.0') {

From 0ea5317d2d35015d6dd4bb87bc2136293afb088f Mon Sep 17 00:00:00 2001
From: Oleg <oleg.smirnov@exactprosystems.com>
Date: Mon, 12 Jun 2023 12:31:16 +0400
Subject: [PATCH 13/20] Add logging example into readme

---
 README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/README.md b/README.md
index 1f55690..b73bd93 100644
--- a/README.md
+++ b/README.md
@@ -154,6 +154,54 @@ spec:
       attributes: ['raw', 'publish', 'store']
 ```
 
+##### Logging
+
+This block describes which classes can be used to get more information about read-log work.
+You can tweak their logging level and you will see extra information in the read-log logs.
+
+###### Log4j2 config
+
+Here is an example of log4j2 logging configuration that can be set in **loggingConfig** parameter for the component:
+
+```yaml
+loggingConfig: |
+  name=Th2Logger
+  # Console appender configuration
+  appender.console.type=Console
+  appender.console.name=consoleLogger
+  appender.console.layout.type=PatternLayout
+  appender.console.layout.pattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-6p [%-15t] %c - %m%n
+  logger.th2.name=com.exactpro.th2
+  logger.th2.level=TRACE
+  rootLogger.level=INFO
+  rootLogger.appenderRef.stdout.ref=consoleLogger
+```
+
+If you need to add extra logging you can do it by adding a line in the following format:
+```properties
+logger.<logger_name>.name=<full_class_name>
+logger.<logger_name>.level=<level>
+```
+
+**NOTE: the _logger_name_ must be unique**.
+
+###### Classes
+
+**com.exactpro.th2.readlog.RegexLogParser**
+
+You can use this class to see how the log line is parsed by the read-log.
+Use TRACE level to get the information.
+
+**com.exactpro.th2.readlog.impl.RegexpContentParser**
+
+You can use this class to see the resulted lines produced by **_RegexLogParser_**.
+Use TRACE level to get the information.
+
+**com.exactpro.th2.read.file.common.AbstractFileReader**
+
+You can use this class to get information about how the reader processes files.
+Use either DEBUG or TRACE level
+
 ### Examples
 
 #### Example 1

From dc456d04b99842130374af19348b78c2a02c9ed2 Mon Sep 17 00:00:00 2001
From: Oleg <oleg.smirnov@exactprosystems.com>
Date: Tue, 13 Jun 2023 18:25:21 +0400
Subject: [PATCH 14/20] [TH2-4951] Set direction in the message. Group by
 session alias only

---
 build.gradle                                  |  2 +-
 .../com/exactpro/th2/readlog/LogData.java     | 11 +++++++
 .../exactpro/th2/readlog/RegexLogParser.java  | 25 +++++++++------
 .../th2/readlog/impl/LogFileReader.java       |  6 ++--
 .../th2/readlog/impl/RegexpContentParser.java |  6 +++-
 .../th2/readlog/impl/CradleReaderState.kt     | 11 ++++---
 .../exactpro/th2/readlog/TestLogParser.java   | 31 +++++++++----------
 .../readlog/TestRegexLogParserJoining.java    | 12 +++----
 8 files changed, 62 insertions(+), 42 deletions(-)

diff --git a/build.gradle b/build.gradle
index 2a0061e..214e4ed 100644
--- a/build.gradle
+++ b/build.gradle
@@ -51,7 +51,7 @@ repositories {
 dependencies {
     api platform('com.exactpro.th2:bom:4.2.0')
     api 'com.exactpro.th2:common:5.2.0'
-    api 'com.exactpro.th2:read-file-common-core:2.0.0-dev-version-2-4315008828-1ed0f94-SNAPSHOT'
+    api 'com.exactpro.th2:read-file-common-core:2.0.0-th2-4951-5256417699-8614fb0-SNAPSHOT'
 
     implementation('com.opencsv:opencsv:5.7.0') {
         because("we need to write a correct CSV in case of free pattern is used")
diff --git a/src/main/java/com/exactpro/th2/readlog/LogData.java b/src/main/java/com/exactpro/th2/readlog/LogData.java
index 839d7e4..137ae37 100644
--- a/src/main/java/com/exactpro/th2/readlog/LogData.java
+++ b/src/main/java/com/exactpro/th2/readlog/LogData.java
@@ -15,6 +15,8 @@
  */
 package com.exactpro.th2.readlog;
 
+import com.exactpro.th2.common.grpc.Direction;
+
 import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
@@ -27,6 +29,7 @@ public final class LogData {
     private List<String> body;
     private String rawTimestamp;
     private Instant parsedTimestamp;
+    private Direction direction;
 
     public LogData() {
         this(null);
@@ -61,6 +64,14 @@ public void setParsedTimestamp(Instant localDateTime) {
         this.parsedTimestamp = localDateTime;
     }
 
+    public Direction getDirection() {
+        return direction;
+    }
+
+    public void setDirection(Direction direction) {
+        this.direction = direction;
+    }
+
     private void initIfNeeded() {
         if (body == null) {
             body = new ArrayList<>();
diff --git a/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java b/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java
index 85b870f..d0f2f1c 100644
--- a/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java
+++ b/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java
@@ -25,12 +25,14 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import com.exactpro.th2.common.grpc.Direction;
 import com.exactpro.th2.read.file.common.StreamId;
 import com.exactpro.th2.readlog.cfg.AliasConfiguration;
 import com.opencsv.CSVWriter;
@@ -65,17 +67,22 @@ public LogData parse(StreamId streamId, String raw) {
             throw new IllegalArgumentException("Unknown alias '" + sessionAlias +"'. No configuration found" );
         }
 
-        LogData resultData = new LogData();
-
-        Pattern directionPattern = Objects.requireNonNull(configuration.getDirectionToPattern().get(streamId.getDirection()),
-                () -> "Pattern for direction " + streamId.getDirection() + " and session " + sessionAlias);
-        Matcher matcher = directionPattern.matcher(raw);
-        // check if the a string matches the direction from streamId
-        // skip line if it is not ours direction
-        if (!matcher.find()) {
-            return resultData;
+        Direction direction = null;
+        for (Entry<Direction, Pattern> entry : configuration.getDirectionToPattern().entrySet()) {
+            if (entry.getValue().matcher(raw).find()) {
+                direction = entry.getKey();
+                break;
+            }
+        }
+        // check whether the line matches any direction regex
+        // if not it is not our line
+        if (direction == null) {
+            return LogData.EMPTY;
         }
 
+        LogData resultData = new LogData();
+        resultData.setDirection(direction);
+
         List<Integer> regexGroups = configuration.getGroups();
         if (configuration.isJoinGroups()) {
             parseBodyJoined(raw, configuration, resultData);
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java b/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
index 04c997b..a83537a 100644
--- a/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
+++ b/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
@@ -76,10 +76,8 @@ private static DirectoryChecker getDirectoryChecker(LogReaderConfiguration confi
                 configuration.getLogDirectory(),
                 (Path path) -> configuration.getAliases().entrySet().stream()
                         .filter(entry -> entry.getValue().getPathFilter().matcher(path.getFileName().toString()).matches())
-                        .flatMap(entry -> entry.getValue().getDirectionToPattern()
-                                .keySet().stream()
-                                .map(direction -> new StreamId(entry.getKey(), direction))
-                        ).collect(Collectors.toSet()),
+                        .map(entry -> new StreamId(entry.getKey()))
+                        .collect(Collectors.toSet()),
                 files -> files.sort(pathComparator),
                 path -> true
         );
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java b/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java
index 60c655b..494a10e 100644
--- a/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java
+++ b/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java
@@ -37,12 +37,14 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static java.util.Objects.requireNonNull;
+
 public class RegexpContentParser extends LineParser {
     private static final Logger LOGGER = LoggerFactory.getLogger(RegexpContentParser.class);
     private final RegexLogParser parser;
 
     public RegexpContentParser(RegexLogParser parser) {
-        this.parser = Objects.requireNonNull(parser, "'Parser' parameter");
+        this.parser = requireNonNull(parser, "'Parser' parameter");
     }
 
     @Nonnull
@@ -59,6 +61,8 @@ protected List<RawMessage.Builder> lineToMessages(@Nonnull StreamId streamId, @N
     }
 
     private void setupMetadata(RawMessageMetadata.Builder builder, LogData logData) {
+        builder.getIdBuilder().setDirection(requireNonNull(logData.getDirection(),
+                "direction is not set"));
         if (logData.getParsedTimestamp() != null) {
             builder.getIdBuilder().setTimestamp(MessageUtils.toTimestamp(logData.getParsedTimestamp()));
         }
diff --git a/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt b/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
index 7b13472..2b31972 100644
--- a/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
+++ b/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
@@ -19,6 +19,7 @@ package com.exactpro.th2.readlog.impl
 import com.exactpro.cradle.BookId
 import com.exactpro.cradle.CradleStorage
 import com.exactpro.cradle.Order
+import com.exactpro.cradle.messages.GroupedMessageFilter
 import com.exactpro.cradle.messages.MessageFilter
 import com.exactpro.cradle.messages.StoredMessageId
 import com.exactpro.th2.common.grpc.RawMessage
@@ -30,6 +31,7 @@ import com.exactpro.th2.read.file.common.state.impl.InMemoryReaderState
 import com.google.protobuf.ByteString
 import com.google.protobuf.UnsafeByteOperations
 import java.time.Instant
+import java.time.temporal.ChronoUnit
 
 class CradleReaderState private constructor(
     private val cradleStorage: CradleStorage,
@@ -40,16 +42,15 @@ class CradleReaderState private constructor(
             : this(cradleStorage, InMemoryReaderState(), bookSupplier)
 
     override fun get(streamId: StreamId): StreamData? {
-        return delegate[streamId] ?: cradleStorage.getMessages(
-            MessageFilter.builder()
-                .sessionAlias(streamId.sessionAlias)
-                .direction(streamId.direction.toCradleDirection())
+        return delegate[streamId] ?: cradleStorage.getGroupedMessageBatches(
+            GroupedMessageFilter.builder()
+                .groupName(streamId.sessionAlias)
                 .bookId(BookId(bookSupplier(streamId)))
                 .timestampTo().isLessThanOrEqualTo(Instant.now())
                 .limit(1)
                 .order(Order.REVERSE)
                 .build()
-        ).asSequence().firstOrNull()?.run {
+        ).asSequence().firstOrNull()?.lastMessage?.run {
             StreamData(
                 timestamp,
                 sequence,
diff --git a/src/test/java/com/exactpro/th2/readlog/TestLogParser.java b/src/test/java/com/exactpro/th2/readlog/TestLogParser.java
index 652dda3..4bcb646 100644
--- a/src/test/java/com/exactpro/th2/readlog/TestLogParser.java
+++ b/src/test/java/com/exactpro/th2/readlog/TestLogParser.java
@@ -32,8 +32,8 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.EnumSource;
 import org.junit.jupiter.params.provider.MethodSource;
-import org.junit.jupiter.params.provider.ValueSource;
 
 import static org.junit.jupiter.api.Assertions.assertAll;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -46,7 +46,8 @@ public class TestLogParser {
     static final String TEST_MESSAGE_ALIAS = "tma";
     static final String TEST_MESSAGE_ALIAS_WRONG_TIMESTAMP_FORMAT = "tma_wrong_format";
     static final String TEST_MESSAGE_ALIAS_WRONG_TIMESTAMP_PATTERN = "tma_wrong_pattern";
-    static final String RAW_LOG = "2021-03-23 13:21:37.991337479 QUICK.TEST INFO quicktest (Test.cpp:99) - incoming fix message fix NewOrderSingle={ AuthenticationBlock={ UserID=\"qwrqwrq\" SessionKey=123456 } Header={ MsgTime=2021-Mar-21 21:21:21.210000000 CreationTime=2021-Mar-21 21:21:21.210000000 } NewOrder={ InstrumentBlock={ InstrSymbol=\"TEST_SYMBOL\" SecurityID=\"212121\" SecurityIDSource=TestSource SecurityExchange=\"test\" }}}";
+    static final String RAW_LOG_IN = "2021-03-23 13:21:37.991337479 QUICK.TEST INFO quicktest (Test.cpp:99) - incoming fix message fix NewOrderSingle={ AuthenticationBlock={ UserID=\"qwrqwrq\" SessionKey=123456 } Header={ MsgTime=2021-Mar-21 21:21:21.210000000 CreationTime=2021-Mar-21 21:21:21.210000000 } NewOrder={ InstrumentBlock={ InstrSymbol=\"TEST_SYMBOL\" SecurityID=\"212121\" SecurityIDSource=TestSource SecurityExchange=\"test\" }}}";
+    static final String RAW_LOG_OUT = "2021-03-23 13:21:37.991337479 QUICK.TEST INFO quicktest (Test.cpp:99) - outgoing fix message fix NewOrderSingle={ AuthenticationBlock={ UserID=\"qwrqwrq\" SessionKey=123456 } Header={ MsgTime=2021-Mar-21 21:21:21.210000000 CreationTime=2021-Mar-21 21:21:21.210000000 } NewOrder={ InstrumentBlock={ InstrSymbol=\"TEST_SYMBOL\" SecurityID=\"212121\" SecurityIDSource=TestSource SecurityExchange=\"test\" }}}";
 
     @ParameterizedTest(name = "Has message: {1}, Skips before: {0}")
     @MethodSource("skipMessageParams")
@@ -65,7 +66,10 @@ void skipMessageByTimestamp(Instant skipBefore, boolean hasMessages) {
                 "test", cfg
         ));
 
-        LogData data = parser.parse(new StreamId("test", Direction.FIRST), "2021-03-23 13:21:37.991337479 some data");
+        LogData data = parser.parse(new StreamId("test"), "2021-03-23 13:21:37.991337479 some data");
+        if (hasMessages) {
+            assertEquals(Direction.FIRST, data.getDirection(), "unexpected direction");
+        }
         assertEquals(hasMessages, !data.getBody().isEmpty(), () -> "unexpected data " + data.getBody());
         if (hasMessages) {
             assertEquals(List.of("2021-03-23 13:21:37.991337479 some data"), data.getBody());
@@ -79,23 +83,18 @@ static List<Arguments> skipMessageParams() {
         );
     }
 
-    @Test
-    void parser() {
+    @ParameterizedTest
+    @EnumSource(value = Direction.class, mode = EnumSource.Mode.EXCLUDE, names = "UNRECOGNIZED")
+    void parser(Direction direction) {
         RegexLogParser logParser = new RegexLogParser(getConfiguration());
-        LogData data = logParser.parse(new StreamId(TEST_MESSAGE_ALIAS, Direction.FIRST), RAW_LOG);
+        LogData data = logParser.parse(new StreamId(TEST_MESSAGE_ALIAS), direction == Direction.FIRST ? RAW_LOG_IN : RAW_LOG_OUT);
+        assertEquals(direction, data.getDirection(), "unexpected direction");
         assertEquals(1, data.getBody().size());
         assertEquals("NewOrderSingle={ AuthenticationBlock={ UserID=\"qwrqwrq\" SessionKey=123456 } Header={ MsgTime=2021-Mar-21 21:21:21.210000000 CreationTime=2021-Mar-21 21:21:21.210000000 } NewOrder={ InstrumentBlock={ InstrSymbol=\"TEST_SYMBOL\" SecurityID=\"212121\" SecurityIDSource=TestSource SecurityExchange=\"test\" }}}", data.getBody().get(0));
         assertEquals("2021-03-23 13:21:37.991337479", data.getRawTimestamp());
         assertEquals(Instant.parse("2021-03-23T13:21:37.991337479Z"), data.getParsedTimestamp());
     }
 
-    @Test
-    void skipIfDirectionPatternDoesNotMatch() {
-        RegexLogParser logParser = new RegexLogParser(getConfiguration());
-        LogData data = logParser.parse(new StreamId(TEST_MESSAGE_ALIAS, Direction.SECOND), RAW_LOG);
-        assertTrue(data.getBody().isEmpty(), () -> "Unexpected data read: " + data.getBody());
-    }
-
     @Test
     void parserErrors() {
         RegexLogParser logParser = new RegexLogParser(getConfiguration());
@@ -103,21 +102,21 @@ void parserErrors() {
         assertAll(
                 () -> {
                     var ex = assertThrows(IllegalStateException.class,
-                            () -> logParser.parse(new StreamId(TEST_MESSAGE_ALIAS_WRONG_TIMESTAMP_FORMAT, Direction.FIRST), RAW_LOG));
+                            () -> logParser.parse(new StreamId(TEST_MESSAGE_ALIAS_WRONG_TIMESTAMP_FORMAT), RAW_LOG_IN));
                     Assertions.assertTrue(
                             ex.getMessage().startsWith("The timestamp '2021-03-23 13:21:37.991337479' cannot be parsed"),
                             () -> "Actual error: " + ex.getMessage());
                 },
                 () -> {
                     var ex = assertThrows(IllegalStateException.class,
-                            () -> logParser.parse(new StreamId(TEST_MESSAGE_ALIAS_WRONG_TIMESTAMP_PATTERN, Direction.FIRST), RAW_LOG));
+                            () -> logParser.parse(new StreamId(TEST_MESSAGE_ALIAS_WRONG_TIMESTAMP_PATTERN), RAW_LOG_IN));
                     Assertions.assertTrue(
                             ex.getMessage().startsWith("The pattern '3012.*' cannot extract the timestamp from the string"),
                             () -> "Actual error: " + ex.getMessage());
                 },
                 () -> {
                     var ex = assertThrows(IllegalArgumentException.class,
-                            () -> logParser.parse(new StreamId("wrong_alias", Direction.FIRST), RAW_LOG));
+                            () -> logParser.parse(new StreamId("wrong_alias"), RAW_LOG_IN));
                     Assertions.assertTrue(
                             ex.getMessage().startsWith("Unknown alias 'wrong_alias'. No configuration found"),
                             () -> "Actual error: " + ex.getMessage());
diff --git a/src/test/java/com/exactpro/th2/readlog/TestRegexLogParserJoining.java b/src/test/java/com/exactpro/th2/readlog/TestRegexLogParserJoining.java
index ddac41d..abcc08f 100644
--- a/src/test/java/com/exactpro/th2/readlog/TestRegexLogParserJoining.java
+++ b/src/test/java/com/exactpro/th2/readlog/TestRegexLogParserJoining.java
@@ -43,7 +43,7 @@ void joinsIfOneMatchFound() {
         ));
         RegexLogParser parser = new RegexLogParser(Map.of("test", configuration));
 
-        LogData data = parser.parse(new StreamId("test", Direction.FIRST), "this a test string, 123 and no more");
+        LogData data = parser.parse(new StreamId("test"), "this a test string, 123 and no more");
         List<String> body = data.getBody();
         Assertions.assertEquals(1, body.size(), () -> "Unexpected strings: " + body);
         Assertions.assertEquals(
@@ -67,7 +67,7 @@ void doesNotTryToSubstituteInResultString() {
         ));
         RegexLogParser parser = new RegexLogParser(Map.of("test", configuration));
 
-        LogData data = parser.parse(new StreamId("test", Direction.FIRST), "[test] should not try to process ${3} and ${variable}");
+        LogData data = parser.parse(new StreamId("test"), "[test] should not try to process ${3} and ${variable}");
         List<String> body = data.getBody();
         Assertions.assertEquals(1, body.size(), () -> "Unexpected strings: " + body);
         Assertions.assertEquals(
@@ -92,7 +92,7 @@ void joinsIfManyMatchesFound() {
         ));
         RegexLogParser parser = new RegexLogParser(Map.of("test", configuration));
 
-        LogData data = parser.parse(new StreamId("test", Direction.FIRST), "A, 42; B, 53");
+        LogData data = parser.parse(new StreamId("test"), "A, 42; B, 53");
         List<String> body = data.getBody();
         Assertions.assertEquals(1, body.size(), () -> "Unexpected strings: " + body);
         Assertions.assertEquals(
@@ -119,7 +119,7 @@ void usesCorrectDelimiter() {
         ));
         RegexLogParser parser = new RegexLogParser(Map.of("test", configuration));
 
-        LogData data = parser.parse(new StreamId("test", Direction.FIRST), "A, 42");
+        LogData data = parser.parse(new StreamId("test"), "A, 42");
         List<String> body = data.getBody();
         Assertions.assertEquals(1, body.size(), () -> "Unexpected strings: " + body);
         Assertions.assertEquals(
@@ -145,7 +145,7 @@ void usesConstantsFromMap() {
         ));
         RegexLogParser parser = new RegexLogParser(Map.of("test", configuration));
 
-        LogData data = parser.parse(new StreamId("test", Direction.FIRST), "A, 42; B, 53");
+        LogData data = parser.parse(new StreamId("test"), "A, 42; B, 53");
         List<String> body = data.getBody();
         Assertions.assertEquals(1, body.size(), () -> "Unexpected strings: " + body);
         Assertions.assertEquals(
@@ -171,7 +171,7 @@ void correctlyAcceptsGroupsByName() {
         ));
         RegexLogParser parser = new RegexLogParser(Map.of("test", configuration));
 
-        LogData data = parser.parse(new StreamId("test", Direction.FIRST), "A, 42; B, 53");
+        LogData data = parser.parse(new StreamId("test"), "A, 42; B, 53");
         List<String> body = data.getBody();
         Assertions.assertEquals(1, body.size(), () -> "Unexpected strings: " + body);
         Assertions.assertEquals(

From 70c22d2f709deb643772dc44fa59cd364ee8aa94 Mon Sep 17 00:00:00 2001
From: Oleg Smelov <45400511+lumber1000@users.noreply.github.com>
Date: Tue, 21 Nov 2023 10:48:26 +0400
Subject: [PATCH 15/20] [TH2-4954] th2 transport protocol support (#48)

* th2 transport protocol support

* alias group equals to session alias

* dependencies

* [th2-4954] Updated dependencies

* dependencies, dev release workflow, README.md

---------

Co-authored-by: Oleg Smelov <oleg.smelov@exactprosystems.com>
Co-authored-by: nikita.smirnov <nikita.smirnov@exactprosystems.com>
---
 ...ease-java-publish-sonatype-and-docker.yml} |  13 +-
 README.md                                     |  17 +-
 build.gradle                                  | 238 ++----------------
 gradle.properties                             |   6 +-
 .../com/exactpro/th2/readlog/LogData.java     |   9 +-
 .../java/com/exactpro/th2/readlog/Main.java   |  93 +++++--
 .../exactpro/th2/readlog/RegexLogParser.java  |  12 +-
 .../readlog/cfg/LogReaderConfiguration.java   |  15 +-
 .../th2/readlog/impl/LogFileReader.java       |  51 +++-
 ...ser.java => ProtoRegexpContentParser.java} |  25 +-
 .../impl/TransportRegexpContentParser.java    |  67 +++++
 ...eamData.java => ProtoForOnStreamData.java} |  12 +-
 .../lambdas/TransportForOnStreamData.java     |  27 ++
 .../th2/readlog/impl/CradleReaderState.kt     |  29 ++-
 .../exactpro/th2/readlog/TestLogParser.java   |  10 +-
 15 files changed, 309 insertions(+), 315 deletions(-)
 rename .github/workflows/{dev-release-docker-publish.yml => dev-release-java-publish-sonatype-and-docker.yml} (73%)
 rename src/main/java/com/exactpro/th2/readlog/impl/{RegexpContentParser.java => ProtoRegexpContentParser.java} (83%)
 create mode 100644 src/main/java/com/exactpro/th2/readlog/impl/TransportRegexpContentParser.java
 rename src/main/java/com/exactpro/th2/readlog/impl/lambdas/{ForOnStreamData.java => ProtoForOnStreamData.java} (70%)
 create mode 100644 src/main/java/com/exactpro/th2/readlog/impl/lambdas/TransportForOnStreamData.java

diff --git a/.github/workflows/dev-release-docker-publish.yml b/.github/workflows/dev-release-java-publish-sonatype-and-docker.yml
similarity index 73%
rename from .github/workflows/dev-release-docker-publish.yml
rename to .github/workflows/dev-release-java-publish-sonatype-and-docker.yml
index 1f878bf..d943c63 100644
--- a/.github/workflows/dev-release-docker-publish.yml
+++ b/.github/workflows/dev-release-java-publish-sonatype-and-docker.yml
@@ -1,18 +1,17 @@
-name: Build and publish Docker distributions to Github Container Registry ghcr.io
+name: Dev release build and publish - Docker and Sonatype
 
 on:
-  workflow_dispatch:
   push:
-    branches:
-      - dev-version-*
-    paths:
-      - gradle.properties
+    tags:
+      - \d+.\d+.\d+-dev
 
 jobs:
-  build-job:
+  build:
     uses: th2-net/.github/.github/workflows/compound-java.yml@main
     with:
       build-target: 'Docker,Sonatype'
+      runsOn: ubuntu-latest
+      gradleVersion: '7'
       docker-username: ${{ github.actor }}
       devRelease: true
     secrets:
diff --git a/README.md b/README.md
index b73bd93..e35c6bf 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Log Reader User Manual 4.0.1
+# Log Reader User Manual 4.1.0
 
 ## Document Information
 
@@ -28,6 +28,7 @@ spec:
   custom-config:
       logDirectory: "log/dir"
       syncWithCradle: true
+      useTransport: true
       aliases:
         A:
           regexp: ".*"
@@ -91,6 +92,7 @@ spec:
 
 + logDirectory - the directory to watch files
 + syncWithCradle - enables synchronization with Cradle for timestamps and sequences that correspond to the alias
++ useTransport - enables using th2 transport protocol (default value: `false`)
 + aliases - the mapping between alias and files that correspond to that alias
     + pathFilter - filter for files that correspond to that alias
     + regexp - the regular expression to extract data from the source lines
@@ -192,10 +194,11 @@ logger.<logger_name>.level=<level>
 You can use this class to see how the log line is parsed by the read-log.
 Use TRACE level to get the information.
 
-**com.exactpro.th2.readlog.impl.RegexpContentParser**
+**com.exactpro.th2.readlog.impl.ProtoRegexpContentParser**
+**com.exactpro.th2.readlog.impl.TransportRegexpContentParser**
 
-You can use this class to see the resulted lines produced by **_RegexLogParser_**.
-Use TRACE level to get the information.
+You can use these classes to see the resulted lines produced by **_RegexLogParser_**.
+Produce Proto or Transport messages, respectively. Use TRACE level to get the information.
 
 **com.exactpro.th2.read.file.common.AbstractFileReader**
 
@@ -224,6 +227,10 @@ Output: 8=FIXT.1.1\u00019=66\u000135=A\u000134=1\u000149=NFT2_FIX1\u000156=FGW\u
 
 ## Changes
 
+### 4.1.0
+
++ Added support for th2 transport protocol
+
 ### 4.0.1
 
 + Added dev-release GitHub workflow
@@ -296,4 +303,4 @@ Output: 8=FIXT.1.1\u00019=66\u000135=A\u000134=1\u000149=NFT2_FIX1\u000156=FGW\u
 
 ### 3.0.0
 
-+ Migrate to a common read-core
++ Migrate to a common read-core
\ No newline at end of file
diff --git a/build.gradle b/build.gradle
index 214e4ed..fccc4b1 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,225 +1,42 @@
-/*
- *  Copyright 2020-2021 Exactpro (Exactpro Systems Limited)
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-plugins {
-    id 'java'
-    id 'java-library'
-    id 'application'
-    id 'maven-publish'
-    id 'signing'
-    id 'com.palantir.docker' version '0.25.0'
-    id 'org.jetbrains.kotlin.jvm' version '1.6.21'
-    id "org.owasp.dependencycheck" version "8.1.2"
-    id "io.github.gradle-nexus.publish-plugin" version "1.0.0"
-
-}
-
-group 'com.exactpro.th2'
-version release_version
-
-repositories {
-    mavenCentral()
-    maven {
-        name 'Sonatype_snapshots'
-        url 'https://s01.oss.sonatype.org/content/repositories/snapshots/'
-    }
-    maven {
-        name 'Sonatype_releases'
-        url 'https://s01.oss.sonatype.org/content/repositories/releases/'
+buildscript {
+    repositories {
+        gradlePluginPortal()
+        maven {
+            url = "https://s01.oss.sonatype.org/content/repositories/snapshots/"
+        }
     }
-    mavenLocal()
-
-    configurations.all {
-        resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
-        resolutionStrategy.cacheDynamicVersionsFor 0, 'seconds'
+    dependencies {
+        classpath "com.exactpro.th2:th2-gradle-plugin:0.0.1-dev-5915968839-41381e5-SNAPSHOT"
     }
 }
 
+apply plugin: "com.exactpro.th2.common-conventions"
+apply plugin: "com.exactpro.th2.docker-conventions"
+apply plugin: 'kotlin-kapt'
+
 dependencies {
-    api platform('com.exactpro.th2:bom:4.2.0')
-    api 'com.exactpro.th2:common:5.2.0'
-    api 'com.exactpro.th2:read-file-common-core:2.0.0-th2-4951-5256417699-8614fb0-SNAPSHOT'
+    api platform("com.exactpro.th2:bom:4.5.0")
+    implementation "com.exactpro.th2:common:5.4.2-dev"
+    implementation "com.exactpro.th2:common-utils:2.2.0-dev"
+    implementation "com.exactpro.th2:read-file-common-core:3.0.0-dev"
 
-    implementation('com.opencsv:opencsv:5.7.0') {
+    implementation ("com.opencsv:opencsv:5.8") {
         because("we need to write a correct CSV in case of free pattern is used")
     }
 
-    implementation 'javax.annotation:javax.annotation-api:1.3.2'
+    implementation "javax.annotation:javax.annotation-api:1.3.2"
     implementation "org.jetbrains.kotlin:kotlin-stdlib"
 
-    implementation "org.slf4j:slf4j-api"
-
-    implementation 'com.fasterxml.jackson.core:jackson-databind'
-    implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
-    implementation 'com.fasterxml.jackson.module:jackson-module-kotlin'
-
-    testImplementation 'org.junit.jupiter:junit-jupiter:5.9.0'
-    testImplementation 'org.mockito:mockito-core:3.6.0'
-}
-
-test {
-    useJUnitPlatform()
-    testLogging.showStandardStreams = true
-}
-
-jar {
-    manifest {
-        attributes(
-                'Created-By': "${System.getProperty('java.version')} (${System.getProperty('java.vendor')})",
-                'Specification-Title': '',
-                'Specification-Vendor': 'Exactpro Systems LLC',
-                'Implementation-Title': project.archivesBaseName,
-                'Implementation-Vendor': 'Exactpro Systems LLC',
-                'Implementation-Vendor-Id': 'com.exactpro',
-                'Implementation-Version': project.version
-        )
-    }
-}
-
-description = 'DataReaderClient'
-sourceCompatibility = JavaVersion.VERSION_11
-applicationName = 'service'
-
-tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile) {
-    kotlinOptions {
-        jvmTarget = "11"
-    }
-}
-
-distTar {
-    archiveFileName.set("${applicationName}.tar")
-}
-
-dockerPrepare {
-    dependsOn distTar
-}
-
-docker {
-    copySpec.from(tarTree("$buildDir/distributions/${applicationName}.tar"))
-}
-
-application {
-    mainClass.set("com.exactpro.th2.readlog.Main")
-}
-
-dependencyCheck {
-    formats=['SARIF']
-}
-
-dependencyLocking {
-    lockAllConfigurations()
-}
-
-
-java {
-    withJavadocJar()
-    withSourcesJar()
-}
-
-// conditionals for publications
-tasks.withType(PublishToMavenRepository) {
-    onlyIf {
-        (repository == publishing.repositories.nexusRepository &&
-                project.hasProperty('nexus_user') &&
-                project.hasProperty('nexus_password') &&
-                project.hasProperty('nexus_url')) ||
-                (repository == publishing.repositories.sonatype &&
-                        project.hasProperty('sonatypeUsername') &&
-                        project.hasProperty('sonatypePassword'))
-    }
-}
-tasks.withType(Sign) {
-    onlyIf { project.hasProperty('signingKey') &&
-            project.hasProperty('signingPassword')
-    }
-}
-// disable running task 'initializeSonatypeStagingRepository' on a gitlab
-tasks.whenTaskAdded {task ->
-    if(task.name.equals('initializeSonatypeStagingRepository') &&
-            !(project.hasProperty('sonatypeUsername') && project.hasProperty('sonatypePassword'))
-    ) {
-        task.enabled = false
-    }
-}
-
-publishing {
-    publications {
-        mavenJava(MavenPublication) {
-            from(components.java)
-            pom {
-                name = rootProject.name
-                packaging = 'jar'
-                description = rootProject.description
-                url = vcs_url
-                scm {
-                    url = vcs_url
-                }
-                licenses {
-                    license {
-                        name = 'The Apache License, Version 2.0'
-                        url = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
-                    }
-                }
-                developers {
-                    developer {
-                        id = 'developer'
-                        name = 'developer'
-                        email = 'developer@exactpro.com'
-                    }
-                }
-                scm {
-                    url = vcs_url
-                }
-            }
-        }
-    }
-    repositories {
-//Nexus repo to publish from gitlab
-        maven {
-            name = 'nexusRepository'
-            credentials {
-                username = project.findProperty('nexus_user')
-                password = project.findProperty('nexus_password')
-            }
-            url = project.findProperty('nexus_url')
-        }
-    }
-}
-
-nexusPublishing {
-    repositories {
-        sonatype {
-            nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/"))
-            snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/"))
-        }
-    }
-}
-
-
-signing {
-    def signingKey = findProperty("signingKey")
-    def signingPassword = findProperty("signingPassword")
-    useInMemoryPgpKeys(signingKey, signingPassword)
-    sign publishing.publications.mavenJava
+    implementation "com.fasterxml.jackson.core:jackson-databind"
+    implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310"
+    implementation "com.fasterxml.jackson.module:jackson-module-kotlin"
 }
 
 test {
     useJUnitPlatform {
         excludeTags('integration-test')
     }
+    testLogging.showStandardStreams = true
 }
 
 tasks.register('integrationTest', Test) {
@@ -229,13 +46,6 @@ tasks.register('integrationTest', Test) {
     }
 }
 
-dependencyCheck {
-    formats=['SARIF', 'JSON', 'HTML']
-    failBuildOnCVSS=5
-
-    analyzers {
-        assemblyEnabled = false
-        nugetconfEnabled = false
-        nodeEnabled = false
-    }
+dependencyLocking {
+    lockAllConfigurations()
 }
\ No newline at end of file
diff --git a/gradle.properties b/gradle.properties
index 524ce60..e874dbd 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,2 +1,4 @@
-release_version=4.0.1
-vcs_url=https://github.com/th2-net/th2-read-log
\ No newline at end of file
+release_version=4.1.0
+vcs_url=https://github.com/th2-net/th2-read-log
+description=DataReaderClient
+app_main_class=com.exactpro.th2.readlog.Main
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/LogData.java b/src/main/java/com/exactpro/th2/readlog/LogData.java
index 137ae37..8a1eae6 100644
--- a/src/main/java/com/exactpro/th2/readlog/LogData.java
+++ b/src/main/java/com/exactpro/th2/readlog/LogData.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2021 Exactpro (Exactpro Systems Limited)
+ * Copyright 2020-2023 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,13 +13,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package com.exactpro.th2.readlog;
 
-import com.exactpro.th2.common.grpc.Direction;
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.Direction;
 
 import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -77,4 +76,4 @@ private void initIfNeeded() {
             body = new ArrayList<>();
         }
     }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/Main.java b/src/main/java/com/exactpro/th2/readlog/Main.java
index ba95d7c..1b58ff9 100644
--- a/src/main/java/com/exactpro/th2/readlog/Main.java
+++ b/src/main/java/com/exactpro/th2/readlog/Main.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2020 Exactpro (Exactpro Systems Limited)
+ * Copyright 2020-2023 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,11 +21,13 @@
 import com.exactpro.th2.common.event.EventUtils;
 import com.exactpro.th2.common.grpc.EventBatch;
 import com.exactpro.th2.common.grpc.EventID;
-import com.exactpro.th2.common.grpc.RawMessage;
-import com.exactpro.th2.common.grpc.RawMessageBatch;
 import com.exactpro.th2.common.metrics.CommonMetrics;
 import com.exactpro.th2.common.schema.factory.CommonFactory;
 import com.exactpro.th2.common.schema.message.MessageRouter;
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.GroupBatch;
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.MessageGroup;
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.MessageId;
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.RawMessage;
 import com.exactpro.th2.read.file.common.AbstractFileReader;
 import com.exactpro.th2.read.file.common.StreamId;
 import com.exactpro.th2.read.file.common.state.impl.InMemoryReaderState;
@@ -41,6 +43,7 @@
 import java.io.LineNumberReader;
 import java.nio.file.Path;
 import java.util.ArrayDeque;
+import java.util.ArrayList;
 import java.util.Deque;
 import java.util.List;
 import java.util.concurrent.Executors;
@@ -65,7 +68,6 @@ public static void main(String[] args) {
         var boxBookName = commonFactory.getBoxConfiguration().getBookName();
         toDispose.add(commonFactory);
 
-        MessageRouter<RawMessageBatch> rawMessageBatchRouter = commonFactory.getMessageRouterRawBatch();
         MessageRouter<EventBatch> eventBatchRouter = commonFactory.getEventBatchRouter();
 
         LogReaderConfiguration configuration = commonFactory.getCustomConfiguration(LogReaderConfiguration.class, LogReaderConfiguration.MAPPER);
@@ -91,20 +93,41 @@ public static void main(String[] args) {
 
             CommonMetrics.READINESS_MONITOR.enable();
 
-            AbstractFileReader<LineNumberReader> reader
-                    = LogFileReader.getLogFileReader(
-                    configuration,
-                    configuration.isSyncWithCradle()
-                            ? new CradleReaderState(commonFactory.getCradleManager().getStorage(),
-                            streamId -> commonFactory.newMessageIDBuilder().getBookName())
-                            : new InMemoryReaderState(),
-                    streamId -> commonFactory.newMessageIDBuilder().build(),
-                    (streamId, builders) -> publishMessages(rawMessageBatchRouter, streamId, builders),
-                    (streamId, message, ex) -> publishErrorEvent(eventBatchRouter, streamId, message, ex, rootId),
-                    (streamId, path, e) -> publishSourceCorruptedEvent(eventBatchRouter, path, streamId, e, rootId)
-                    );
-
-            toDispose.add(reader);
+            final Runnable processUpdates;
+
+            if (configuration.isUseTransport()) {
+                AbstractFileReader<LineNumberReader, RawMessage.Builder, MessageId.Builder> reader
+                        = LogFileReader.getTransportLogFileReader(
+                        configuration,
+                        configuration.isSyncWithCradle()
+                                ? new CradleReaderState(commonFactory.getCradleManager().getStorage(),
+                                streamId -> commonFactory.newMessageIDBuilder().getBookName(),
+                                CradleReaderState.WRAP_TRANSPORT)
+                                : new InMemoryReaderState(),
+                        streamId -> MessageId.builder(),
+                        (streamId, builders) -> publishTransportMessages(commonFactory.getTransportGroupBatchRouter(), streamId, builders, boxBookName),
+                        (streamId, message, ex) -> publishErrorEvent(eventBatchRouter, streamId, message, ex, rootId),
+                        (streamId, path, e) -> publishSourceCorruptedEvent(eventBatchRouter, path, streamId, e, rootId)
+                );
+
+                processUpdates = reader::processUpdates;
+                toDispose.add(reader);
+            } else {
+                AbstractFileReader<LineNumberReader,com.exactpro.th2.common.grpc.RawMessage.Builder, com.exactpro.th2.common.grpc.MessageID> reader
+                        = LogFileReader.getProtoLogFileReader(
+                        configuration,
+                        configuration.isSyncWithCradle()
+                                ? new CradleReaderState(commonFactory.getCradleManager().getStorage(), streamId -> boxBookName, CradleReaderState.WRAP_PROTO)
+                                : new InMemoryReaderState(),
+                        streamId -> commonFactory.newMessageIDBuilder().build(),
+                        (streamId, builders) -> publishProtoMessages(commonFactory.getMessageRouterRawBatch(), streamId, builders),
+                        (streamId, message, ex) -> publishErrorEvent(eventBatchRouter, streamId, message, ex, rootId),
+                        (streamId, path, e) -> publishSourceCorruptedEvent(eventBatchRouter, path, streamId, e, rootId)
+                );
+
+                processUpdates = reader::processUpdates;
+                toDispose.add(reader);
+            }
 
             ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
             toDispose.add(() -> {
@@ -115,8 +138,7 @@ public static void main(String[] args) {
                 }
             });
 
-
-            ScheduledFuture<?> future = executorService.scheduleWithFixedDelay(reader::processUpdates, 0, configuration.getPullingInterval().toMillis(), TimeUnit.MILLISECONDS);
+            ScheduledFuture<?> future = executorService.scheduleWithFixedDelay(processUpdates, 0, configuration.getPullingInterval().toMillis(), TimeUnit.MILLISECONDS);
             awaitShutdown(lock, condition);
             future.cancel(true);
         } catch (IOException | InterruptedException e) {
@@ -157,19 +179,39 @@ private static Unit publishError(MessageRouter<EventBatch> eventBatchRouter, Str
     }
 
     @NotNull
-    private static Unit publishMessages(MessageRouter<RawMessageBatch> rawMessageBatchRouter, StreamId streamId, List<RawMessage.Builder> builders) {
+    private static Unit publishProtoMessages(MessageRouter<com.exactpro.th2.common.grpc.RawMessageBatch> rawMessageBatchRouter, StreamId streamId, List<? extends com.exactpro.th2.common.grpc.RawMessage.Builder> builders) {
         try {
-            RawMessageBatch.Builder builder = RawMessageBatch.newBuilder();
-            for (RawMessage.Builder msg : builders) {
+            com.exactpro.th2.common.grpc.RawMessageBatch.Builder builder = com.exactpro.th2.common.grpc.RawMessageBatch.newBuilder();
+            for (com.exactpro.th2.common.grpc.RawMessage.Builder msg : builders) {
                 builder.addMessages(msg);
             }
-            rawMessageBatchRouter.sendAll(builder.build());
+            rawMessageBatchRouter.sendAll(builder.build(), "raw");
         } catch (Exception e) {
             LOGGER.error("Cannot publish batch for {}", streamId, e);
         }
         return Unit.INSTANCE;
     }
 
+    @NotNull
+    private static Unit publishTransportMessages(MessageRouter<GroupBatch> rawMessageBatchRouter, StreamId streamId, List<? extends RawMessage.Builder> builders, String bookName) {
+        try {
+            // messages are grouped by session aliases
+            String sessionGroup = builders.get(0).idBuilder().getSessionAlias();
+
+            List<MessageGroup> groups = new ArrayList<>(builders.size());
+            for (RawMessage.Builder msgBuilder : builders) {
+                groups.add(new MessageGroup(List.of(msgBuilder.build())));
+            }
+
+            var batch = new GroupBatch(bookName, sessionGroup, groups);
+            rawMessageBatchRouter.sendAll(batch, "transport-group");
+        } catch (Exception e) {
+            LOGGER.error("Cannot publish batch for {}", streamId, e);
+        }
+
+        return Unit.INSTANCE;
+    }
+
     private static void configureShutdownHook(Deque<AutoCloseable> resources, ReentrantLock lock, Condition condition) {
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             LOGGER.info("Shutdown start");
@@ -203,5 +245,4 @@ private static void awaitShutdown(ReentrantLock lock, Condition condition) throw
             lock.unlock();
         }
     }
-
-}
+}
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java b/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java
index d0f2f1c..e0133aa 100644
--- a/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java
+++ b/src/main/java/com/exactpro/th2/readlog/RegexLogParser.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2020 Exactpro (Exactpro Systems Limited)
+ * Copyright 2020-2023 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package com.exactpro.th2.readlog;
 
 import java.io.StringWriter;
@@ -35,6 +36,7 @@
 import com.exactpro.th2.common.grpc.Direction;
 import com.exactpro.th2.read.file.common.StreamId;
 import com.exactpro.th2.readlog.cfg.AliasConfiguration;
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.TransportUtilsKt;
 import com.opencsv.CSVWriter;
 import com.opencsv.ICSVWriter;
 import org.apache.commons.text.StringSubstitutor;
@@ -81,7 +83,7 @@ public LogData parse(StreamId streamId, String raw) {
         }
 
         LogData resultData = new LogData();
-        resultData.setDirection(direction);
+        resultData.setDirection(TransportUtilsKt.getTransport(direction));
 
         List<Integer> regexGroups = configuration.getGroups();
         if (configuration.isJoinGroups()) {
@@ -91,7 +93,7 @@ public LogData parse(StreamId streamId, String raw) {
         }
 
         if (resultData.getBody().isEmpty()) {
-            // fast way, nothing matches the regexp so we don't need to check for date pattern
+            // fast way, nothing matches the regexp, so we don't need to check for date pattern
             return resultData;
         }
 
@@ -187,7 +189,7 @@ private void parseBodyJoined(String raw, AliasConfiguration configuration, LogDa
 
     private void addJoined(LogData data, List<List<String>> values, char delimiter) {
         var writer = new StringWriter();
-        ICSVWriter csvPrinter = createCsvWriter(writer, delimiter); // we can ignore closing because there is not IO
+        ICSVWriter csvPrinter = createCsvWriter(writer, delimiter); // we can ignore closing because there is no IO
         for (List<String> value : values) {
             csvPrinter.writeNext(value.toArray(String[]::new));
         }
@@ -225,4 +227,4 @@ private Integer tryParse(String value) {
             return null;
         }
     }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/cfg/LogReaderConfiguration.java b/src/main/java/com/exactpro/th2/readlog/cfg/LogReaderConfiguration.java
index 7ddf5be..6f0b5e7 100644
--- a/src/main/java/com/exactpro/th2/readlog/cfg/LogReaderConfiguration.java
+++ b/src/main/java/com/exactpro/th2/readlog/cfg/LogReaderConfiguration.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2020 Exactpro (Exactpro Systems Limited)
+ * Copyright 2020-2023 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -52,6 +52,9 @@ public class LogReaderConfiguration {
     @JsonPropertyDescription("Enables synchronization information about last timestamp and sequence for stream with Cradle")
     private boolean syncWithCradle = true;
 
+    @JsonPropertyDescription("Enables using th2 transport protocol")
+    private boolean useTransport = false;
+
     @JsonCreator
     public LogReaderConfiguration(@JsonProperty("logDirectory") Path logDirectory) {
         this.logDirectory = Objects.requireNonNull(logDirectory, "'Log directory' parameter");
@@ -92,4 +95,12 @@ public boolean isSyncWithCradle() {
     public void setSyncWithCradle(boolean syncWithCradle) {
         this.syncWithCradle = syncWithCradle;
     }
-}
+
+    public void setUseTransport(boolean useTransport) {
+        this.useTransport = useTransport;
+    }
+
+    public boolean isUseTransport() {
+        return useTransport;
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java b/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
index a83537a..4f4a7bb 100644
--- a/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
+++ b/src/main/java/com/exactpro/th2/readlog/impl/LogFileReader.java
@@ -1,5 +1,5 @@
-/*******************************************************************************
- * Copyright 2022 Exactpro (Exactpro Systems Limited)
+/*
+ * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,24 +12,27 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- ******************************************************************************/
+ */
 
 package com.exactpro.th2.readlog.impl;
 
-import com.exactpro.th2.common.grpc.MessageID;
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.MessageId;
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.RawMessage;
 import com.exactpro.th2.read.file.common.AbstractFileReader;
 import com.exactpro.th2.read.file.common.DirectoryChecker;
 import com.exactpro.th2.read.file.common.FileSourceWrapper;
 import com.exactpro.th2.read.file.common.MovedFileTracker;
 import com.exactpro.th2.read.file.common.StreamId;
-import com.exactpro.th2.read.file.common.impl.DefaultFileReader.Builder;
+import com.exactpro.th2.read.file.common.impl.ProtoDefaultFileReader;
 import com.exactpro.th2.read.file.common.impl.RecoverableBufferedReaderWrapper;
+import com.exactpro.th2.read.file.common.impl.TransportDefaultFileReader;
 import com.exactpro.th2.read.file.common.state.ReaderState;
 import com.exactpro.th2.readlog.RegexLogParser;
 import com.exactpro.th2.readlog.cfg.LogReaderConfiguration;
 import com.exactpro.th2.readlog.impl.lambdas.ForOnError;
 import com.exactpro.th2.readlog.impl.lambdas.ForOnSourceCorrupted;
-import com.exactpro.th2.readlog.impl.lambdas.ForOnStreamData;
+import com.exactpro.th2.readlog.impl.lambdas.ProtoForOnStreamData;
+import com.exactpro.th2.readlog.impl.lambdas.TransportForOnStreamData;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 
 import java.io.IOException;
@@ -44,18 +47,18 @@
 
 public class LogFileReader {
 
-    public static AbstractFileReader<LineNumberReader> getLogFileReader(
+    public static AbstractFileReader<LineNumberReader, com.exactpro.th2.common.grpc.RawMessage.Builder, com.exactpro.th2.common.grpc.MessageID> getProtoLogFileReader(
             LogReaderConfiguration configuration,
             ReaderState readerState,
-            Function<StreamId, MessageID> initialMessageId,
-            ForOnStreamData forStream,
+            Function<StreamId, com.exactpro.th2.common.grpc.MessageID> initialMessageId,
+            ProtoForOnStreamData forStream,
             ForOnError forError,
             ForOnSourceCorrupted forCorrupted
     ){
-            return new Builder<>(
+            return new ProtoDefaultFileReader.Builder<>(
                     configuration.getCommon(),
                     getDirectoryChecker(configuration),
-                    new RegexpContentParser(new RegexLogParser(configuration.getAliases())),
+                    new ProtoRegexpContentParser(new RegexLogParser(configuration.getAliases())),
                     new MovedFileTracker(configuration.getLogDirectory()),
                     readerState,
                     initialMessageId::apply,
@@ -67,7 +70,31 @@ public static AbstractFileReader<LineNumberReader> getLogFileReader(
                     .onError(forError::action)
                     .onSourceCorrupted(forCorrupted::action)
                     .build();
+    }
 
+    public static AbstractFileReader<LineNumberReader, RawMessage.Builder, MessageId.Builder> getTransportLogFileReader(
+            LogReaderConfiguration configuration,
+            ReaderState readerState,
+            Function<StreamId, MessageId.Builder> initialMessageId,
+            TransportForOnStreamData forStream,
+            ForOnError forError,
+            ForOnSourceCorrupted forCorrupted
+    ){
+        return new TransportDefaultFileReader.Builder<>(
+                configuration.getCommon(),
+                getDirectoryChecker(configuration),
+                new TransportRegexpContentParser(new RegexLogParser(configuration.getAliases())),
+                new MovedFileTracker(configuration.getLogDirectory()),
+                readerState,
+                initialMessageId::apply,
+                LogFileReader::createSource
+        )
+                .readFileImmediately()
+                .acceptNewerFiles()
+                .onStreamData(forStream::action)
+                .onError(forError::action)
+                .onSourceCorrupted(forCorrupted::action)
+                .build();
     }
 
     private static DirectoryChecker getDirectoryChecker(LogReaderConfiguration configuration) {
@@ -90,4 +117,4 @@ private static FileSourceWrapper<LineNumberReader> createSource(StreamId streamI
             return ExceptionUtils.rethrow(e);
         }
     }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java b/src/main/java/com/exactpro/th2/readlog/impl/ProtoRegexpContentParser.java
similarity index 83%
rename from src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java
rename to src/main/java/com/exactpro/th2/readlog/impl/ProtoRegexpContentParser.java
index 494a10e..1371093 100644
--- a/src/main/java/com/exactpro/th2/readlog/impl/RegexpContentParser.java
+++ b/src/main/java/com/exactpro/th2/readlog/impl/ProtoRegexpContentParser.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2021 Exactpro (Exactpro Systems Limited)
+ * Copyright 2020-2023 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,25 +13,22 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package com.exactpro.th2.readlog.impl;
 
 import com.exactpro.th2.common.grpc.RawMessageMetadata;
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.TransportUtilsKt;
 import com.exactpro.th2.common.message.MessageUtils;
+import com.exactpro.th2.read.file.common.impl.LineParser;
 import com.exactpro.th2.readlog.LogData;
 import com.google.protobuf.ByteString;
 import java.nio.charset.StandardCharsets;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZoneOffset;
 import java.util.List;
-import java.util.Objects;
-
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 
 import com.exactpro.th2.common.grpc.RawMessage;
 import com.exactpro.th2.read.file.common.StreamId;
-import com.exactpro.th2.read.file.common.impl.LineParser;
 import com.exactpro.th2.readlog.RegexLogParser;
 
 import org.slf4j.Logger;
@@ -39,11 +36,12 @@
 
 import static java.util.Objects.requireNonNull;
 
-public class RegexpContentParser extends LineParser {
-    private static final Logger LOGGER = LoggerFactory.getLogger(RegexpContentParser.class);
+public class ProtoRegexpContentParser extends LineParser<RawMessage.Builder> {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ProtoRegexpContentParser.class);
     private final RegexLogParser parser;
 
-    public RegexpContentParser(RegexLogParser parser) {
+    public ProtoRegexpContentParser(RegexLogParser parser) {
+        super(LineParser.PROTO);
         this.parser = requireNonNull(parser, "'Parser' parameter");
     }
 
@@ -61,8 +59,8 @@ protected List<RawMessage.Builder> lineToMessages(@Nonnull StreamId streamId, @N
     }
 
     private void setupMetadata(RawMessageMetadata.Builder builder, LogData logData) {
-        builder.getIdBuilder().setDirection(requireNonNull(logData.getDirection(),
-                "direction is not set"));
+        builder.getIdBuilder().setDirection(TransportUtilsKt.getProto(requireNonNull(logData.getDirection(),
+                "direction is not set")));
         if (logData.getParsedTimestamp() != null) {
             builder.getIdBuilder().setTimestamp(MessageUtils.toTimestamp(logData.getParsedTimestamp()));
         }
@@ -70,5 +68,4 @@ private void setupMetadata(RawMessageMetadata.Builder builder, LogData logData)
             builder.putProperties("logTimestamp", logData.getRawTimestamp());
         }
     }
-
-}
+}
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/TransportRegexpContentParser.java b/src/main/java/com/exactpro/th2/readlog/impl/TransportRegexpContentParser.java
new file mode 100644
index 0000000..d183ab4
--- /dev/null
+++ b/src/main/java/com/exactpro/th2/readlog/impl/TransportRegexpContentParser.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2023 Exactpro (Exactpro Systems Limited)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.exactpro.th2.readlog.impl;
+
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.RawMessage;
+import com.exactpro.th2.read.file.common.StreamId;
+import com.exactpro.th2.read.file.common.impl.LineParser;
+import com.exactpro.th2.readlog.LogData;
+import com.exactpro.th2.readlog.RegexLogParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static java.util.Objects.requireNonNull;
+
+public class TransportRegexpContentParser extends LineParser<RawMessage.Builder> {
+    private static final Logger LOGGER = LoggerFactory.getLogger(TransportRegexpContentParser.class);
+    private final RegexLogParser parser;
+
+    public TransportRegexpContentParser(RegexLogParser parser) {
+        super(LineParser.TRANSPORT);
+        this.parser = requireNonNull(parser, "'Parser' parameter");
+    }
+
+    @Nonnull
+    @Override
+    protected List<RawMessage.Builder> lineToMessages(@Nonnull StreamId streamId, @Nonnull String readLine) {
+        LogData logData = parser.parse(streamId, readLine);
+        LOGGER.trace("{} line(s) extracted from {}: {}", logData.getBody().size(), readLine, logData.getBody());
+        return logData.getBody().stream().map(it -> {
+            RawMessage.Builder builder = RawMessage.builder();
+            setupMetadata(builder, logData);
+            builder.setBody(it.getBytes(StandardCharsets.UTF_8));
+            return builder;
+        }).collect(Collectors.toList());
+    }
+
+    private void setupMetadata(RawMessage.Builder builder, LogData logData) {
+        builder.idBuilder().setDirection(requireNonNull(logData.getDirection(), "direction is not set"));
+
+        if (logData.getParsedTimestamp() != null) {
+            builder.idBuilder().setTimestamp(logData.getParsedTimestamp());
+        }
+
+        if (logData.getRawTimestamp() != null) {
+            builder.addMetadataProperty("logTimestamp", logData.getRawTimestamp());
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnStreamData.java b/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ProtoForOnStreamData.java
similarity index 70%
rename from src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnStreamData.java
rename to src/main/java/com/exactpro/th2/readlog/impl/lambdas/ProtoForOnStreamData.java
index c35a662..b36fe48 100644
--- a/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ForOnStreamData.java
+++ b/src/main/java/com/exactpro/th2/readlog/impl/lambdas/ProtoForOnStreamData.java
@@ -1,5 +1,5 @@
-/*******************************************************************************
- * Copyright 2022 Exactpro (Exactpro Systems Limited)
+/*
+ * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- ******************************************************************************/
+ */
 
 package com.exactpro.th2.readlog.impl.lambdas;
 
@@ -22,6 +22,6 @@
 
 import java.util.List;
 
-public interface ForOnStreamData {
-    Unit action(StreamId id, List<RawMessage.Builder> builder);
-}
+public interface ProtoForOnStreamData {
+    Unit action(StreamId id, List<? extends RawMessage.Builder> builder);
+}
\ No newline at end of file
diff --git a/src/main/java/com/exactpro/th2/readlog/impl/lambdas/TransportForOnStreamData.java b/src/main/java/com/exactpro/th2/readlog/impl/lambdas/TransportForOnStreamData.java
new file mode 100644
index 0000000..40fb3c5
--- /dev/null
+++ b/src/main/java/com/exactpro/th2/readlog/impl/lambdas/TransportForOnStreamData.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2023 Exactpro (Exactpro Systems Limited)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.exactpro.th2.readlog.impl.lambdas;
+
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.RawMessage;
+import com.exactpro.th2.read.file.common.StreamId;
+import kotlin.Unit;
+
+import java.util.List;
+
+public interface TransportForOnStreamData {
+    Unit action(StreamId id, List<? extends RawMessage.Builder> builder);
+}
\ No newline at end of file
diff --git a/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt b/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
index 2b31972..9423696 100644
--- a/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
+++ b/src/main/kotlin/com/exactpro/th2/readlog/impl/CradleReaderState.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2022. Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2023. Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,27 +20,24 @@ import com.exactpro.cradle.BookId
 import com.exactpro.cradle.CradleStorage
 import com.exactpro.cradle.Order
 import com.exactpro.cradle.messages.GroupedMessageFilter
-import com.exactpro.cradle.messages.MessageFilter
-import com.exactpro.cradle.messages.StoredMessageId
-import com.exactpro.th2.common.grpc.RawMessage
-import com.exactpro.th2.common.util.toCradleDirection
 import com.exactpro.th2.read.file.common.StreamId
 import com.exactpro.th2.read.file.common.state.ReaderState
+import com.exactpro.th2.read.file.common.state.Content
+import com.exactpro.th2.read.file.common.state.ProtoContent
+import com.exactpro.th2.read.file.common.state.TransportContent
 import com.exactpro.th2.read.file.common.state.StreamData
 import com.exactpro.th2.read.file.common.state.impl.InMemoryReaderState
 import com.google.protobuf.ByteString
 import com.google.protobuf.UnsafeByteOperations
+import io.netty.buffer.Unpooled
 import java.time.Instant
-import java.time.temporal.ChronoUnit
 
-class CradleReaderState private constructor(
+class CradleReaderState @JvmOverloads constructor(
     private val cradleStorage: CradleStorage,
-    private val delegate: ReaderState,
+    private val delegate: ReaderState = InMemoryReaderState(),
     private val bookSupplier: (StreamId) -> String,
+    private val wrapContent: (ByteArray) -> Content
 ): ReaderState by delegate {
-    constructor(cradleStorage: CradleStorage, bookSupplier: (StreamId) -> String)
-            : this(cradleStorage, InMemoryReaderState(), bookSupplier)
-
     override fun get(streamId: StreamId): StreamData? {
         return delegate[streamId] ?: cradleStorage.getGroupedMessageBatches(
             GroupedMessageFilter.builder()
@@ -54,8 +51,16 @@ class CradleReaderState private constructor(
             StreamData(
                 timestamp,
                 sequence,
-                content?.let(UnsafeByteOperations::unsafeWrap) ?: ByteString.EMPTY,
+                wrapContent(content)
             )
         }
     }
+
+    companion object {
+        @JvmField
+        val WRAP_PROTO: (ByteArray?) -> Content = { ProtoContent(it?.let(UnsafeByteOperations::unsafeWrap) ?: ByteString.EMPTY) }
+
+        @JvmField
+        val WRAP_TRANSPORT:  (ByteArray?) -> Content = { TransportContent(it?.let(Unpooled::wrappedBuffer) ?: Unpooled.EMPTY_BUFFER) }
+    }
 }
\ No newline at end of file
diff --git a/src/test/java/com/exactpro/th2/readlog/TestLogParser.java b/src/test/java/com/exactpro/th2/readlog/TestLogParser.java
index 4bcb646..9f9485a 100644
--- a/src/test/java/com/exactpro/th2/readlog/TestLogParser.java
+++ b/src/test/java/com/exactpro/th2/readlog/TestLogParser.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2021 Exactpro (Exactpro Systems Limited)
+ * Copyright 2020-2023 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 package com.exactpro.th2.readlog;
 
 import com.exactpro.th2.common.grpc.Direction;
+import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.TransportUtilsKt;
 import com.exactpro.th2.read.file.common.StreamId;
 import com.exactpro.th2.readlog.cfg.AliasConfiguration;
 
@@ -38,7 +39,6 @@
 import static org.junit.jupiter.api.Assertions.assertAll;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.params.provider.Arguments.arguments;
 
 public class TestLogParser {
@@ -68,7 +68,7 @@ void skipMessageByTimestamp(Instant skipBefore, boolean hasMessages) {
 
         LogData data = parser.parse(new StreamId("test"), "2021-03-23 13:21:37.991337479 some data");
         if (hasMessages) {
-            assertEquals(Direction.FIRST, data.getDirection(), "unexpected direction");
+            assertEquals(Direction.FIRST, TransportUtilsKt.getProto(data.getDirection()), "unexpected direction");
         }
         assertEquals(hasMessages, !data.getBody().isEmpty(), () -> "unexpected data " + data.getBody());
         if (hasMessages) {
@@ -88,7 +88,7 @@ static List<Arguments> skipMessageParams() {
     void parser(Direction direction) {
         RegexLogParser logParser = new RegexLogParser(getConfiguration());
         LogData data = logParser.parse(new StreamId(TEST_MESSAGE_ALIAS), direction == Direction.FIRST ? RAW_LOG_IN : RAW_LOG_OUT);
-        assertEquals(direction, data.getDirection(), "unexpected direction");
+        assertEquals(direction, TransportUtilsKt.getProto(data.getDirection()), "unexpected direction");
         assertEquals(1, data.getBody().size());
         assertEquals("NewOrderSingle={ AuthenticationBlock={ UserID=\"qwrqwrq\" SessionKey=123456 } Header={ MsgTime=2021-Mar-21 21:21:21.210000000 CreationTime=2021-Mar-21 21:21:21.210000000 } NewOrder={ InstrumentBlock={ InstrSymbol=\"TEST_SYMBOL\" SecurityID=\"212121\" SecurityIDSource=TestSource SecurityExchange=\"test\" }}}", data.getBody().get(0));
         assertEquals("2021-03-23 13:21:37.991337479", data.getRawTimestamp());
@@ -144,4 +144,4 @@ private Map<String, AliasConfiguration> getConfiguration() {
 
         return result;
     }
-}
+}
\ No newline at end of file

From 403bf054f90353c658604383d10a5ce455092acf Mon Sep 17 00:00:00 2001
From: Oleg <oleg.smirnov@exactprosystems.com>
Date: Fri, 24 Nov 2023 14:38:51 +0400
Subject: [PATCH 16/20] [TH2-5131] Update core version to fix problems with
 transport message ID

---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index fccc4b1..7846c99 100644
--- a/build.gradle
+++ b/build.gradle
@@ -16,9 +16,9 @@ apply plugin: 'kotlin-kapt'
 
 dependencies {
     api platform("com.exactpro.th2:bom:4.5.0")
-    implementation "com.exactpro.th2:common:5.4.2-dev"
+    implementation "com.exactpro.th2:common:5.7.1-dev"
     implementation "com.exactpro.th2:common-utils:2.2.0-dev"
-    implementation "com.exactpro.th2:read-file-common-core:3.0.0-dev"
+    implementation "com.exactpro.th2:read-file-common-core:3.1.0-dev"
 
     implementation ("com.opencsv:opencsv:5.8") {
         because("we need to write a correct CSV in case of free pattern is used")

From 1fc12798fc1fe259225151c22b60fffb39df454f Mon Sep 17 00:00:00 2001
From: Oleg <oleg.smirnov@exactprosystems.com>
Date: Fri, 24 Nov 2023 14:41:12 +0400
Subject: [PATCH 17/20] [TH2-5131] Update version and readme

---
 README.md         | 7 ++++++-
 gradle.properties | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index e35c6bf..e6d53e7 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Log Reader User Manual 4.1.0
+# Log Reader User Manual 4.1.1
 
 ## Document Information
 
@@ -227,6 +227,11 @@ Output: 8=FIXT.1.1\u00019=66\u000135=A\u000134=1\u000149=NFT2_FIX1\u000156=FGW\u
 
 ## Changes
 
+### 4.1.1
+
++ Update core version to 3.1.0-dev
++ Update common to 5.7.1-dev
+
 ### 4.1.0
 
 + Added support for th2 transport protocol
diff --git a/gradle.properties b/gradle.properties
index e874dbd..7a14641 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,4 +1,4 @@
-release_version=4.1.0
+release_version=4.1.1
 vcs_url=https://github.com/th2-net/th2-read-log
 description=DataReaderClient
 app_main_class=com.exactpro.th2.readlog.Main
\ No newline at end of file

From 36c5b2f8cc30618533843f8c11e47e1482ff1139 Mon Sep 17 00:00:00 2001
From: Oleg <oleg.smirnov@exactprosystems.com>
Date: Fri, 24 Nov 2023 14:43:10 +0400
Subject: [PATCH 18/20] Execute dev build on dev branch

---
 .github/workflows/dev-publish.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/dev-publish.yml b/.github/workflows/dev-publish.yml
index 4591b1f..181a8f8 100644
--- a/.github/workflows/dev-publish.yml
+++ b/.github/workflows/dev-publish.yml
@@ -5,7 +5,6 @@ on:
     branches-ignore:
       - master
       - version-*
-      - dev-version-*
 
 jobs:
   build-job:

From c160c04c1bff01b7c4eb904e94e3674382562ea1 Mon Sep 17 00:00:00 2001
From: Oleg <oleg.smirnov@exactprosystems.com>
Date: Fri, 24 Nov 2023 14:43:34 +0400
Subject: [PATCH 19/20] Remove sonatype publication

---
 .github/workflows/dev-publish.yml                               | 2 +-
 .../workflows/dev-release-java-publish-sonatype-and-docker.yml  | 2 +-
 .github/workflows/release-publish.yml                           | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/dev-publish.yml b/.github/workflows/dev-publish.yml
index 181a8f8..04848d3 100644
--- a/.github/workflows/dev-publish.yml
+++ b/.github/workflows/dev-publish.yml
@@ -10,7 +10,7 @@ jobs:
   build-job:
     uses: th2-net/.github/.github/workflows/compound-java-dev.yml@main
     with:
-      build-target: 'Docker,Sonatype'
+      build-target: 'Docker'
       docker-username: ${{ github.actor }}
     secrets:
       docker-password: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/dev-release-java-publish-sonatype-and-docker.yml b/.github/workflows/dev-release-java-publish-sonatype-and-docker.yml
index d943c63..096313c 100644
--- a/.github/workflows/dev-release-java-publish-sonatype-and-docker.yml
+++ b/.github/workflows/dev-release-java-publish-sonatype-and-docker.yml
@@ -9,7 +9,7 @@ jobs:
   build:
     uses: th2-net/.github/.github/workflows/compound-java.yml@main
     with:
-      build-target: 'Docker,Sonatype'
+      build-target: 'Docker'
       runsOn: ubuntu-latest
       gradleVersion: '7'
       docker-username: ${{ github.actor }}
diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml
index 6e709f4..8df17dd 100644
--- a/.github/workflows/release-publish.yml
+++ b/.github/workflows/release-publish.yml
@@ -11,7 +11,7 @@ jobs:
   build-job:
     uses: th2-net/.github/.github/workflows/compound-java.yml@main
     with:
-      build-target: 'Docker,Sonatype'
+      build-target: 'Docker'
       docker-username: ${{ github.actor }}
     secrets:
       docker-password: ${{ secrets.GITHUB_TOKEN }}

From 70556c95813ae36463647a46f23217beb2649e3f Mon Sep 17 00:00:00 2001
From: Nikita Smirnov
 <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com>
Date: Wed, 15 May 2024 10:53:01 +0400
Subject: [PATCH 20/20] Migrate to th2 gradle plugin `0.0.6` (#49)

* Migrate to th2 gradle plugin `0.0.6`
+ Updated bom: `4.6.1`
+ Updated common: `5.11.0-dev`
+ Updated read-file-common-core: `3.3.0-dev`
+ Updated jakarta.annotation-api: `3.0.0`
+ Updated opencsv: `5.9`
---
 .github/workflows/build-dev-release.yml       | 15 +++++
 .github/workflows/build-release.yml           | 15 +++++
 .github/workflows/build-sanpshot.yml          | 20 +++++++
 .github/workflows/ci-unwelcome-words.yml      |  4 +-
 .github/workflows/dev-publish.yml             | 20 -------
 ...lease-java-publish-sonatype-and-docker.yml | 22 -------
 .github/workflows/release-publish.yml         | 21 -------
 README.md                                     | 11 +++-
 build.gradle                                  | 60 ++++++++++++-------
 gradle.properties                             |  5 +-
 10 files changed, 103 insertions(+), 90 deletions(-)
 create mode 100644 .github/workflows/build-dev-release.yml
 create mode 100644 .github/workflows/build-release.yml
 create mode 100644 .github/workflows/build-sanpshot.yml
 delete mode 100644 .github/workflows/dev-publish.yml
 delete mode 100644 .github/workflows/dev-release-java-publish-sonatype-and-docker.yml
 delete mode 100644 .github/workflows/release-publish.yml

diff --git a/.github/workflows/build-dev-release.yml b/.github/workflows/build-dev-release.yml
new file mode 100644
index 0000000..b438d11
--- /dev/null
+++ b/.github/workflows/build-dev-release.yml
@@ -0,0 +1,15 @@
+name: Build and publish dev release Docker image to Github Container Registry ghcr.io
+
+on: workflow_dispatch
+
+jobs:
+  build:
+    uses: th2-net/.github/.github/workflows/compound-java.yml@main
+    with:
+      build-target: 'Docker'
+      devRelease: true
+      createTag: true
+      docker-username: ${{ github.actor }}
+    secrets:
+      docker-password: ${{ secrets.GITHUB_TOKEN }}
+      nvd-api-key: ${{ secrets.NVD_APIKEY }}
\ No newline at end of file
diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml
new file mode 100644
index 0000000..dcf70be
--- /dev/null
+++ b/.github/workflows/build-release.yml
@@ -0,0 +1,15 @@
+name: Build and publish release Docker image to Github Container Registry ghcr.io
+
+on: workflow_dispatch
+
+jobs:
+  build:
+    uses: th2-net/.github/.github/workflows/compound-java.yml@main
+    with:
+      build-target: 'Docker'
+      devRelease: false
+      createTag: true
+      docker-username: ${{ github.actor }}
+    secrets:
+      docker-password: ${{ secrets.GITHUB_TOKEN }}
+      nvd-api-key: ${{ secrets.NVD_APIKEY }}
\ No newline at end of file
diff --git a/.github/workflows/build-sanpshot.yml b/.github/workflows/build-sanpshot.yml
new file mode 100644
index 0000000..9366a44
--- /dev/null
+++ b/.github/workflows/build-sanpshot.yml
@@ -0,0 +1,20 @@
+name: Build and publish Docker image to Github Container Registry ghcr.io
+
+on:
+  push:
+    branches-ignore:
+      - master
+      - version-*
+      - dependabot**
+    paths-ignore:
+      - README.md
+
+jobs:
+  build-job:
+    uses: th2-net/.github/.github/workflows/compound-java-dev.yml@main
+    with:
+      build-target: 'Docker'
+      docker-username: ${{ github.actor }}
+    secrets:
+      docker-password: ${{ secrets.GITHUB_TOKEN }}
+      nvd-api-key: ${{ secrets.NVD_APIKEY }}
\ No newline at end of file
diff --git a/.github/workflows/ci-unwelcome-words.yml b/.github/workflows/ci-unwelcome-words.yml
index a7e7f3a..4e5f3a6 100644
--- a/.github/workflows/ci-unwelcome-words.yml
+++ b/.github/workflows/ci-unwelcome-words.yml
@@ -7,11 +7,11 @@ jobs:
   test:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.sha }}
     - name: Checkout tool
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         repository: exactpro-th2/ci-github-action
         ref: master
diff --git a/.github/workflows/dev-publish.yml b/.github/workflows/dev-publish.yml
deleted file mode 100644
index 04848d3..0000000
--- a/.github/workflows/dev-publish.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-name: Dev build and publish - Docker and Sonatype
-
-on:
-  push:
-    branches-ignore:
-      - master
-      - version-*
-
-jobs:
-  build-job:
-    uses: th2-net/.github/.github/workflows/compound-java-dev.yml@main
-    with:
-      build-target: 'Docker'
-      docker-username: ${{ github.actor }}
-    secrets:
-      docker-password: ${{ secrets.GITHUB_TOKEN }}
-      sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
-      sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
-      sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
-      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
diff --git a/.github/workflows/dev-release-java-publish-sonatype-and-docker.yml b/.github/workflows/dev-release-java-publish-sonatype-and-docker.yml
deleted file mode 100644
index 096313c..0000000
--- a/.github/workflows/dev-release-java-publish-sonatype-and-docker.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-name: Dev release build and publish - Docker and Sonatype
-
-on:
-  push:
-    tags:
-      - \d+.\d+.\d+-dev
-
-jobs:
-  build:
-    uses: th2-net/.github/.github/workflows/compound-java.yml@main
-    with:
-      build-target: 'Docker'
-      runsOn: ubuntu-latest
-      gradleVersion: '7'
-      docker-username: ${{ github.actor }}
-      devRelease: true
-    secrets:
-      docker-password: ${{ secrets.GITHUB_TOKEN }}
-      sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
-      sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
-      sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
-      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
\ No newline at end of file
diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml
deleted file mode 100644
index 8df17dd..0000000
--- a/.github/workflows/release-publish.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-name: Release build and publish - Docker and Sonatype
-on:
-  push:
-    branches:
-      - master
-      - version-*
-    paths:
-      - gradle.properties
-
-jobs:
-  build-job:
-    uses: th2-net/.github/.github/workflows/compound-java.yml@main
-    with:
-      build-target: 'Docker'
-      docker-username: ${{ github.actor }}
-    secrets:
-      docker-password: ${{ secrets.GITHUB_TOKEN }}
-      sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
-      sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
-      sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
-      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
diff --git a/README.md b/README.md
index e6d53e7..9856845 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Log Reader User Manual 4.1.1
+# Log Reader User Manual 4.2.0
 
 ## Document Information
 
@@ -227,6 +227,15 @@ Output: 8=FIXT.1.1\u00019=66\u000135=A\u000134=1\u000149=NFT2_FIX1\u000156=FGW\u
 
 ## Changes
 
+### 4.2.0
+
++ Migrate to th2 gradle plugin `0.0.6`
++ Updated bom: `4.6.1`
++ Updated common: `5.11.0-dev`
++ Updated read-file-common-core: `3.3.0-dev`
++ Updated jakarta.annotation-api: `3.0.0`
++ Updated opencsv: `5.9`
+
 ### 4.1.1
 
 + Update core version to 3.1.0-dev
diff --git a/build.gradle b/build.gradle
index 7846c99..3e19f9f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,35 +1,53 @@
-buildscript {
-    repositories {
-        gradlePluginPortal()
-        maven {
-            url = "https://s01.oss.sonatype.org/content/repositories/snapshots/"
-        }
+plugins {
+    id 'application'
+    id 'org.jetbrains.kotlin.jvm' version '1.8.22'
+    id 'org.jetbrains.kotlin.kapt' version '1.8.22'
+    id 'com.exactpro.th2.gradle.component' version '0.0.6'
+}
+
+group 'com.exactpro.th2'
+version release_version
+
+kotlin {
+    jvmToolchain(11)
+}
+
+repositories {
+    mavenCentral()
+    maven {
+        name 'Sonatype_snapshots'
+        url 'https://s01.oss.sonatype.org/content/repositories/snapshots/'
     }
-    dependencies {
-        classpath "com.exactpro.th2:th2-gradle-plugin:0.0.1-dev-5915968839-41381e5-SNAPSHOT"
+    maven {
+        name 'Sonatype_releases'
+        url 'https://s01.oss.sonatype.org/content/repositories/releases/'
     }
-}
+    mavenLocal()
 
-apply plugin: "com.exactpro.th2.common-conventions"
-apply plugin: "com.exactpro.th2.docker-conventions"
-apply plugin: 'kotlin-kapt'
+    configurations.configureEach {
+        resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
+        resolutionStrategy.cacheDynamicVersionsFor 0, 'seconds'
+    }
+}
 
 dependencies {
-    api platform("com.exactpro.th2:bom:4.5.0")
-    implementation "com.exactpro.th2:common:5.7.1-dev"
-    implementation "com.exactpro.th2:common-utils:2.2.0-dev"
-    implementation "com.exactpro.th2:read-file-common-core:3.1.0-dev"
+    implementation "com.exactpro.th2:common:5.11.0-dev"
+    implementation "com.exactpro.th2:common-utils:2.2.3-dev"
+    implementation "com.exactpro.th2:read-file-common-core:3.3.0-dev"
 
-    implementation ("com.opencsv:opencsv:5.8") {
+    implementation ("com.opencsv:opencsv:5.9") {
         because("we need to write a correct CSV in case of free pattern is used")
     }
 
-    implementation "javax.annotation:javax.annotation-api:1.3.2"
-    implementation "org.jetbrains.kotlin:kotlin-stdlib"
+    implementation 'jakarta.annotation:jakarta.annotation-api:3.0.0'
+    implementation "io.github.microutils:kotlin-logging:3.0.5"
 
     implementation "com.fasterxml.jackson.core:jackson-databind"
     implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310"
     implementation "com.fasterxml.jackson.module:jackson-module-kotlin"
+
+    testImplementation "org.junit.jupiter:junit-jupiter:5.10.2"
+    testImplementation "org.mockito:mockito-core:5.12.0"
 }
 
 test {
@@ -46,6 +64,6 @@ tasks.register('integrationTest', Test) {
     }
 }
 
-dependencyLocking {
-    lockAllConfigurations()
+application {
+    mainClassName = "com.exactpro.th2.readlog.Main"
 }
\ No newline at end of file
diff --git a/gradle.properties b/gradle.properties
index 7a14641..b81539e 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,4 +1,3 @@
-release_version=4.1.1
+release_version=4.2.0
 vcs_url=https://github.com/th2-net/th2-read-log
-description=DataReaderClient
-app_main_class=com.exactpro.th2.readlog.Main
\ No newline at end of file
+description=DataReaderClient
\ No newline at end of file