From 8b7143285aa36524cb7926310d64dcaadbec2517 Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Tue, 18 Sep 2018 09:58:26 +0300
Subject: [PATCH 01/14] adding in queue support using logzio-java-sender 1.0.16

---
 README.md                                     |  22 +-
 pom.xml                                       |   2 +-
 .../logz/logback/LogzioLogbackAppender.java   | 174 ++++++++++++----
 .../logz/logback/BaseLogbackAppenderTest.java |  42 +---
 .../logback/ExceptionLogbackAppenderTest.java |  54 -----
 .../logback/LogzioLogbackAppenderTest.java    | 190 ++++++++++++------
 6 files changed, 283 insertions(+), 201 deletions(-)
 delete mode 100644 src/test/java/io/logz/logback/ExceptionLogbackAppenderTest.java

diff --git a/README.md b/README.md
index ee1d696..b0605e8 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
 This appender sends logs to your [Logz.io](http://logz.io) account, using non-blocking threading, bulks, and HTTPS encryption. Please note that this appendr requires logback version 1.1.7 and up, and java 8 and up.
 
 ### Technical Information
-This appender uses [LogzioSender](https://github.com/logzio/logzio-java-sender) implementation. All logs are backed up to a local file system before being sent. Once you send a log, it will be enqueued in the buffer and 100% non-blocking. There is a background task that will handle the log shipment for you. This jar is an "Uber-Jar" that shades both BigQueue, Gson and Guava to avoid "dependency hell".
+This appender uses [LogzioSender](https://github.com/logzio/logzio-java-sender) implementation. All logs are backed up to a local file system before being sent. Once you send a log, it will be enqueued in the queue and 100% non-blocking. There is a background task that will handle the log shipment for you. This jar is an "Uber-Jar" that shades both BigQueue, Gson and Guava to avoid "dependency hell".
 
 ### Installation from maven
 ```xml
@@ -40,9 +40,7 @@ This appender uses [LogzioSender](https://github.com/logzio/logzio-java-sender)
 | **token**              | *None*                                 | Your Logz.io token, which can be found under "settings" in your account, If the value begins with `$` then the appender looks for an environment variable or system property with the name specified. For example: `$LOGZIO_TOKEN` will look for environment variable named `LOGZIO_TOKEN` |
 | **logzioType**               | *java*                                 | The [log type](http://support.logz.io/support/solutions/articles/6000103063-what-is-type-) for that appender, it must not contain spaces |
 | **logzioUrl**               | *https://listener.logz.io:8071*                                 | The url that the appender sends to.  If your account is in the EU you must use https://listener-eu.logz.io:8071 |
-| **drainTimeoutSec**       | *5*                                    | How often the appender should drain the buffer (in seconds) |
-| **fileSystemFullPercentThreshold** | *98*                                   | The percent of used file system space at which the appender will stop buffering. When we will reach that percentage, the file system in which the buffer rests will drop all new logs until the percentage of used space drops below that threshold. Set to -1 to never stop processing new logs |
-| **bufferDir**          | *System.getProperty("java.io.tmpdir")* | Where the appender should store the buffer |
+| **drainTimeoutSec**       | *5*                                    | How often the appender should drain the queue (in seconds) |
 | **socketTimeout**       | *10 * 1000*                                    | The socket timeout during log shipment |
 | **connectTimeout**       | *10 * 1000*                                    | The connection timeout during log shipment |
 | **addHostname**       | *false*                                    | Optional. If true, then a field named 'hostname' will be added holding the host name of the machine. If from some reason there's no defined hostname, this field won't be added |
@@ -53,6 +51,22 @@ This appender uses [LogzioSender](https://github.com/logzio/logzio-java-sender)
 | **format** | *text*  | Optional. `json` if the logged message is to be parsed as a one level JSON (in such a way that each JSON node will be a field in logz.io) or `text` if the logged message is to be treated as plain text.|
 | **markersFormat** | *text*  | Optional. `json` if the marker is to be parsed as a one level JSON (in such a way that each JSON node will be a field in logz.io), `text` if the marker is to be treated as plain text, or ignore if the marker is to be ignored and not included in the log.
 
+#### Parameters for in-memory queue
+| Parameter          | Default                              | Explained  |
+| ------------------ | ------------------------------------ | ----- |
+| **inMemoryQueueCapacityBytes**       | *1024 * 1024 * 100*                                | The amount of memory(bytes) we are allowed to use for the memory queue. If the value is -1 the sender will not limit the queue size.|
+| **inMemoryQueue**       | *false*                                | Set to true if the appender uses in memory queue. By default the appender uses disk queue|
+
+
+#### Parameters for disk queue
+| Parameter          | Default                              | Explained  |
+| ------------------ | ------------------------------------ | ----- |
+| **fileSystemFullPercentThreshold** | *98*                                   | The percent of used file system space at which the sender will stop queueing. When we will reach that percentage, the file system in which the queue is stored will drop all new logs until the percentage of used space drops below that threshold. Set to -1 to never stop processing new logs |
+| **gcPersistedQueueFilesIntervalSeconds**       | *30*                                    | How often the disk queue should clean sent logs from disk |
+| **bufferDir**(deprecated, use queueDir)          | *System.getProperty("java.io.tmpdir")* | Where the appender should store the queue |
+| **queueDir**          | *System.getProperty("java.io.tmpdir")* | Where the appender should store the queue |
+
+
 ### Code Example
 ```java
 import org.slf4j.Logger;
diff --git a/pom.xml b/pom.xml
index 292b37d..1c0ee71 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
     </scm>
 
     <properties>
-        <logzio-sender-version>1.0.10</logzio-sender-version>
+        <logzio-sender-version>1.0.16</logzio-sender-version>
     </properties>
 
     <build>
diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
index 65dcb48..91a1cdc 100644
--- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java
+++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -5,6 +5,7 @@
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.UnsynchronizedAppenderBase;
 import com.google.common.base.Splitter;
+import io.logz.sender.HttpsRequestConfiguration;
 import io.logz.sender.LogzioSender;
 import io.logz.sender.SenderStatusReporter;
 import io.logz.sender.com.google.gson.Gson;
@@ -18,7 +19,6 @@
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -39,6 +39,9 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEv
     private static final String FORMAT_TEXT = "text";
     private static final String FORMAT_JSON = "json";
     private static final String FORMAT_IGNORE = "ignore";
+    private static final int DONT_LIMIT_QUEUE_SPACE = -1;
+    private static final int LOWER_PERCENTAGE_FS_SPACE = 1;
+    private static final int UPPER_PERCENTAGE_FS_SPACE = 100;
 
     private static final Set<String> reservedFields =  new HashSet<>(Arrays.asList(TIMESTAMP,LOGLEVEL, MARKER, MESSAGE,LOGGER,THREAD,EXCEPTION));
 
@@ -52,7 +55,7 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEv
     private String logzioType = "java";
     private int drainTimeoutSec = 5;
     private int fileSystemFullPercentThreshold = 98;
-    private String bufferDir;
+    private String queueDir;
     private String logzioUrl;
     private int connectTimeout = 10 * 1000;
     private int socketTimeout = 10 * 1000;
@@ -60,6 +63,8 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEv
     private boolean addHostname = false;
     private boolean line = false;
     private boolean compressRequests = false;
+    private boolean inMemoryQueue = false;
+    private long inMemoryQueueCapacityBytes = 100 * 1024 * 1024;
     private int gcPersistedQueueFilesIntervalSeconds = 30;
     private String format = FORMAT_TEXT;
     private String markersFormat = FORMAT_TEXT;
@@ -106,8 +111,18 @@ public void setFileSystemFullPercentThreshold(int fileSystemFullPercentThreshold
         this.fileSystemFullPercentThreshold = fileSystemFullPercentThreshold;
     }
 
+    /**
+     *
+     * @param bufferDir: queue dir path
+     * @deprecated use {@link #setQueueDir(String)}
+     */
+    @Deprecated
     public void setBufferDir(String bufferDir) {
-        this.bufferDir = bufferDir;
+        this.queueDir = bufferDir;
+    }
+
+    public void setQueueDir(String queueDir) {
+        this.queueDir = queueDir;
     }
 
     public void setLogzioUrl(String logzioUrl) {
@@ -130,11 +145,26 @@ public void setDebug(boolean debug) {
         this.debug = debug;
     }
 
+    public void setInMemoryQueue(boolean inMemoryQueue) {
+        this.inMemoryQueue = true;
+    }
+
+    public boolean isInMemoryQueue() {
+        return inMemoryQueue;
+    }
+
+    public void setInMemoryQueueCapacityBytes(long inMemoryQueueCapacityBytes) {
+        this.inMemoryQueueCapacityBytes = inMemoryQueueCapacityBytes;
+    }
+
+    public long getInMemoryQueueCapacityBytes() {
+        return inMemoryQueueCapacityBytes;
+    }
+
     public boolean isCompressRequests() { return compressRequests; }
 
     public void setCompressRequests(boolean compressRequests) { this.compressRequests = compressRequests; }
 
-
     public void setAdditionalFields(String additionalFields) {
        if (additionalFields != null) {
            Splitter.on(';').omitEmptyStrings().withKeyValueSeparator('=').split(additionalFields).forEach((k, v) -> {
@@ -173,16 +203,98 @@ public void setGcPersistedQueueFilesIntervalSeconds(int gcPersistedQueueFilesInt
 
     @Override
     public void start() {
-        if (logzioToken == null) {
-            addError("Logz.io Token is missing! Bailing out..");
+        setHostname();
+        LogzioSender.Builder logzioSenderBuilder = getSenderBuilder();
+        if (inMemoryQueue) {
+            if (!validateInMemoryQueueCapacityBytes()) {
+                return;
+            }
+            logzioSenderBuilder
+                .withInMemoryQueue()
+                    .setCapacityInBytes(inMemoryQueueCapacityBytes)
+                .endInMemoryQueue();
+        } else {
+            if (!validateFsPercentThreshold()) {
+                return;
+            }
+            File queueDirFile = getQueueFile();
+            if (queueDirFile == null) {
+                return;
+            }
+            logzioSenderBuilder
+                    .withDiskQueue()
+                        .setQueueDir(queueDirFile)
+                        .setGcPersistedQueueFilesIntervalSeconds(gcPersistedQueueFilesIntervalSeconds)
+                        .setFsPercentThreshold(fileSystemFullPercentThreshold)
+                    .endDiskQueue();
+        }
+        try {
+            logzioSender = logzioSenderBuilder.build();
+        } catch (LogzioParameterErrorException e) {
+            addError("Could not create logzio sender", e);
             return;
         }
-        if (!(fileSystemFullPercentThreshold >= 1 && fileSystemFullPercentThreshold <= 100)) {
-            if (fileSystemFullPercentThreshold != -1) {
+
+        logzioSender.start();
+        throwableProxyConverter = new ThrowableProxyConverter();
+        lineOfCallerConverter = new LineOfCallerConverter();
+        throwableProxyConverter.setOptionList(Collections.singletonList("full"));
+        throwableProxyConverter.start();
+        super.start();
+    }
+
+    private LogzioSender.Builder getSenderBuilder() {
+        SenderStatusReporter reporter = new StatusReporter();
+        HttpsRequestConfiguration conf = getHttpsRequestConfiguration();
+        return LogzioSender
+                    .builder()
+                    .setDebug(debug)
+                    .setDrainTimeoutSec(drainTimeoutSec)
+                    .setHttpsRequestConfiguration(conf)
+                    .setReporter(reporter)
+                    .setTasksExecutor(context.getScheduledExecutorService());
+    }
+
+    private File getQueueFile() {
+        if (queueDir != null) {
+            queueDir += File.separator + logzioType;
+            File queueFile = new File(queueDir);
+            if (queueFile.exists()) {
+                if (!queueFile.canWrite()) {
+                    addError("We cant write to your queueDir location: " + queueFile.getAbsolutePath());
+                    return null;
+                }
+            } else {
+                if (!queueFile.mkdirs()) {
+                    addError("We cant create your queueDir location: " + queueFile.getAbsolutePath());
+                    return null;
+                }
+            }
+        } else {
+            queueDir = System.getProperty("java.io.tmpdir") + File.separator + "logzio-logback-queue" + File.separator + logzioType;
+        }
+        return new File(queueDir,"logzio-logback-appender");
+    }
+
+    private boolean validateInMemoryQueueCapacityBytes() {
+        if (inMemoryQueueCapacityBytes <= 0 && inMemoryQueueCapacityBytes != DONT_LIMIT_QUEUE_SPACE) {
+            addError("inMemoryQueueCapacityBytes should be a non zero integer or -1");
+            return false;
+        }
+        return true;
+    }
+
+    private boolean validateFsPercentThreshold() {
+        if (!(fileSystemFullPercentThreshold >= LOWER_PERCENTAGE_FS_SPACE && fileSystemFullPercentThreshold <= UPPER_PERCENTAGE_FS_SPACE)) {
+            if (fileSystemFullPercentThreshold != DONT_LIMIT_QUEUE_SPACE) {
                 addError("fileSystemFullPercentThreshold should be a number between 1 and 100, or -1");
-                return;
+                return false;
             }
         }
+        return true;
+    }
+
+    private void setHostname() {
         try {
             if (addHostname) {
                 String hostname = InetAddress.getLocalHost().getHostName();
@@ -191,39 +303,23 @@ public void start() {
         } catch (UnknownHostException e) {
             addWarn("The configuration addHostName was specified but the host could not be resolved, thus the field 'hostname' will not be added", e);
         }
-        if (bufferDir != null) {
-            bufferDir += File.separator + logzioType;
-            File bufferFile = new File(bufferDir);
-            if (bufferFile.exists()) {
-                if (!bufferFile.canWrite()) {
-                    addError("We cant write to your bufferDir location: "+bufferFile.getAbsolutePath());
-                    return;
-                }
-            } else {
-                if (!bufferFile.mkdirs()) {
-                    addError("We cant create your bufferDir location: "+bufferFile.getAbsolutePath());
-                    return;
-                }
-            }
-        } else {
-            bufferDir = System.getProperty("java.io.tmpdir") + File.separator+"logzio-logback-buffer"+File.separator + logzioType;
-        }
-        File bufferDirFile = new File(bufferDir,"logzio-logback-appender");
+    }
+
+    private HttpsRequestConfiguration getHttpsRequestConfiguration() {
         try {
-            SenderStatusReporter reporter = new StatusReporter();
-            logzioSender = LogzioSender.getOrCreateSenderByType(logzioToken, logzioType, drainTimeoutSec, fileSystemFullPercentThreshold,
-                    bufferDirFile, logzioUrl, socketTimeout, connectTimeout, debug,
-                    reporter, context.getScheduledExecutorService(), gcPersistedQueueFilesIntervalSeconds, compressRequests);
-            logzioSender.start();
+            return HttpsRequestConfiguration
+                    .builder()
+                    .setLogzioListenerUrl(logzioUrl)
+                    .setSocketTimeout(socketTimeout)
+                    .setLogzioType(logzioType)
+                    .setLogzioToken(logzioToken)
+                    .setConnectTimeout(connectTimeout)
+                    .setCompressRequests(compressRequests)
+                    .build();
         } catch (LogzioParameterErrorException e) {
-            addError("Some of the configuration parameters of logz.io is wrong: "+e.getMessage(), e);
-            return;
+            addError("Some of the configuration parameters of logz.io is wrong: " + e.getMessage(), e);
+            return null;
         }
-        throwableProxyConverter = new ThrowableProxyConverter();
-        lineOfCallerConverter = new LineOfCallerConverter();
-        throwableProxyConverter.setOptionList(Collections.singletonList("full"));
-        throwableProxyConverter.start();
-        super.start();
     }
 
     @Override
diff --git a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
index dae7a25..4e2b59d 100644
--- a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
@@ -14,11 +14,11 @@
 /**
  * @author MarinaRazumovsky
  */
-public class BaseLogbackAppenderTest {
+public abstract class BaseLogbackAppenderTest {
 
     private final static Logger logger = LoggerFactory.getLogger(BaseLogbackAppenderTest.class);
 
-    protected MockLogzioBulkListener mockListener;
+    MockLogzioBulkListener mockListener;
 
     @Before
     public void startMockListener() throws Exception {
@@ -31,43 +31,7 @@ public void stopMockListener() {
         mockListener.stop();
     }
 
-    protected Logger createLogger(String token, String type, String loggerName, Integer drainTimeout,
-                                  boolean addHostname,boolean line, String additionalFields, boolean compressRequests) {
-
-        logger.info("Creating logger {}. token={}, type={}, drainTimeout={}, addHostname={}, line={}, additionalFields={} ",
-                loggerName, token, type, drainTimeout, addHostname, line, additionalFields);
-
-        ch.qos.logback.classic.Logger logbackLogger =  (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(loggerName);
-        Context logbackContext = logbackLogger.getLoggerContext();
-        LogzioLogbackAppender logzioLogbackAppender = new LogzioLogbackAppender();
-        logzioLogbackAppender.setContext(logbackContext);
-        logzioLogbackAppender.setToken(token);
-        logzioLogbackAppender.setLogzioType(type);
-        logzioLogbackAppender.setDebug(true);
-        logzioLogbackAppender.setLine(line);
-        logzioLogbackAppender.setLogzioUrl("http://" + mockListener.getHost() + ":" + mockListener.getPort());
-        logzioLogbackAppender.setAddHostname(addHostname);
-        logzioLogbackAppender.setCompressRequests(compressRequests);
-        logzioLogbackAppender.setName("LogzioLogbackAppender");
-        if (drainTimeout != null) {
-            logzioLogbackAppender.setDrainTimeoutSec(drainTimeout);
-        }
-        if (additionalFields != null) {
-            logzioLogbackAppender.setAdditionalFields(additionalFields);
-        }
-        logzioLogbackAppender.start();
-        assertThat(logzioLogbackAppender.isStarted()).isTrue();
-        logbackLogger.addAppender(logzioLogbackAppender);
-        logbackLogger.setAdditive(false);
-        return logbackLogger;
-    }
-
-    protected Logger createLogger(String token, String type, String loggerName, Integer drainTimeout,
-                                  boolean addHostname,boolean line, String additionalFields) {
-        return createLogger(token, type, loggerName, drainTimeout, addHostname, line, additionalFields, false);
-    }
-
-    protected void sleepSeconds(int seconds) {
+    void sleepSeconds(int seconds) {
         logger.info("Sleeping {} [sec]...", seconds);
         try {
             Thread.sleep(seconds * 1000);
diff --git a/src/test/java/io/logz/logback/ExceptionLogbackAppenderTest.java b/src/test/java/io/logz/logback/ExceptionLogbackAppenderTest.java
deleted file mode 100644
index 49498f8..0000000
--- a/src/test/java/io/logz/logback/ExceptionLogbackAppenderTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package io.logz.logback;
-
-import ch.qos.logback.classic.Level;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.CountDownLatch;
-
-import static io.logz.test.MockLogzioBulkListener.LogRequest;
-import static org.assertj.core.api.Assertions.assertThat;
-
-/**
- * Created by roiravhon on 9/11/16.
- */
-public class ExceptionLogbackAppenderTest extends BaseLogbackAppenderTest {
-
-    private final static Logger logger = LoggerFactory.getLogger(LogzioLogbackAppenderTest.class);
-
-    @Test
-    public void checkExactStackTrace() throws Exception {
-        final CountDownLatch countDownLatch = new CountDownLatch(1);
-        final MyRunner.ExceptionGenerator exceptionGenerator = new MyRunner.ExceptionGenerator();
-
-        // We need to generate an exception with constant stack trace
-        new Thread(new MyRunner(countDownLatch, exceptionGenerator)).start();
-        countDownLatch.await();
-
-        String token = "exceptionToken";
-        String type = "stacktraceType";
-        String loggerName = "traceLogger";
-        int drainTimeout = 1;
-
-        String message1 = "Any line change here can cause the test to break";
-
-        String expectedException = "java.lang.RuntimeException: Got NPE!\n" +
-                "\tat io.logz.logback.MyRunner$ExceptionGenerator.generateNPE(MyRunner.java:33)\n" +
-                "\tat io.logz.logback.MyRunner.run(MyRunner.java:18)\n" +
-                "\tat java.lang.Thread.run(Thread.java:748)\n" +
-                "Caused by: java.lang.NullPointerException: null\n" +
-                "\tat io.logz.logback.MyRunner$ExceptionGenerator.generateNPE(MyRunner.java:31)\n" +
-                "\t... 2 common frames omitted\n";
-
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
-
-        testLogger.info(message1, exceptionGenerator.getE());
-        sleepSeconds(drainTimeout * 2);
-
-        mockListener.assertNumberOfReceivedMsgs(1);
-        LogRequest logRequest = mockListener.assertLogReceivedByMessage(message1);
-        mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr);
-        assertThat(logRequest.getStringFieldOrNull("exception")).isEqualTo(expectedException);
-    }
-}
diff --git a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
index edecc48..5101f54 100644
--- a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
@@ -3,9 +3,12 @@
 
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.core.Context;
 import io.logz.sender.com.google.gson.Gson;
 import io.logz.test.MockLogzioBulkListener;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.MDC;
@@ -13,22 +16,46 @@
 import org.slf4j.MarkerFactory;
 
 import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.fail;
 
-
+@RunWith(Parameterized.class)
 public class LogzioLogbackAppenderTest extends BaseLogbackAppenderTest {
 
     private final static Logger logger = LoggerFactory.getLogger(LogzioLogbackAppenderTest.class);
+    private LogzioLogbackAppender logzioLogbackAppender;
+
+    @Parameterized.Parameters
+    public static Collection<Object[]> logzioSenderBuilders() {
+        List<LogzioLogbackAppender> appenders = new ArrayList<>();
+        appenders.add(new LogzioLogbackAppender());
+        LogzioLogbackAppender inMemoryQueueApeender = new LogzioLogbackAppender();
+        inMemoryQueueApeender.setInMemoryQueue(true);
+        appenders.add(inMemoryQueueApeender);
+
+        Collection<Object[]> result = new ArrayList<>();
+        for (LogzioLogbackAppender appender : appenders) {
+            result.add(new Object[]{appender});
+        }
+        return result;
+    }
+
+    public LogzioLogbackAppenderTest(LogzioLogbackAppender logzioLogbackAppender) {
+        this.logzioLogbackAppender = logzioLogbackAppender;
+    }
 
     @Test
     public void validateJsonMessage(){
         String token = "validateJsonMessageToken";
-        String type = "validateJsonMessageType";
-        String loggerName = "validateJsonMessageLogger";
+        String type = "validateJsonMessageType" + random(8);
+        String loggerName = "validateJsonMessageLogger" + random(8);
         int drainTimeout = 1;
         String messageText = "message test";
 
@@ -39,10 +66,9 @@ public void validateJsonMessage(){
 
         String message1 = new Gson().toJson(map);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
-        LogzioLogbackAppender logzioLogbackAppender =
-                (LogzioLogbackAppender)((ch.qos.logback.classic.Logger)testLogger).getAppender("LogzioLogbackAppender");
         logzioLogbackAppender.setFormat("json");
+        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
+
         testLogger.info(message1);
 
         sleepSeconds(2 * drainTimeout);
@@ -58,13 +84,13 @@ public void validateJsonMessage(){
     @Test
     public void simpleAppending() {
         String token = "aBcDeFgHiJkLmNoPqRsT";
-        String type = "awesomeType";
-        String loggerName = "simpleAppending";
+        String type = "awesomeType" + random(8);
+        String loggerName = "simpleAppending" + random(8);
         int drainTimeout = 1;
         String message1 = "Testing.." + random(5);
         String message2 = "Warning test.." + random(5);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
+        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
         testLogger.info(message1);
         testLogger.warn(message2);
 
@@ -78,8 +104,8 @@ public void simpleAppending() {
     @Test
     public void simpleGzipAppending() {
         String token = "aBcDeFgHiJkLmNoPqRsTGzIp";
-        String type = "awesomeGzipType";
-        String loggerName = "simpleGzipAppending";
+        String type = "awesomeGzipType" + random(8);
+        String loggerName = "simpleGzipAppending" + random(8);
         int drainTimeout = 1;
         String message1 = "Testing.." + random(5);
         String message2 = "Warning test.." + random(5);
@@ -98,8 +124,8 @@ public void simpleGzipAppending() {
     @Test
     public void validateAdditionalFields() {
         String token = "validatingAdditionalFields";
-        String type = "willTryWithOrWithoutEnvironmentVariables";
-        String loggerName = "additionalLogger";
+        String type = "willTryWithOrWithoutEnvironmentVariables" + random(8);
+        String loggerName = "additionalLogger" + random(8);
         int drainTimeout = 1;
         String message1 = "Just a log - " + random(5);
         Map<String,String > additionalFields = new HashMap<>();
@@ -107,7 +133,7 @@ public void validateAdditionalFields() {
         additionalFields.put("java_home", System.getenv("JAVA_HOME"));
         additionalFields.put("testing", "yes");
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, additionalFieldsString);
+        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, additionalFieldsString, false);
         testLogger.info(message1);
 
         sleepSeconds(2 * drainTimeout);
@@ -121,12 +147,12 @@ public void validateAdditionalFields() {
     @Test
     public void existingHostname() throws Exception {
         String token = "checkingHostname";
-        String type = "withOrWithoutHostnamr";
-        String loggerName = "runningOutOfIdeasHere";
+        String type = "withOrWithoutHostnamr" + random(8);
+        String loggerName = "runningOutOfIdeasHere" + random(8);
         int drainTimeout = 1;
         String message1 = "Hostname log - " + random(5);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, true, false, null);
+        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, true, false, null, false);
         testLogger.info(message1);
 
         // Sleep double time the drain timeout
@@ -143,12 +169,12 @@ public void existingHostname() throws Exception {
     @Test
     public void existingLine() {
         String token = "checkingLine";
-        String type = "withLineType";
-        String loggerName = "test";
+        String type = "withLineType" + random(8);
+        String loggerName = "test" + random(8);
         int drainTimeout = 1;
         String message1 = "Hostname log - " + random(5);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, true, null);
+        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, true, null, false);
         testLogger.info(message1);
 
         // Sleep double time the drain timeout
@@ -165,13 +191,13 @@ public void existingLine() {
     @Test
     public void sendException() {
         String token = "checkingExceptions";
-        String type = "badType";
-        String loggerName = "exceptionProducer";
+        String type = "badType" + random(8);
+        String loggerName = "exceptionProducer" + random(8);
         int drainTimeout = 1;
         Throwable exception = null;
         String message1 = "This is not an int..";
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
+        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
 
         try {
             Integer.parseInt(message1);
@@ -196,8 +222,8 @@ public void sendException() {
     @Test
     public void testMDC() {
         String token = "mdcTokensAreTheBest";
-        String type = "mdcType";
-        String loggerName = "mdcTesting";
+        String type = "mdcType" + random(8);
+        String loggerName = "mdcTesting" + random(8);
         int drainTimeout = 1;
         String message1 = "Simple log line - " + random(5);
         String mdcKey = "mdc-key";
@@ -208,7 +234,7 @@ public void testMDC() {
         mdcKv.put("logger", "Doesn't matter");
         MDC.put(mdcKey, mdcValue);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
+        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
         testLogger.info(message1);
 
         sleepSeconds(2 * drainTimeout);
@@ -222,14 +248,14 @@ public void testMDC() {
     @Test
     public void testMarker() {
         String token = "markerToken";
-        String type = "markerType";
-        String loggerName = "markerTesting";
+        String type = "markerType" + random(8);
+        String loggerName = "markerTesting" + random(8);
         String markerKey = "marker";
         String markerTestValue = "MyMarker";
         int drainTimeout = 1;
         String message1 = "Simple log line - "+random(5);
         Marker marker = MarkerFactory.getMarker(markerTestValue);
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
+        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
 
         testLogger.info(marker, message1);
         sleepSeconds(2 * drainTimeout);
@@ -240,46 +266,18 @@ public void testMarker() {
         assertThat(logRequest.getStringFieldOrNull(markerKey)).isEqualTo(markerTestValue);
     }
 
-    @Test
-    public void testJsonMarkers() {
-        String token = "markersJsonToken";
-        String type = "markersJsonType";
-        String loggerName = "markersJsonTesting";
-        String markerKey = "marker";
-        int drainTimeout = 1;
-        String message1 = "Simple log line - " + random(5);
-
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
-        LogzioLogbackAppender logzioLogbackAppender =
-                (LogzioLogbackAppender)((ch.qos.logback.classic.Logger)testLogger).getAppender("LogzioLogbackAppender");
-        logzioLogbackAppender.setMarkersFormat("json");
-
-        Map markerMap = new HashMap();
-        markerMap.put("customkey_str", "value1");
-        markerMap.put("projectid_int", 5);
-        testLogger.info(MarkerFactory.getMarker(markerMap.toString()), message1);
-
-        sleepSeconds(2 * drainTimeout);
-
-        mockListener.assertNumberOfReceivedMsgs(1);
-        MockLogzioBulkListener.LogRequest logRequest = mockListener.assertLogReceivedByMessage(message1);
-        mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr);
-        assertThat(logRequest.getStringFieldOrNull("customkey_str")).isEqualTo("value1");
-        assertThat(logRequest.getStringFieldOrNull("projectid_int")).isEqualTo("5");
-    }
-
     @Test
     public void testContextReset() {
         logger.info("context.reset() is called when logback loads a new logback.xml in-flight");
         String token = "testingContextReset";
-        String type = "contextResetType";
-        String loggerName = "ContextResetLogger";
+        String type = "contextResetType" + random(8);
+        String loggerName = "ContextResetLogger" + random(8);
         int drainTimeout = 1;
 
         String message1 = "Before Reset Line - " + random(5);
         String message2 = "After Reset Line - " + random(5);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
+        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
 
         testLogger.info(message1);
         sleepSeconds(2 * drainTimeout);
@@ -294,7 +292,7 @@ public void testContextReset() {
         loggerContext.reset();
 
         // We need to add the appender again
-        testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null);
+        testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
 
         testLogger.warn(message2);
         sleepSeconds(2 * drainTimeout);
@@ -307,12 +305,12 @@ public void testContextReset() {
     @Test
     public void testTokenAndLogzioUrlFromSystemEnvironment() {
         String token = System.getenv("JAVA_HOME");
-        String type = "testType";
-        String loggerName = "testLogger";
+        String type = "testType" + random(8);
+        String loggerName = "testLogger" + random(8);
         int drainTimeout = 1;
 
         String message1 = "Just a log - " + random(5);
-        Logger testLogger = createLogger("$JAVA_HOME", type, loggerName, drainTimeout, false, false, null);
+        Logger testLogger = createLogger("$JAVA_HOME", type, loggerName, drainTimeout, false, false, null, false);
 
         testLogger.info(message1);
 
@@ -323,6 +321,41 @@ public void testTokenAndLogzioUrlFromSystemEnvironment() {
         mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr);
     }
 
+    @Test
+    public void checkExactStackTrace() throws Exception {
+        final CountDownLatch countDownLatch = new CountDownLatch(1);
+        final MyRunner.ExceptionGenerator exceptionGenerator = new MyRunner.ExceptionGenerator();
+
+        // We need to generate an exception with constant stack trace
+        new Thread(new MyRunner(countDownLatch, exceptionGenerator)).start();
+        countDownLatch.await();
+
+        String token = "exceptionToken";
+        String type = "stacktraceType" + random(8);
+        String loggerName = "traceLogger" + random(8);
+        int drainTimeout = 1;
+
+        String message1 = "Any line change here can cause the test to break";
+
+        String expectedException = "java.lang.RuntimeException: Got NPE!\n" +
+                "\tat io.logz.logback.MyRunner$ExceptionGenerator.generateNPE(MyRunner.java:33)\n" +
+                "\tat io.logz.logback.MyRunner.run(MyRunner.java:18)\n" +
+                "\tat java.lang.Thread.run(Thread.java:748)\n" +
+                "Caused by: java.lang.NullPointerException: null\n" +
+                "\tat io.logz.logback.MyRunner$ExceptionGenerator.generateNPE(MyRunner.java:31)\n" +
+                "\t... 2 common frames omitted\n";
+
+        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
+
+        testLogger.info(message1, exceptionGenerator.getE());
+        sleepSeconds(drainTimeout * 2);
+
+        mockListener.assertNumberOfReceivedMsgs(1);
+        MockLogzioBulkListener.LogRequest logRequest = mockListener.assertLogReceivedByMessage(message1);
+        mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr);
+        assertThat(logRequest.getStringFieldOrNull("exception")).isEqualTo(expectedException);
+    }
+
     private void assertAdditionalFields(MockLogzioBulkListener.LogRequest logRequest, Map<String, String> additionalFields) {
         additionalFields.forEach((field, value) -> {
             String fieldValueInLog = logRequest.getStringFieldOrNull(field);
@@ -333,4 +366,33 @@ private void assertAdditionalFields(MockLogzioBulkListener.LogRequest logRequest
         });
     }
 
+    private Logger createLogger(String token, String type, String loggerName, Integer drainTimeout,
+                                boolean addHostname, boolean line, String additionalFields, boolean compressRequests) {
+        logger.info("Creating logger {}. token={}, type={}, drainTimeout={}, addHostname={}, line={}, additionalFields={} ",
+                loggerName, token, type, drainTimeout, addHostname, line, additionalFields);
+
+        ch.qos.logback.classic.Logger logbackLogger =  (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(loggerName);
+        Context logbackContext = logbackLogger.getLoggerContext();
+        logzioLogbackAppender.setContext(logbackContext);
+        logzioLogbackAppender.setToken(token);
+        logzioLogbackAppender.setLogzioType(type);
+        logzioLogbackAppender.setDebug(true);
+        logzioLogbackAppender.setLine(line);
+        logzioLogbackAppender.setLogzioUrl("http://" + mockListener.getHost() + ":" + mockListener.getPort());
+        logzioLogbackAppender.setAddHostname(addHostname);
+        logzioLogbackAppender.setCompressRequests(compressRequests);
+        logzioLogbackAppender.setName("LogzioLogbackAppender");
+        if (drainTimeout != null) {
+            logzioLogbackAppender.setDrainTimeoutSec(drainTimeout);
+        }
+        if (additionalFields != null) {
+            logzioLogbackAppender.setAdditionalFields(additionalFields);
+        }
+        logzioLogbackAppender.start();
+        assertThat(logzioLogbackAppender.isStarted()).isTrue();
+        logbackLogger.addAppender(logzioLogbackAppender);
+        logbackLogger.setAdditive(false);
+        return logbackLogger;
+    }
+
 }
\ No newline at end of file

From 7badf8c8495aafa788bc7b974589f93cb2412fe5 Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Tue, 18 Sep 2018 11:21:44 +0300
Subject: [PATCH 02/14] opt imports, change code style

---
 .../logz/logback/LogzioLogbackAppender.java   | 44 +++++++++----------
 .../logz/logback/BaseLogbackAppenderTest.java |  5 +--
 .../logback/LogzioLogbackAppenderTest.java    | 10 +++--
 3 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
index 91a1cdc..f382fa4 100644
--- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java
+++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -204,7 +204,14 @@ public void setGcPersistedQueueFilesIntervalSeconds(int gcPersistedQueueFilesInt
     @Override
     public void start() {
         setHostname();
-        LogzioSender.Builder logzioSenderBuilder = getSenderBuilder();
+        HttpsRequestConfiguration conf;
+        try {
+            conf = getHttpsRequestConfiguration();
+        } catch (LogzioParameterErrorException e) {
+            addError("Some of the configuration parameters of logz.io is wrong: " + e.getMessage(), e);
+            return;
+        }
+        LogzioSender.Builder logzioSenderBuilder = getSenderBuilder(conf);
         if (inMemoryQueue) {
             if (!validateInMemoryQueueCapacityBytes()) {
                 return;
@@ -243,15 +250,13 @@ public void start() {
         super.start();
     }
 
-    private LogzioSender.Builder getSenderBuilder() {
-        SenderStatusReporter reporter = new StatusReporter();
-        HttpsRequestConfiguration conf = getHttpsRequestConfiguration();
+    private LogzioSender.Builder getSenderBuilder(HttpsRequestConfiguration conf) {
         return LogzioSender
                     .builder()
                     .setDebug(debug)
                     .setDrainTimeoutSec(drainTimeoutSec)
                     .setHttpsRequestConfiguration(conf)
-                    .setReporter(reporter)
+                    .setReporter(new StatusReporter())
                     .setTasksExecutor(context.getScheduledExecutorService());
     }
 
@@ -305,27 +310,22 @@ private void setHostname() {
         }
     }
 
-    private HttpsRequestConfiguration getHttpsRequestConfiguration() {
-        try {
-            return HttpsRequestConfiguration
-                    .builder()
-                    .setLogzioListenerUrl(logzioUrl)
-                    .setSocketTimeout(socketTimeout)
-                    .setLogzioType(logzioType)
-                    .setLogzioToken(logzioToken)
-                    .setConnectTimeout(connectTimeout)
-                    .setCompressRequests(compressRequests)
-                    .build();
-        } catch (LogzioParameterErrorException e) {
-            addError("Some of the configuration parameters of logz.io is wrong: " + e.getMessage(), e);
-            return null;
-        }
+    private HttpsRequestConfiguration getHttpsRequestConfiguration() throws LogzioParameterErrorException {
+        return HttpsRequestConfiguration
+                .builder()
+                .setLogzioListenerUrl(logzioUrl)
+                .setSocketTimeout(socketTimeout)
+                .setLogzioType(logzioType)
+                .setLogzioToken(logzioToken)
+                .setConnectTimeout(connectTimeout)
+                .setCompressRequests(compressRequests)
+                .build();
     }
 
     @Override
     public void stop() {
         if (logzioSender != null) logzioSender.stop();
-        if ( throwableProxyConverter != null ) throwableProxyConverter.stop();
+        if (throwableProxyConverter != null) throwableProxyConverter.stop();
         super.stop();
     }
 
@@ -334,7 +334,7 @@ private String getValueFromSystemEnvironmentIfNeeded(String value) {
             String variableName = value.replace("$", "");
             String envVariable = System.getenv(variableName);
 
-            if(envVariable == null || envVariable.isEmpty()) {
+            if (envVariable == null || envVariable.isEmpty()) {
                 envVariable = System.getProperty(variableName);
             }
             return envVariable;
diff --git a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
index 4e2b59d..c305338 100644
--- a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
@@ -1,6 +1,5 @@
 package io.logz.logback;
 
-import ch.qos.logback.core.Context;
 import io.logz.test.MockLogzioBulkListener;
 import org.junit.After;
 import org.junit.Before;
@@ -9,8 +8,6 @@
 
 import java.util.UUID;
 
-import static org.assertj.core.api.Assertions.assertThat;
-
 /**
  * @author MarinaRazumovsky
  */
@@ -18,7 +15,7 @@ public abstract class BaseLogbackAppenderTest {
 
     private final static Logger logger = LoggerFactory.getLogger(BaseLogbackAppenderTest.class);
 
-    MockLogzioBulkListener mockListener;
+    protected MockLogzioBulkListener mockListener;
 
     @Before
     public void startMockListener() throws Exception {
diff --git a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
index 5101f54..bec8308 100644
--- a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
@@ -1,6 +1,5 @@
 package io.logz.logback;
 
-
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.LoggerContext;
 import ch.qos.logback.core.Context;
@@ -32,6 +31,7 @@ public class LogzioLogbackAppenderTest extends BaseLogbackAppenderTest {
     private final static Logger logger = LoggerFactory.getLogger(LogzioLogbackAppenderTest.class);
     private LogzioLogbackAppender logzioLogbackAppender;
 
+
     @Parameterized.Parameters
     public static Collection<Object[]> logzioSenderBuilders() {
         List<LogzioLogbackAppender> appenders = new ArrayList<>();
@@ -48,6 +48,9 @@ public static Collection<Object[]> logzioSenderBuilders() {
     }
 
     public LogzioLogbackAppenderTest(LogzioLogbackAppender logzioLogbackAppender) {
+        if (this.logzioLogbackAppender != null) {
+            this.logzioLogbackAppender.stop();
+        }
         this.logzioLogbackAppender = logzioLogbackAppender;
     }
 
@@ -354,6 +357,7 @@ public void checkExactStackTrace() throws Exception {
         MockLogzioBulkListener.LogRequest logRequest = mockListener.assertLogReceivedByMessage(message1);
         mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr);
         assertThat(logRequest.getStringFieldOrNull("exception")).isEqualTo(expectedException);
+
     }
 
     private void assertAdditionalFields(MockLogzioBulkListener.LogRequest logRequest, Map<String, String> additionalFields) {
@@ -367,7 +371,8 @@ private void assertAdditionalFields(MockLogzioBulkListener.LogRequest logRequest
     }
 
     private Logger createLogger(String token, String type, String loggerName, Integer drainTimeout,
-                                boolean addHostname, boolean line, String additionalFields, boolean compressRequests) {
+                                boolean addHostname, boolean line, String additionalFields,
+                                boolean compressRequests) {
         logger.info("Creating logger {}. token={}, type={}, drainTimeout={}, addHostname={}, line={}, additionalFields={} ",
                 loggerName, token, type, drainTimeout, addHostname, line, additionalFields);
 
@@ -394,5 +399,4 @@ private Logger createLogger(String token, String type, String loggerName, Intege
         logbackLogger.setAdditive(false);
         return logbackLogger;
     }
-
 }
\ No newline at end of file

From fcb20fdf25caf883a18fff251b5982c89ce63066 Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Tue, 18 Sep 2018 11:33:45 +0300
Subject: [PATCH 03/14] fix travis

---
 src/main/java/io/logz/logback/LogzioLogbackAppender.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
index f382fa4..d1511d5 100644
--- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java
+++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -112,7 +112,6 @@ public void setFileSystemFullPercentThreshold(int fileSystemFullPercentThreshold
     }
 
     /**
-     *
      * @param bufferDir: queue dir path
      * @deprecated use {@link #setQueueDir(String)}
      */

From ecd544ba6493e2110081bede8941bdbf091b4f34 Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Tue, 18 Sep 2018 11:38:02 +0300
Subject: [PATCH 04/14] fix travis

---
 src/main/java/io/logz/logback/LogzioLogbackAppender.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
index c1050f9..1ade691 100644
--- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java
+++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -18,6 +18,7 @@
 import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -36,12 +37,11 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEv
     private static final String EXCEPTION = "exception";
     private static final String FORMAT_TEXT = "text";
     private static final String FORMAT_JSON = "json";
-    private static final String FORMAT_IGNORE = "ignore";
     private static final int DONT_LIMIT_QUEUE_SPACE = -1;
     private static final int LOWER_PERCENTAGE_FS_SPACE = 1;
     private static final int UPPER_PERCENTAGE_FS_SPACE = 100;
 
-    private static final Set<String> reservedFields =  new HashSet<>(Arrays.asList(new String[] {TIMESTAMP,LOGLEVEL, MARKER, MESSAGE,LOGGER,THREAD,EXCEPTION}));
+    private static final Set<String> reservedFields =  new HashSet<>(Arrays.asList(TIMESTAMP,LOGLEVEL, MARKER, MESSAGE,LOGGER,THREAD,EXCEPTION));
 
     private LogzioSender logzioSender;
     private ThrowableProxyConverter throwableProxyConverter;
@@ -233,7 +233,7 @@ public void start() {
         logzioSender.start();
         throwableProxyConverter = new ThrowableProxyConverter();
         lineOfCallerConverter = new LineOfCallerConverter();
-        throwableProxyConverter.setOptionList(Arrays.asList("full"));
+        throwableProxyConverter.setOptionList(Collections.singletonList("full"));
         throwableProxyConverter.start();
         super.start();
     }

From 88574f77e84e141a1e18127269e8ea985667ff50 Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Thu, 20 Sep 2018 15:55:38 +0300
Subject: [PATCH 05/14] typo

---
 src/main/java/io/logz/logback/LogzioLogbackAppender.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
index 1ade691..9dac5da 100644
--- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java
+++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -196,7 +196,7 @@ public void start() {
         try {
             conf = getHttpsRequestConfiguration();
         } catch (LogzioParameterErrorException e) {
-            addError("Some of the configuration parameters of logz.io is wrong: " + e.getMessage(), e);
+            addError("Some of the configuration parameters of logz.io are wrong: " + e.getMessage(), e);
             return;
         }
         LogzioSender.Builder logzioSenderBuilder = getSenderBuilder(conf);

From f02659871909c8e6d80b2bb384edf974bad01c6b Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Tue, 2 Oct 2018 16:18:30 +0300
Subject: [PATCH 06/14] support in memory log count limit

---
 .../logz/logback/LogzioLogbackAppender.java   |  18 +++-
 .../logz/logback/BaseLogbackAppenderTest.java |  51 ++++++++-
 .../java/io/logz/logback/InMemoryTest.java    |  59 ++++++++++
 .../logback/LogzioLogbackAppenderTest.java    | 102 +++++-------------
 4 files changed, 151 insertions(+), 79 deletions(-)
 create mode 100644 src/test/java/io/logz/logback/InMemoryTest.java

diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
index 9dac5da..5a8f5e5 100644
--- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java
+++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -63,6 +63,7 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEv
     private boolean compressRequests = false;
     private boolean inMemoryQueue = false;
     private long inMemoryQueueCapacityBytes = 100 * 1024 * 1024;
+    private long inMemoryLogsCountLimit = DONT_LIMIT_QUEUE_SPACE;
     private int gcPersistedQueueFilesIntervalSeconds = 30;
     private String format = FORMAT_TEXT;
 
@@ -149,6 +150,14 @@ public long getInMemoryQueueCapacityBytes() {
         return inMemoryQueueCapacityBytes;
     }
 
+    public void setInMemoryLogsCountLimit(long inMemoryLogsCountLimit) {
+        this.inMemoryLogsCountLimit = inMemoryLogsCountLimit;
+    }
+
+    public long getInMemoryLogsCountLimit() {
+        return inMemoryLogsCountLimit;
+    }
+
     public boolean isCompressRequests() { return compressRequests; }
 
     public void setCompressRequests(boolean compressRequests) { this.compressRequests = compressRequests; }
@@ -201,12 +210,13 @@ public void start() {
         }
         LogzioSender.Builder logzioSenderBuilder = getSenderBuilder(conf);
         if (inMemoryQueue) {
-            if (!validateInMemoryQueueCapacityBytes()) {
+            if (!validateInMemoryThresholds()) {
                 return;
             }
             logzioSenderBuilder
                 .withInMemoryQueue()
                     .setCapacityInBytes(inMemoryQueueCapacityBytes)
+                    .setLogsCountLimit(inMemoryLogsCountLimit)
                 .endInMemoryQueue();
         } else {
             if (!validateFsPercentThreshold()) {
@@ -269,11 +279,15 @@ private File getQueueFile() {
         return new File(queueDir,"logzio-logback-appender");
     }
 
-    private boolean validateInMemoryQueueCapacityBytes() {
+    private boolean validateInMemoryThresholds() {
         if (inMemoryQueueCapacityBytes <= 0 && inMemoryQueueCapacityBytes != DONT_LIMIT_QUEUE_SPACE) {
             addError("inMemoryQueueCapacityBytes should be a non zero integer or -1");
             return false;
         }
+        if (inMemoryLogsCountLimit <= 0 && inMemoryLogsCountLimit != DONT_LIMIT_QUEUE_SPACE) {
+            addError("inMemoryLogsCountLimit should be a non zero integer or -1");
+            return false;
+        }
         return true;
     }
 
diff --git a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
index c305338..06f314e 100644
--- a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
@@ -1,21 +1,24 @@
 package io.logz.logback;
 
+import ch.qos.logback.core.Context;
 import io.logz.test.MockLogzioBulkListener;
 import org.junit.After;
 import org.junit.Before;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Map;
 import java.util.UUID;
 
+import static org.assertj.core.api.Assertions.assertThat;
+
 /**
  * @author MarinaRazumovsky
  */
 public abstract class BaseLogbackAppenderTest {
-
-    private final static Logger logger = LoggerFactory.getLogger(BaseLogbackAppenderTest.class);
-
+    protected final static Logger logger = LoggerFactory.getLogger(BaseLogbackAppenderTest.class);
     protected MockLogzioBulkListener mockListener;
+    protected enum QueueType { DISK, MEMORY }
 
     @Before
     public void startMockListener() throws Exception {
@@ -28,7 +31,7 @@ public void stopMockListener() {
         mockListener.stop();
     }
 
-    void sleepSeconds(int seconds) {
+    protected void sleepSeconds(int seconds) {
         logger.info("Sleeping {} [sec]...", seconds);
         try {
             Thread.sleep(seconds * 1000);
@@ -40,4 +43,44 @@ void sleepSeconds(int seconds) {
     protected String random(int numberOfChars) {
         return UUID.randomUUID().toString().substring(0, numberOfChars-1);
     }
+
+    protected void assertAdditionalFields(MockLogzioBulkListener.LogRequest logRequest, Map<String, String> additionalFields) {
+        additionalFields.forEach((field, value) -> {
+            String fieldValueInLog = logRequest.getStringFieldOrNull(field);
+            assertThat(fieldValueInLog)
+                    .describedAs("Field '{}' in Log [{}]", field, logRequest.getJsonObject().toString())
+                    .isNotNull()
+                    .isEqualTo(value);
+        });
+    }
+
+    protected Logger createLogger(LogzioLogbackAppender logzioLogbackAppender, String token, String type, String loggerName, Integer drainTimeout,
+                                boolean addHostname, boolean line, String additionalFields,
+                                boolean compressRequests) {
+        logger.info("Creating logger {}. token={}, type={}, drainTimeout={}, addHostname={}, line={}, additionalFields={} ",
+                loggerName, token, type, drainTimeout, addHostname, line, additionalFields);
+
+        ch.qos.logback.classic.Logger logbackLogger =  (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(loggerName);
+        Context logbackContext = logbackLogger.getLoggerContext();
+        logzioLogbackAppender.setContext(logbackContext);
+        logzioLogbackAppender.setToken(token);
+        logzioLogbackAppender.setLogzioType(type);
+        logzioLogbackAppender.setDebug(true);
+        logzioLogbackAppender.setLine(line);
+        logzioLogbackAppender.setLogzioUrl("http://" + mockListener.getHost() + ":" + mockListener.getPort());
+        logzioLogbackAppender.setAddHostname(addHostname);
+        logzioLogbackAppender.setCompressRequests(compressRequests);
+        logzioLogbackAppender.setName("LogzioLogbackAppender");
+        if (drainTimeout != null) {
+            logzioLogbackAppender.setDrainTimeoutSec(drainTimeout);
+        }
+        if (additionalFields != null) {
+            logzioLogbackAppender.setAdditionalFields(additionalFields);
+        }
+        logzioLogbackAppender.start();
+        assertThat(logzioLogbackAppender.isStarted()).isTrue();
+        logbackLogger.addAppender(logzioLogbackAppender);
+        logbackLogger.setAdditive(false);
+        return logbackLogger;
+    }
 }
diff --git a/src/test/java/io/logz/logback/InMemoryTest.java b/src/test/java/io/logz/logback/InMemoryTest.java
new file mode 100644
index 0000000..f78dede
--- /dev/null
+++ b/src/test/java/io/logz/logback/InMemoryTest.java
@@ -0,0 +1,59 @@
+package io.logz.logback;
+
+import ch.qos.logback.classic.Level;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+
+import java.nio.charset.StandardCharsets;
+
+public class InMemoryTest extends BaseLogbackAppenderTest {
+    private LogzioLogbackAppender logzioLogbackAppender;
+
+    @Before
+    public void setUp() {
+        logzioLogbackAppender = new LogzioLogbackAppender();
+        logzioLogbackAppender.setInMemoryQueue(true);
+    }
+
+    @Test
+    public void validateQueueCapacityLimit() {
+        String token = "verifyCapacityInBytesToken";
+        String type = random(8);
+        String loggerName = "verifyCapacityInBytes" + random(8);
+        int drainTimeout = 2;
+        String message1 = "Testing.." + random(5);
+        String message2 = "Don't get here test! ";
+
+        logzioLogbackAppender.setInMemoryQueueCapacityBytes(message1.getBytes(StandardCharsets.UTF_8).length);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, false);
+
+        testLogger.info(message1);
+        testLogger.warn(message2);
+
+        sleepSeconds(drainTimeout * 2);
+        mockListener.assertNumberOfReceivedMsgs(1);
+        mockListener.assertLogReceivedIs(message1, token, type, loggerName, Level.INFO.levelStr);
+    }
+
+    @Test
+    public void validateQueueLogCountLimit() {
+        String token = "verifyLogCountLimit";
+        String type = random(8);
+        String loggerName = "verifyLogCountLimit" + random(8);
+        int drainTimeout = 1;
+        String message1 = "Testing.." + random(5);
+        String message2 = "Don't get here test! ";
+
+        logzioLogbackAppender.setInMemoryLogsCountLimit(1);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, false);
+
+        testLogger.info(message1);
+        testLogger.warn(message2);
+
+        sleepSeconds(drainTimeout * 2);
+        mockListener.assertNumberOfReceivedMsgs(1);
+        mockListener.assertLogReceivedIs(message1, token, type, loggerName, Level.INFO.levelStr);
+    }
+}
+
diff --git a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
index 3b4e2df..9834187 100644
--- a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
@@ -2,9 +2,9 @@
 
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.LoggerContext;
-import ch.qos.logback.core.Context;
 import io.logz.sender.com.google.gson.Gson;
 import io.logz.test.MockLogzioBulkListener;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -18,7 +18,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 
@@ -27,31 +26,28 @@
 
 @RunWith(Parameterized.class)
 public class LogzioLogbackAppenderTest extends BaseLogbackAppenderTest {
-
-    private final static Logger logger = LoggerFactory.getLogger(LogzioLogbackAppenderTest.class);
     private LogzioLogbackAppender logzioLogbackAppender;
+    private QueueType queueType;
 
+    @Before
+    public void setUp() {
+        logzioLogbackAppender = new LogzioLogbackAppender();
+        if (queueType == QueueType.MEMORY) {
+            logzioLogbackAppender.setInMemoryQueue(true);
+        }
+    }
 
     @Parameterized.Parameters
-    public static Collection<Object[]> logzioSenderBuilders() {
-        List<LogzioLogbackAppender> appenders = new ArrayList<>();
-        appenders.add(new LogzioLogbackAppender());
-        LogzioLogbackAppender inMemoryQueueApeender = new LogzioLogbackAppender();
-        inMemoryQueueApeender.setInMemoryQueue(true);
-        appenders.add(inMemoryQueueApeender);
-
-        Collection<Object[]> result = new ArrayList<>();
-        for (LogzioLogbackAppender appender : appenders) {
-            result.add(new Object[]{appender});
+    public static Collection<QueueType[]> logzioSenderBuilders() {
+        Collection<QueueType[]> queueTypes = new ArrayList<>();
+        for (QueueType type : QueueType.values()) {
+            queueTypes.add(new QueueType[]{type});
         }
-        return result;
+        return queueTypes;
     }
 
-    public LogzioLogbackAppenderTest(LogzioLogbackAppender logzioLogbackAppender) {
-        if (this.logzioLogbackAppender != null) {
-            this.logzioLogbackAppender.stop();
-        }
-        this.logzioLogbackAppender = logzioLogbackAppender;
+    public LogzioLogbackAppenderTest(QueueType queueType) {
+        this.queueType = queueType;
     }
 
     @Test
@@ -70,7 +66,7 @@ public void validateJsonMessage(){
         String message1 = new Gson().toJson(map);
 
         logzioLogbackAppender.setFormat("json");
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, false);
 
         testLogger.info(message1);
 
@@ -93,7 +89,7 @@ public void simpleAppending() throws Exception {
         String message1 = "Testing.." + random(5);
         String message2 = "Warning test.." + random(5);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, false);
         testLogger.info(message1);
         testLogger.warn(message2);
 
@@ -113,7 +109,7 @@ public void simpleGzipAppending() throws Exception {
         String message1 = "Testing.." + random(5);
         String message2 = "Warning test.." + random(5);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, true);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, true);
         testLogger.info(message1);
         testLogger.warn(message2);
 
@@ -136,7 +132,7 @@ public void validateAdditionalFields() throws Exception {
         additionalFields.put("java_home", System.getenv("JAVA_HOME"));
         additionalFields.put("testing", "yes");
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, additionalFieldsString, false);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, additionalFieldsString, false);
         testLogger.info(message1);
 
         sleepSeconds(2 * drainTimeout);
@@ -155,7 +151,7 @@ public void existingHostname() throws Exception {
         int drainTimeout = 1;
         String message1 = "Hostname log - " +  random(5);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, true, false, null, false);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, true, false, null, false);
         testLogger.info(message1);
 
         // Sleep double time the drain timeout
@@ -177,7 +173,7 @@ public void existingLine() throws Exception {
         int drainTimeout = 1;
         String message1 = "Hostname log - " +  random(5);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, true, null, false);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, true, null, false);
         testLogger.info(message1);
 
         // Sleep double time the drain timeout
@@ -200,7 +196,7 @@ public void sendException() throws Exception {
         Throwable exception = null;
         String message1 = "This is not an int..";
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, false);
 
         try {
             Integer.parseInt(message1);
@@ -237,7 +233,7 @@ public void testMDC() throws Exception {
         mdcKv.put("logger", "Doesn't matter");
         MDC.put(mdcKey, mdcValue);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, false);
         testLogger.info(message1);
 
         sleepSeconds(2 * drainTimeout);
@@ -259,7 +255,7 @@ public void testMarker() throws Exception {
         String message1 = "Simple log line - "+random(5);
 
         Marker marker = MarkerFactory.getMarker(markerTestValue);
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, false);
 
         testLogger.info(marker, message1);
         sleepSeconds(2 * drainTimeout);
@@ -281,7 +277,7 @@ public void testContextReset() {
         String message1 = "Before Reset Line - "+random(5);
         String message2 = "After Reset Line - "+random(5);
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, false);
 
         testLogger.info(message1);
         sleepSeconds(2 * drainTimeout);
@@ -296,7 +292,7 @@ public void testContextReset() {
         loggerContext.reset();
 
         // We need to add the appender again
-        testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
+        testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, false);
 
         testLogger.warn(message2);
         sleepSeconds(2 * drainTimeout);
@@ -314,7 +310,7 @@ public void testTokenAndLogzioUrlFromSystemEnvironment() {
         int drainTimeout = 1;
 
         String message1 = "Just a log - " + random(5);
-        Logger testLogger = createLogger("$JAVA_HOME", type, loggerName, drainTimeout, false, false, null, false);
+        Logger testLogger = createLogger(logzioLogbackAppender, "$JAVA_HOME", type, loggerName, drainTimeout, false, false, null, false);
 
         testLogger.info(message1);
 
@@ -349,7 +345,7 @@ public void checkExactStackTrace() throws Exception {
                 "\tat io.logz.logback.MyRunner$ExceptionGenerator.generateNPE(MyRunner.java:31)\n" +
                 "\t... 2 common frames omitted\n";
 
-        Logger testLogger = createLogger(token, type, loggerName, drainTimeout, false, false, null, false);
+        Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, false);
 
         testLogger.info(message1, exceptionGenerator.getE());
         sleepSeconds(drainTimeout * 2);
@@ -360,44 +356,4 @@ public void checkExactStackTrace() throws Exception {
         assertThat(logRequest.getStringFieldOrNull("exception")).isEqualTo(expectedException);
 
     }
-
-    private void assertAdditionalFields(MockLogzioBulkListener.LogRequest logRequest, Map<String, String> additionalFields) {
-        additionalFields.forEach((field, value) -> {
-            String fieldValueInLog = logRequest.getStringFieldOrNull(field);
-            assertThat(fieldValueInLog)
-                    .describedAs("Field '{}' in Log [{}]", field, logRequest.getJsonObject().toString())
-                    .isNotNull()
-                    .isEqualTo(value);
-        });
-    }
-
-    private Logger createLogger(String token, String type, String loggerName, Integer drainTimeout,
-                                boolean addHostname, boolean line, String additionalFields,
-                                boolean compressRequests) {
-        logger.info("Creating logger {}. token={}, type={}, drainTimeout={}, addHostname={}, line={}, additionalFields={} ",
-                loggerName, token, type, drainTimeout, addHostname, line, additionalFields);
-
-        ch.qos.logback.classic.Logger logbackLogger =  (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(loggerName);
-        Context logbackContext = logbackLogger.getLoggerContext();
-        logzioLogbackAppender.setContext(logbackContext);
-        logzioLogbackAppender.setToken(token);
-        logzioLogbackAppender.setLogzioType(type);
-        logzioLogbackAppender.setDebug(true);
-        logzioLogbackAppender.setLine(line);
-        logzioLogbackAppender.setLogzioUrl("http://" + mockListener.getHost() + ":" + mockListener.getPort());
-        logzioLogbackAppender.setAddHostname(addHostname);
-        logzioLogbackAppender.setCompressRequests(compressRequests);
-        logzioLogbackAppender.setName("LogzioLogbackAppender");
-        if (drainTimeout != null) {
-            logzioLogbackAppender.setDrainTimeoutSec(drainTimeout);
-        }
-        if (additionalFields != null) {
-            logzioLogbackAppender.setAdditionalFields(additionalFields);
-        }
-        logzioLogbackAppender.start();
-        assertThat(logzioLogbackAppender.isStarted()).isTrue();
-        logbackLogger.addAppender(logzioLogbackAppender);
-        logbackLogger.setAdditive(false);
-        return logbackLogger;
-    }
 }
\ No newline at end of file

From 9d74e25eea0141f81f33a4556e180fd39cdaf602 Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Tue, 2 Oct 2018 16:19:03 +0300
Subject: [PATCH 07/14] update readme

---
 README.md | 1 +
 pom.xml   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 77e2cf5..e832ede 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,7 @@ This appender uses [LogzioSender](https://github.com/logzio/logzio-java-sender)
 | Parameter          | Default                              | Explained  |
 | ------------------ | ------------------------------------ | ----- |
 | **inMemoryQueueCapacityBytes**       | *1024 * 1024 * 100*                                | The amount of memory(bytes) we are allowed to use for the memory queue. If the value is -1 the sender will not limit the queue size.|
+| **inMemoryLogsCountLimit**       | *-1*                                | Number of logs we are allowed to have in the queue before dropping logs. If the value is -1 the sender will not limit the number of logs allowed.|
 | **inMemoryQueue**       | *false*                                | Set to true if the appender uses in memory queue. By default the appender uses disk queue|
 
 
diff --git a/pom.xml b/pom.xml
index 1c0ee71..4abf5dd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
     </scm>
 
     <properties>
-        <logzio-sender-version>1.0.16</logzio-sender-version>
+        <logzio-sender-version>1.0.17</logzio-sender-version>
     </properties>
 
     <build>

From 03db1f8b2acce6f2a376211cb78c280ac562ce47 Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Wed, 3 Oct 2018 15:14:14 +0300
Subject: [PATCH 08/14] add support for encoder

---
 pom.xml                                       |  6 ++
 .../logz/logback/LogzioLogbackAppender.java   | 38 ++++++++++-
 .../logz/logback/BaseLogbackAppenderTest.java | 22 +++++++
 .../logback/LogzioLogbackAppenderTest.java    | 63 +++++++++++++++++++
 4 files changed, 127 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 4abf5dd..98fe402 100644
--- a/pom.xml
+++ b/pom.xml
@@ -181,5 +181,11 @@
             <version>9.3.9.M1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>net.logstash.logback</groupId>
+            <artifactId>logstash-logback-encoder</artifactId>
+            <version>5.2</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>
\ No newline at end of file
diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
index 5a8f5e5..458fb61 100644
--- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java
+++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -4,6 +4,7 @@
 import ch.qos.logback.classic.pattern.ThrowableProxyConverter;
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.UnsynchronizedAppenderBase;
+import ch.qos.logback.core.encoder.Encoder;
 import com.google.common.base.Splitter;
 import io.logz.sender.HttpsRequestConfiguration;
 import io.logz.sender.LogzioSender;
@@ -16,6 +17,7 @@
 import java.io.File;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
@@ -66,11 +68,20 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEv
     private long inMemoryLogsCountLimit = DONT_LIMIT_QUEUE_SPACE;
     private int gcPersistedQueueFilesIntervalSeconds = 30;
     private String format = FORMAT_TEXT;
+    private Encoder<ILoggingEvent> encoder = null;
 
     public LogzioLogbackAppender() {
         super();
     }
 
+    public void setEncoder(Encoder<ILoggingEvent> encoder) {
+        this.encoder = encoder;
+    }
+
+    public Encoder<ILoggingEvent> getEncoder() {
+        return encoder;
+    }
+
     public String getFormat() {
         return format;
     }
@@ -344,7 +355,30 @@ private String getValueFromSystemEnvironmentIfNeeded(String value) {
         return value;
     }
 
-    private JsonObject formatMessageAsJson(ILoggingEvent loggingEvent) {
+    private void formatMessageAndSend(ILoggingEvent loggingEvent) {
+        try {
+            logzioSender.send(formatMessageAsJson(loggingEvent));
+        } catch (Exception e) {
+            addWarn("Failed to format json", e);
+        }
+
+    }
+
+    private JsonObject formatMessageAsJson(ILoggingEvent loggingEvent) throws Exception {
+        if (encoder == null) {
+            return formatMessageAsJsonInternal(loggingEvent);
+        }
+        return formatMessageAsJsonExternal(loggingEvent);
+    }
+
+
+    private JsonObject formatMessageAsJsonExternal(ILoggingEvent loggingEvent) throws Exception{
+        final byte[] payload = encoder.encode(loggingEvent);
+        JsonElement jsonElement = gson.fromJson(new String(payload, StandardCharsets.UTF_8), JsonElement.class);
+        return jsonElement.getAsJsonObject();
+    }
+
+    private JsonObject formatMessageAsJsonInternal(ILoggingEvent loggingEvent) {
         JsonObject logMessage;
 
         if (format.equals(FORMAT_JSON)) {
@@ -392,7 +426,7 @@ private JsonObject formatMessageAsJson(ILoggingEvent loggingEvent) {
     @Override
     protected void append(ILoggingEvent loggingEvent) {
         if (!loggingEvent.getLoggerName().contains("io.logz.sender")) {
-            logzioSender.send(formatMessageAsJson(loggingEvent));
+            formatMessageAndSend(loggingEvent);
         }
     }
 
diff --git a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
index 06f314e..df631bf 100644
--- a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
@@ -1,7 +1,15 @@
 package io.logz.logback;
 
+import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.Context;
 import io.logz.test.MockLogzioBulkListener;
+import net.logstash.logback.composite.ContextJsonProvider;
+import net.logstash.logback.composite.loggingevent.ArgumentsJsonProvider;
+import net.logstash.logback.composite.loggingevent.LoggingEventFormattedTimestampJsonProvider;
+import net.logstash.logback.composite.loggingevent.LoggingEventJsonProviders;
+import net.logstash.logback.composite.loggingevent.LoggingEventPatternJsonProvider;
+import net.logstash.logback.composite.loggingevent.MdcJsonProvider;
+import net.logstash.logback.composite.loggingevent.MessageJsonProvider;
 import org.junit.After;
 import org.junit.Before;
 import org.slf4j.Logger;
@@ -54,6 +62,16 @@ protected void assertAdditionalFields(MockLogzioBulkListener.LogRequest logReque
         });
     }
 
+    protected LoggingEventJsonProviders getCommonJsonProviders() {
+        LoggingEventJsonProviders jsonProviders = new LoggingEventJsonProviders();
+        jsonProviders.addPattern(new LoggingEventPatternJsonProvider());
+        jsonProviders.addArguments(new ArgumentsJsonProvider());
+        jsonProviders.addMessage(new MessageJsonProvider());
+        jsonProviders.addContext(new ContextJsonProvider<ILoggingEvent>());
+        jsonProviders.addMdc(new MdcJsonProvider());
+        return jsonProviders;
+    }
+
     protected Logger createLogger(LogzioLogbackAppender logzioLogbackAppender, String token, String type, String loggerName, Integer drainTimeout,
                                 boolean addHostname, boolean line, String additionalFields,
                                 boolean compressRequests) {
@@ -77,6 +95,10 @@ protected Logger createLogger(LogzioLogbackAppender logzioLogbackAppender, Strin
         if (additionalFields != null) {
             logzioLogbackAppender.setAdditionalFields(additionalFields);
         }
+        if (logzioLogbackAppender.getEncoder() != null) {
+            logzioLogbackAppender.getEncoder().setContext(logbackContext);
+            logzioLogbackAppender.getEncoder().start();
+        }
         logzioLogbackAppender.start();
         assertThat(logzioLogbackAppender.isStarted()).isTrue();
         logbackLogger.addAppender(logzioLogbackAppender);
diff --git a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
index 9834187..d5265e1 100644
--- a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
@@ -2,8 +2,27 @@
 
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Context;
+import ch.qos.logback.core.encoder.Encoder;
+import ch.qos.logback.core.spi.DeferredProcessingAware;
+import ch.qos.logback.core.status.Status;
+import com.fasterxml.jackson.core.JsonGenerator;
 import io.logz.sender.com.google.gson.Gson;
 import io.logz.test.MockLogzioBulkListener;
+import net.logstash.logback.composite.AbstractFieldJsonProvider;
+import net.logstash.logback.composite.AbstractPatternJsonProvider;
+import net.logstash.logback.composite.CompositeJsonFormatter;
+import net.logstash.logback.composite.FormattedTimestampJsonProvider;
+import net.logstash.logback.composite.JsonProvider;
+import net.logstash.logback.composite.JsonProviders;
+import net.logstash.logback.composite.loggingevent.LogLevelJsonProvider;
+import net.logstash.logback.composite.loggingevent.LoggerNameJsonProvider;
+import net.logstash.logback.composite.loggingevent.LoggingEventFormattedTimestampJsonProvider;
+import net.logstash.logback.composite.loggingevent.LoggingEventPatternJsonProvider;
+import net.logstash.logback.composite.loggingevent.MessageJsonProvider;
+import net.logstash.logback.composite.loggingevent.StackTraceJsonProvider;
+import net.logstash.logback.composite.loggingevent.ThreadNameJsonProvider;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -14,6 +33,7 @@
 import org.slf4j.Marker;
 import org.slf4j.MarkerFactory;
 
+import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -24,6 +44,8 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.fail;
 
+import net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder;
+
 @RunWith(Parameterized.class)
 public class LogzioLogbackAppenderTest extends BaseLogbackAppenderTest {
     private LogzioLogbackAppender logzioLogbackAppender;
@@ -354,6 +376,47 @@ public void checkExactStackTrace() throws Exception {
         MockLogzioBulkListener.LogRequest logRequest = mockListener.assertLogReceivedByMessage(message1);
         mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr);
         assertThat(logRequest.getStringFieldOrNull("exception")).isEqualTo(expectedException);
+    }
+
+    @Test
+    public void testEncoder() {
+        String token = "testEncoder";
+        String type = "testEncoder" + random(8);
+        String loggerName = "testEncoder" + random(8);
+        int drainTimeout = 1;
+        String message1 = "Just a log - " + random(5);
+
+        LoggingEventCompositeJsonEncoder encoder = new LoggingEventCompositeJsonEncoder();
+        encoder.getProviders().addProvider(new LogLevelJsonProvider());
+        encoder.getProviders().addProvider(new MessageJsonProvider());
+        encoder.getProviders().addProvider(withPattern(String.format("{ \"loglevel\": \"%s\" }", "INFO"), new LoggingEventPatternJsonProvider()));
+        encoder.getProviders().addProvider(withPattern(String.format("{ \"type\": \"%s\" }", type), new LoggingEventPatternJsonProvider()));
+        encoder.getProviders().addProvider(withPattern(String.format("{ \"logger\": \"%s\" }", loggerName), new LoggingEventPatternJsonProvider()));
+        encoder.getProviders().addProvider(withName("timestamp", new LoggingEventFormattedTimestampJsonProvider()));
+
+        logzioLogbackAppender.setEncoder(encoder);
+
+        Logger testLogger = createLogger(logzioLogbackAppender, "testEncoder", type, loggerName, drainTimeout, false, false, null, false);
+        encoder.start();
+
+        testLogger.info(message1);
+
+        sleepSeconds(2 * drainTimeout);
+
+        mockListener.assertNumberOfReceivedMsgs(1);
+        MockLogzioBulkListener.LogRequest logRequest = mockListener.assertLogReceivedByMessage(message1);
+        mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr);
+    }
+
+    private AbstractPatternJsonProvider<ILoggingEvent> withPattern(String pattern,
+                                                                   AbstractPatternJsonProvider<ILoggingEvent> provider) {
+        provider.setPattern(pattern);
+        return provider;
+    }
 
+    private <T extends DeferredProcessingAware> AbstractFieldJsonProvider<T> withName(String name,
+                                                                                      AbstractFieldJsonProvider<T> provider) {
+        provider.setFieldName(name);
+        return provider;
     }
 }
\ No newline at end of file

From 28d14a35d99ea502b0397d7dbd5d28bddd26e9f9 Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Thu, 4 Oct 2018 14:10:02 +0300
Subject: [PATCH 09/14] change term limit to capacity

---
 .../logz/logback/LogzioLogbackAppender.java   | 24 +++++++++----------
 .../logz/logback/BaseLogbackAppenderTest.java |  1 -
 .../java/io/logz/logback/InMemoryTest.java    | 10 ++++----
 .../logback/LogzioLogbackAppenderTest.java    | 15 +-----------
 4 files changed, 18 insertions(+), 32 deletions(-)

diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
index 458fb61..11d132a 100644
--- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java
+++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -39,7 +39,7 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEv
     private static final String EXCEPTION = "exception";
     private static final String FORMAT_TEXT = "text";
     private static final String FORMAT_JSON = "json";
-    private static final int DONT_LIMIT_QUEUE_SPACE = -1;
+    private static final int DONT_LIMIT_CAPACITY = -1;
     private static final int LOWER_PERCENTAGE_FS_SPACE = 1;
     private static final int UPPER_PERCENTAGE_FS_SPACE = 100;
 
@@ -65,7 +65,7 @@ public class LogzioLogbackAppender extends UnsynchronizedAppenderBase<ILoggingEv
     private boolean compressRequests = false;
     private boolean inMemoryQueue = false;
     private long inMemoryQueueCapacityBytes = 100 * 1024 * 1024;
-    private long inMemoryLogsCountLimit = DONT_LIMIT_QUEUE_SPACE;
+    private long inMemoryLogsCountCapacity = DONT_LIMIT_CAPACITY;
     private int gcPersistedQueueFilesIntervalSeconds = 30;
     private String format = FORMAT_TEXT;
     private Encoder<ILoggingEvent> encoder = null;
@@ -161,12 +161,12 @@ public long getInMemoryQueueCapacityBytes() {
         return inMemoryQueueCapacityBytes;
     }
 
-    public void setInMemoryLogsCountLimit(long inMemoryLogsCountLimit) {
-        this.inMemoryLogsCountLimit = inMemoryLogsCountLimit;
+    public void setInMemoryLogsCountCapacity(long inMemoryLogsCountCapacity) {
+        this.inMemoryLogsCountCapacity = inMemoryLogsCountCapacity;
     }
 
     public long getInMemoryLogsCountLimit() {
-        return inMemoryLogsCountLimit;
+        return inMemoryLogsCountCapacity;
     }
 
     public boolean isCompressRequests() { return compressRequests; }
@@ -227,7 +227,7 @@ public void start() {
             logzioSenderBuilder
                 .withInMemoryQueue()
                     .setCapacityInBytes(inMemoryQueueCapacityBytes)
-                    .setLogsCountLimit(inMemoryLogsCountLimit)
+                    .setLogsCountLimit(inMemoryLogsCountCapacity)
                 .endInMemoryQueue();
         } else {
             if (!validateFsPercentThreshold()) {
@@ -291,12 +291,12 @@ private File getQueueFile() {
     }
 
     private boolean validateInMemoryThresholds() {
-        if (inMemoryQueueCapacityBytes <= 0 && inMemoryQueueCapacityBytes != DONT_LIMIT_QUEUE_SPACE) {
-            addError("inMemoryQueueCapacityBytes should be a non zero integer or -1");
+        if (inMemoryQueueCapacityBytes <= 0 && inMemoryQueueCapacityBytes != DONT_LIMIT_CAPACITY) {
+            addError("inMemoryQueueCapacityBytes should be a non zero integer or " + DONT_LIMIT_CAPACITY);
             return false;
         }
-        if (inMemoryLogsCountLimit <= 0 && inMemoryLogsCountLimit != DONT_LIMIT_QUEUE_SPACE) {
-            addError("inMemoryLogsCountLimit should be a non zero integer or -1");
+        if (inMemoryLogsCountCapacity <= 0 && inMemoryLogsCountCapacity != DONT_LIMIT_CAPACITY) {
+            addError("inMemoryLogsCountCapacity should be a non zero integer or " + DONT_LIMIT_CAPACITY);
             return false;
         }
         return true;
@@ -304,8 +304,8 @@ private boolean validateInMemoryThresholds() {
 
     private boolean validateFsPercentThreshold() {
         if (!(fileSystemFullPercentThreshold >= LOWER_PERCENTAGE_FS_SPACE && fileSystemFullPercentThreshold <= UPPER_PERCENTAGE_FS_SPACE)) {
-            if (fileSystemFullPercentThreshold != DONT_LIMIT_QUEUE_SPACE) {
-                addError("fileSystemFullPercentThreshold should be a number between 1 and 100, or -1");
+            if (fileSystemFullPercentThreshold != DONT_LIMIT_CAPACITY) {
+                addError("fileSystemFullPercentThreshold should be a number between 1 and 100, or " + DONT_LIMIT_CAPACITY);
                 return false;
             }
         }
diff --git a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
index df631bf..70807e1 100644
--- a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
@@ -5,7 +5,6 @@
 import io.logz.test.MockLogzioBulkListener;
 import net.logstash.logback.composite.ContextJsonProvider;
 import net.logstash.logback.composite.loggingevent.ArgumentsJsonProvider;
-import net.logstash.logback.composite.loggingevent.LoggingEventFormattedTimestampJsonProvider;
 import net.logstash.logback.composite.loggingevent.LoggingEventJsonProviders;
 import net.logstash.logback.composite.loggingevent.LoggingEventPatternJsonProvider;
 import net.logstash.logback.composite.loggingevent.MdcJsonProvider;
diff --git a/src/test/java/io/logz/logback/InMemoryTest.java b/src/test/java/io/logz/logback/InMemoryTest.java
index f78dede..ff918bd 100644
--- a/src/test/java/io/logz/logback/InMemoryTest.java
+++ b/src/test/java/io/logz/logback/InMemoryTest.java
@@ -17,7 +17,7 @@ public void setUp() {
     }
 
     @Test
-    public void validateQueueCapacityLimit() {
+    public void validateQueueCapacity() {
         String token = "verifyCapacityInBytesToken";
         String type = random(8);
         String loggerName = "verifyCapacityInBytes" + random(8);
@@ -37,15 +37,15 @@ public void validateQueueCapacityLimit() {
     }
 
     @Test
-    public void validateQueueLogCountLimit() {
-        String token = "verifyLogCountLimit";
+    public void validateQueueLogCountCapacity() {
+        String token = "verifyLogCountCapacity";
         String type = random(8);
-        String loggerName = "verifyLogCountLimit" + random(8);
+        String loggerName = "verifyLogCountCapacity" + random(8);
         int drainTimeout = 1;
         String message1 = "Testing.." + random(5);
         String message2 = "Don't get here test! ";
 
-        logzioLogbackAppender.setInMemoryLogsCountLimit(1);
+        logzioLogbackAppender.setInMemoryLogsCountCapacity(1);
         Logger testLogger = createLogger(logzioLogbackAppender, token, type, loggerName, drainTimeout, false, false, null, false);
 
         testLogger.info(message1);
diff --git a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
index d5265e1..c35de3c 100644
--- a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
@@ -3,26 +3,16 @@
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.LoggerContext;
 import ch.qos.logback.classic.spi.ILoggingEvent;
-import ch.qos.logback.core.Context;
-import ch.qos.logback.core.encoder.Encoder;
 import ch.qos.logback.core.spi.DeferredProcessingAware;
-import ch.qos.logback.core.status.Status;
-import com.fasterxml.jackson.core.JsonGenerator;
 import io.logz.sender.com.google.gson.Gson;
 import io.logz.test.MockLogzioBulkListener;
 import net.logstash.logback.composite.AbstractFieldJsonProvider;
 import net.logstash.logback.composite.AbstractPatternJsonProvider;
-import net.logstash.logback.composite.CompositeJsonFormatter;
-import net.logstash.logback.composite.FormattedTimestampJsonProvider;
-import net.logstash.logback.composite.JsonProvider;
-import net.logstash.logback.composite.JsonProviders;
 import net.logstash.logback.composite.loggingevent.LogLevelJsonProvider;
-import net.logstash.logback.composite.loggingevent.LoggerNameJsonProvider;
 import net.logstash.logback.composite.loggingevent.LoggingEventFormattedTimestampJsonProvider;
 import net.logstash.logback.composite.loggingevent.LoggingEventPatternJsonProvider;
 import net.logstash.logback.composite.loggingevent.MessageJsonProvider;
-import net.logstash.logback.composite.loggingevent.StackTraceJsonProvider;
-import net.logstash.logback.composite.loggingevent.ThreadNameJsonProvider;
+import net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -33,7 +23,6 @@
 import org.slf4j.Marker;
 import org.slf4j.MarkerFactory;
 
-import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -44,8 +33,6 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.fail;
 
-import net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder;
-
 @RunWith(Parameterized.class)
 public class LogzioLogbackAppenderTest extends BaseLogbackAppenderTest {
     private LogzioLogbackAppender logzioLogbackAppender;

From 88035f31053b03f3f4c8e1437ee87024e0deab79 Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Mon, 8 Oct 2018 12:44:09 +0300
Subject: [PATCH 10/14] fixing set in memory queue and using the new java
 sender with send byte[]

---
 pom.xml                                       |  2 +-
 .../logz/logback/LogzioLogbackAppender.java   |  2 +-
 .../logback/LogzioLogbackAppenderTest.java    | 38 +++++++++----------
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/pom.xml b/pom.xml
index 98fe402..103d60c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,7 +36,7 @@
     </scm>
 
     <properties>
-        <logzio-sender-version>1.0.17</logzio-sender-version>
+        <logzio-sender-version>1.0.18</logzio-sender-version>
     </properties>
 
     <build>
diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
index 11d132a..a7efd99 100644
--- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java
+++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -146,7 +146,7 @@ public void setDebug(boolean debug) {
     }
 
     public void setInMemoryQueue(boolean inMemoryQueue) {
-        this.inMemoryQueue = true;
+        this.inMemoryQueue = inMemoryQueue;
     }
 
     public boolean isInMemoryQueue() {
diff --git a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
index c35de3c..5216a26 100644
--- a/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/LogzioLogbackAppenderTest.java
@@ -311,25 +311,6 @@ public void testContextReset() {
         mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.WARN.levelStr);
     }
 
-    @Test
-    public void testTokenAndLogzioUrlFromSystemEnvironment() {
-        String token = System.getenv("JAVA_HOME");
-        String type = "testType" + random(8);
-        String loggerName = "testLogger" + random(8);
-        int drainTimeout = 1;
-
-        String message1 = "Just a log - " + random(5);
-        Logger testLogger = createLogger(logzioLogbackAppender, "$JAVA_HOME", type, loggerName, drainTimeout, false, false, null, false);
-
-        testLogger.info(message1);
-
-        sleepSeconds(2 * drainTimeout);
-
-        mockListener.assertNumberOfReceivedMsgs(1);
-        MockLogzioBulkListener.LogRequest logRequest = mockListener.assertLogReceivedByMessage(message1);
-        mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr);
-    }
-
     @Test
     public void checkExactStackTrace() throws Exception {
         final CountDownLatch countDownLatch = new CountDownLatch(1);
@@ -365,6 +346,25 @@ public void checkExactStackTrace() throws Exception {
         assertThat(logRequest.getStringFieldOrNull("exception")).isEqualTo(expectedException);
     }
 
+    @Test
+    public void testTokenAndLogzioUrlFromSystemEnvironment() {
+        String token = System.getenv("JAVA_HOME");
+        String type = "testType" + random(8);
+        String loggerName = "testLogger" + random(8);
+        int drainTimeout = 1;
+
+        String message1 = "Just a log - " + random(5);
+        Logger testLogger = createLogger(logzioLogbackAppender, "$JAVA_HOME", type, loggerName, drainTimeout, false, false, null, false);
+
+        testLogger.info(message1);
+
+        sleepSeconds(2 * drainTimeout);
+
+        mockListener.assertNumberOfReceivedMsgs(1);
+        MockLogzioBulkListener.LogRequest logRequest = mockListener.assertLogReceivedByMessage(message1);
+        mockListener.assertLogReceivedIs(logRequest, token, type, loggerName, Level.INFO.levelStr);
+    }
+
     @Test
     public void testEncoder() {
         String token = "testEncoder";

From 27902d0d6eb4c54de54ab4a520c834da2a11a3cf Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Mon, 8 Oct 2018 12:54:09 +0300
Subject: [PATCH 11/14] simplify formatMessageAndSend

---
 .../logz/logback/LogzioLogbackAppender.java   | 22 +++++--------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
index a7efd99..f452421 100644
--- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java
+++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -357,25 +357,15 @@ private String getValueFromSystemEnvironmentIfNeeded(String value) {
 
     private void formatMessageAndSend(ILoggingEvent loggingEvent) {
         try {
-            logzioSender.send(formatMessageAsJson(loggingEvent));
+            if (encoder == null) {
+                logzioSender.send(formatMessageAsJsonInternal(loggingEvent));
+            } else {
+                logzioSender.send(encoder.encode(loggingEvent));
+            }
         } catch (Exception e) {
-            addWarn("Failed to format json", e);
-        }
-
-    }
-
-    private JsonObject formatMessageAsJson(ILoggingEvent loggingEvent) throws Exception {
-        if (encoder == null) {
-            return formatMessageAsJsonInternal(loggingEvent);
+            addWarn("Failed to format and send message", e);
         }
-        return formatMessageAsJsonExternal(loggingEvent);
-    }
-
 
-    private JsonObject formatMessageAsJsonExternal(ILoggingEvent loggingEvent) throws Exception{
-        final byte[] payload = encoder.encode(loggingEvent);
-        JsonElement jsonElement = gson.fromJson(new String(payload, StandardCharsets.UTF_8), JsonElement.class);
-        return jsonElement.getAsJsonObject();
     }
 
     private JsonObject formatMessageAsJsonInternal(ILoggingEvent loggingEvent) {

From 174e8a4750b652f1d81d570b99689a683ff5b49b Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Mon, 8 Oct 2018 12:54:52 +0300
Subject: [PATCH 12/14] clean imports

---
 src/main/java/io/logz/logback/LogzioLogbackAppender.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/io/logz/logback/LogzioLogbackAppender.java b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
index f452421..150c3ec 100644
--- a/src/main/java/io/logz/logback/LogzioLogbackAppender.java
+++ b/src/main/java/io/logz/logback/LogzioLogbackAppender.java
@@ -17,7 +17,6 @@
 import java.io.File;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;

From 3d4f25afee508c1f9d20b88d7a41a141d042509e Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Wed, 24 Oct 2018 14:09:05 +0300
Subject: [PATCH 13/14] delete not used

---
 .../logz/logback/BaseLogbackAppenderTest.java   | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
index 70807e1..31729e5 100644
--- a/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
+++ b/src/test/java/io/logz/logback/BaseLogbackAppenderTest.java
@@ -1,14 +1,7 @@
 package io.logz.logback;
 
-import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.core.Context;
 import io.logz.test.MockLogzioBulkListener;
-import net.logstash.logback.composite.ContextJsonProvider;
-import net.logstash.logback.composite.loggingevent.ArgumentsJsonProvider;
-import net.logstash.logback.composite.loggingevent.LoggingEventJsonProviders;
-import net.logstash.logback.composite.loggingevent.LoggingEventPatternJsonProvider;
-import net.logstash.logback.composite.loggingevent.MdcJsonProvider;
-import net.logstash.logback.composite.loggingevent.MessageJsonProvider;
 import org.junit.After;
 import org.junit.Before;
 import org.slf4j.Logger;
@@ -61,16 +54,6 @@ protected void assertAdditionalFields(MockLogzioBulkListener.LogRequest logReque
         });
     }
 
-    protected LoggingEventJsonProviders getCommonJsonProviders() {
-        LoggingEventJsonProviders jsonProviders = new LoggingEventJsonProviders();
-        jsonProviders.addPattern(new LoggingEventPatternJsonProvider());
-        jsonProviders.addArguments(new ArgumentsJsonProvider());
-        jsonProviders.addMessage(new MessageJsonProvider());
-        jsonProviders.addContext(new ContextJsonProvider<ILoggingEvent>());
-        jsonProviders.addMdc(new MdcJsonProvider());
-        return jsonProviders;
-    }
-
     protected Logger createLogger(LogzioLogbackAppender logzioLogbackAppender, String token, String type, String loggerName, Integer drainTimeout,
                                 boolean addHostname, boolean line, String additionalFields,
                                 boolean compressRequests) {

From 80113376810caac58a81564f6ffb6945e4cb60c8 Mon Sep 17 00:00:00 2001
From: idohalevi <ido.h@logz.io>
Date: Wed, 24 Oct 2018 14:16:39 +0300
Subject: [PATCH 14/14] update readme

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e832ede..15005e2 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ This appender uses [LogzioSender](https://github.com/logzio/logzio-java-sender)
 <dependency>
     <groupId>io.logz.logback</groupId>
     <artifactId>logzio-logback-appender</artifactId>
-    <version>1.0.18</version>
+    <version>1.0.21</version>
 </dependency>
 ```
 
@@ -138,6 +138,8 @@ Will send a log to Logz.io that looks like this:
 ```
 
 ### Release notes
+ - 1.0.21
+    - added in memory queue support
  - 1.0.18
    - added `compressRequests` parameter to enable gzip compression of the logs before they are sent.
    - added option to inject system property value into additionalFields, logzioUrl and token.