Old impl did fetch all queues (take 2000 as an example from PaISA prod) Did + * a nested iteration, so worst case iterate 2000 times 2000 queues to find the + * matching queue (2'000 * 2'000 = 4'000'000 iterations). New impl now does setup a + * dictionary then does indexed access while iterating the 2000 entries + * (2'000 + 2'000 = 4'000 iterations). Keep in mind 2'000 is just some usual value. + * Under load, this value can increase further and so the old, exponential approach + * just did explode in terms of computational efford.
+ */ +public class QueueStatsService { + + private static final Logger log = getLogger(QueueStatsService.class); + private final Vertx vertx; + private final EventBus eventBus; + private final String redisquesAddress; + private final QueueStatisticsCollector queueStatisticsCollector; + + public QueueStatsService(Vertx vertx, EventBus eventBus, String redisquesAddress, QueueStatisticsCollector queueStatisticsCollector) { + this.vertx = vertx; + this.eventBus = eventBus; + this.redisquesAddress = redisquesAddress; + this.queueStatisticsCollector = queueStatisticsCollector; + } + + publicMentors fetching operations and so provides the fetcher the required + * information. Finally it also receives the operations result.
+ * + * @paramReturning true means that all queues will be present in the result. If + * false, empty queues won't show up the result.
+ * + * @param ctx See {@link GetQueueStatsMentor}. + */ + public boolean includeEmptyQueues( CTX ctx ); + + /**Limits the result to the largest N queues.
*/ + public int limit( CTX ctx ); + + public String filter( CTX ctx); + + /**Called ONCE with the final result.
*/ + public void onQueueStatistics(ListCalled as soon an error occurs. After an error occurred, {@link #onQueueStatistics(List, Object)} + * will NOT be called, as the operation did fail.
+ */ + public void onError(Throwable ex, CTX ctx); + } + +} diff --git a/src/main/java/org/swisspush/redisques/handler/RedisquesHttpRequestHandler.java b/src/main/java/org/swisspush/redisques/handler/RedisquesHttpRequestHandler.java index 410e21c..6ad031d 100644 --- a/src/main/java/org/swisspush/redisques/handler/RedisquesHttpRequestHandler.java +++ b/src/main/java/org/swisspush/redisques/handler/RedisquesHttpRequestHandler.java @@ -2,9 +2,7 @@ import io.netty.util.internal.StringUtil; import io.vertx.core.AsyncResult; -import io.vertx.core.Future; import io.vertx.core.Handler; -import io.vertx.core.Promise; import io.vertx.core.Vertx; import io.vertx.core.buffer.Buffer; import io.vertx.core.eventbus.EventBus; @@ -12,7 +10,6 @@ import io.vertx.core.http.HttpServerOptions; import io.vertx.core.http.HttpServerRequest; import io.vertx.core.http.HttpServerResponse; -import io.vertx.core.json.Json; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import io.vertx.ext.auth.authentication.AuthenticationProvider; @@ -21,69 +18,19 @@ import io.vertx.ext.web.handler.BasicAuthHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.swisspush.redisques.util.DequeueStatistic; -import org.swisspush.redisques.util.QueueStatisticsCollector; -import org.swisspush.redisques.util.RedisquesAPI; -import org.swisspush.redisques.util.RedisquesConfiguration; -import org.swisspush.redisques.util.Result; -import org.swisspush.redisques.util.StatusCode; +import org.swisspush.redisques.QueueStatsService; +import org.swisspush.redisques.QueueStatsService.GetQueueStatsMentor; +import org.swisspush.redisques.util.*; import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Date; -import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import static org.swisspush.redisques.util.HttpServerRequestUtil.decode; -import static org.swisspush.redisques.util.HttpServerRequestUtil.encodePayload; -import static org.swisspush.redisques.util.HttpServerRequestUtil.evaluateUrlParameterToBeEmptyOrTrue; -import static org.swisspush.redisques.util.HttpServerRequestUtil.extractNonEmptyJsonArrayFromBody; -import static org.swisspush.redisques.util.RedisquesAPI.BAD_INPUT; -import static org.swisspush.redisques.util.RedisquesAPI.COUNT; -import static org.swisspush.redisques.util.RedisquesAPI.ERROR_TYPE; -import static org.swisspush.redisques.util.RedisquesAPI.FILTER; -import static org.swisspush.redisques.util.RedisquesAPI.LIMIT; -import static org.swisspush.redisques.util.RedisquesAPI.LOCKS; -import static org.swisspush.redisques.util.RedisquesAPI.MEMORY_FULL; -import static org.swisspush.redisques.util.RedisquesAPI.MESSAGE; -import static org.swisspush.redisques.util.RedisquesAPI.MONITOR_QUEUE_NAME; -import static org.swisspush.redisques.util.RedisquesAPI.MONITOR_QUEUE_SIZE; -import static org.swisspush.redisques.util.RedisquesAPI.NO_SUCH_LOCK; -import static org.swisspush.redisques.util.RedisquesAPI.OK; -import static org.swisspush.redisques.util.RedisquesAPI.QUEUES; -import static org.swisspush.redisques.util.RedisquesAPI.STATISTIC_QUEUE_DEQUEUESTATISTIC; -import static org.swisspush.redisques.util.RedisquesAPI.STATISTIC_QUEUE_LAST_DEQUEUE_ATTEMPT; -import static org.swisspush.redisques.util.RedisquesAPI.STATISTIC_QUEUE_LAST_DEQUEUE_SUCCESS; -import static org.swisspush.redisques.util.RedisquesAPI.STATISTIC_QUEUE_NEXT_DEQUEUE_DUE_TS; -import static org.swisspush.redisques.util.RedisquesAPI.STATUS; -import static org.swisspush.redisques.util.RedisquesAPI.VALUE; -import static org.swisspush.redisques.util.RedisquesAPI.buildAddQueueItemOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildBulkDeleteLocksOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildBulkDeleteQueuesOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildBulkPutLocksOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildDeleteAllLocksOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildDeleteAllQueueItemsOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildDeleteLockOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildDeleteQueueItemOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildEnqueueOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildGetAllLocksOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildGetConfigurationOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildGetLockOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildGetQueueItemOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildGetQueueItemsCountOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildGetQueueItemsOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildGetQueuesCountOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildGetQueuesItemsCountOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildGetQueuesOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildGetQueuesSpeedOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildGetQueuesStatisticsOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildLockedEnqueueOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildPutLockOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildReplaceQueueItemOperation; -import static org.swisspush.redisques.util.RedisquesAPI.buildSetConfigurationOperation; +import static org.swisspush.redisques.util.HttpServerRequestUtil.*; +import static org.swisspush.redisques.util.RedisquesAPI.*; /** * Handler class for HTTP requests providing access to Redisques over HTTP. @@ -95,6 +42,7 @@ public class RedisquesHttpRequestHandler implements HandlerFor why we should NOT use such date formats, see SDCISA-15311. We really + * should utilize ISO dates and include timezone information.
+ * + * @deprecated TODO about date formats + */ + @Deprecated private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("dd.MM.yyyy hh:mm:ss"); private final String redisquesAddress; @@ -114,9 +69,11 @@ public class RedisquesHttpRequestHandler implements Handler