Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Th2-5165] Use cache in BookInfo #253

Merged
merged 38 commits into from
Mar 6, 2024
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
c360d21
[TH2-5165] Use cache in BookInfo
Nikita-Smirnov-Exactpro Feb 15, 2024
19638d1
[TH2-5165] Removed old BookInfo constructor
Nikita-Smirnov-Exactpro Feb 19, 2024
66beb2a
[TH2-5165] Corrected according integration tests
Nikita-Smirnov-Exactpro Feb 19, 2024
10c1056
[TH2-5165] bookId, name, start are unique page identifier
Nikita-Smirnov-Exactpro Feb 19, 2024
ad5f947
[TH2-5165] corrected after review
Nikita-Smirnov-Exactpro Feb 19, 2024
4414d01
[TH2-5165] solved todo/fixme
Nikita-Smirnov-Exactpro Feb 19, 2024
44c5582
[TH2-5165] Added book info metrics
Nikita-Smirnov-Exactpro Feb 20, 2024
9191fd2
[TH2-5165] Added Iterator<PageInfo> getPages
Nikita-Smirnov-Exactpro Feb 20, 2024
a8e2c8f
[TH2-5165] corrected after review
Nikita-Smirnov-Exactpro Feb 21, 2024
2f6c263
[TH2-5165] migrated to IteratorPa
Nikita-Smirnov-Exactpro Feb 22, 2024
d50305e
[TH2-5165] refactored after migration
Nikita-Smirnov-Exactpro Feb 22, 2024
2f298c9
[TH2-5165] Revert API methods
Nikita-Smirnov-Exactpro Feb 22, 2024
6ec65e9
[TH2-5165] Added CradleStorage.getBook method
Nikita-Smirnov-Exactpro Feb 23, 2024
fc0c477
[TH2-5165] getPages(start, end, order) can handle start > end case
Nikita-Smirnov-Exactpro Feb 26, 2024
3a6e9be
[TH2-5165] Removed ThreadSafeProvider class
Nikita-Smirnov-Exactpro Feb 26, 2024
b1d9114
[TH2-5165] Corrected java doc in BookInfo
Nikita-Smirnov-Exactpro Feb 26, 2024
8852126
[TH2-5165] Corrected after review
Nikita-Smirnov-Exactpro Feb 26, 2024
1a848c6
[TH2-5165] Corrected after review
Nikita-Smirnov-Exactpro Feb 26, 2024
fc892ce
[TH2-5165] Fixed problem with Instant MIN/MAX conversion problem
Nikita-Smirnov-Exactpro Feb 27, 2024
2262510
[TH2-5165] Corrected book cache behaviour when book has gap
Nikita-Smirnov-Exactpro Feb 28, 2024
9021c09
[TH2-5165] Added testAddPageInTheMiddleOfExist
Nikita-Smirnov-Exactpro Feb 29, 2024
121d11c
[TH2-5165] corrected after review
Nikita-Smirnov-Exactpro Mar 1, 2024
cb0352e
Update cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/i…
Nikita-Smirnov-Exactpro Mar 1, 2024
6a5a1ad
[TH2-5165] corrected after review
Nikita-Smirnov-Exactpro Mar 1, 2024
77e8734
[TH2-5165] removed java 17 API
Nikita-Smirnov-Exactpro Mar 1, 2024
ebee034
[TH2-5165] corrected book info metrics
Nikita-Smirnov-Exactpro Mar 1, 2024
4ce8946
[TH2-5165] added testAddTheSecondPageBeforeThreshold test
Nikita-Smirnov-Exactpro Mar 4, 2024
4e0ecca
[TH2-5165] added randomAccessDaysInvalidateInterval option
Nikita-Smirnov-Exactpro Mar 4, 2024
c6e2979
[TH2-5165] corrected after review
Nikita-Smirnov-Exactpro Mar 4, 2024
7ecc0ed
[TH2-5165] revert BOOK_LABEL to book
Nikita-Smirnov-Exactpro Mar 5, 2024
0d29959
[TH2-5165] revert logic to skip register load for empty day page inte…
Nikita-Smirnov-Exactpro Mar 5, 2024
b88f91e
[TH2-5165] revert logic to skip register load for empty day page inte…
Nikita-Smirnov-Exactpro Mar 5, 2024
e9b15b6
Merge remote-tracking branch 'origin/dev-version-5' into TH2-5165
Nikita-Smirnov-Exactpro Mar 6, 2024
7f2926b
[TH2-5165] merged with dev-version-5
Nikita-Smirnov-Exactpro Mar 6, 2024
a578802
[TH2-5165] added BookWithoutPageTest
Nikita-Smirnov-Exactpro Mar 6, 2024
31756d3
[TH2-5165] corrected after review
Nikita-Smirnov-Exactpro Mar 6, 2024
ac5ab52
[TH2-5165] Updated github workflow
Nikita-Smirnov-Exactpro Mar 6, 2024
afe632e
[TH2-5165] Updated github workflow
Nikita-Smirnov-Exactpro Mar 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.annotation.concurrent.GuardedBy;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
Expand All @@ -56,8 +55,6 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Function;

import static com.exactpro.cradle.Order.REVERSE;
Expand All @@ -84,10 +81,7 @@ abstract public class AbstractMessageIteratorProvider<T> extends IteratorProvide
protected final MessageBatchIteratorFilter<MessageBatchEntity> batchFilter;
protected final MessageBatchIteratorCondition<MessageBatchEntity> iterationCondition;

private final Lock lock = new ReentrantLock();
@GuardedBy("lock")
private final Iterator<PageInfo> pageProvider;
Nikita-Smirnov-Exactpro marked this conversation as resolved.
Show resolved Hide resolved
@GuardedBy("lock")
// only get / set operation are guarded by lock
private TakeWhileIterator<MessageBatchEntity> takeWhileIterator;

Expand Down Expand Up @@ -219,22 +213,17 @@ protected CassandraStoredMessageFilter createFilter(@Nonnull PageInfo pageInfo,
}

protected @Nullable PageInfo nextPage() {
lock.lock();
try {
if (takeWhileIterator != null && takeWhileIterator.isHalted()) {
LOGGER.debug("Iterator was interrupted because iterator condition was not met");
return null;
}

if (limit > 0 && returned.get() >= limit) {
LOGGER.debug("Filtering interrupted because limit for records to return ({}) is reached ({})", limit, returned);
return null;
}
if (takeWhileIterator != null && takeWhileIterator.isHalted()) {
LOGGER.debug("Iterator was interrupted because iterator condition was not met");
return null;
}

return pageProvider.hasNext() ? pageProvider.next() : null;
} finally {
lock.unlock();
if (limit > 0 && returned.get() >= limit) {
LOGGER.debug("Filtering interrupted because limit for records to return ({}) is reached ({})", limit, returned);
return null;
}

return pageProvider.hasNext() ? pageProvider.next() : null;
}

protected Iterator<StoredMessageBatch> getBatchedIterator(PageId pageId,
Expand All @@ -248,19 +237,14 @@ protected Iterator<StoredMessageBatch> getBatchedIterator(PageId pageId,
pagedIterator,
batchFilter::test);

lock.lock();
try {
// We need to store this iterator since
// it gives info whether or no iterator was halted
takeWhileIterator = new TakeWhileIterator<>(
filteringIterator,
iterationCondition);
// We need to store this iterator since
// it gives info whether or no iterator was halted
takeWhileIterator = new TakeWhileIterator<>(
filteringIterator,
iterationCondition);

return new ConvertingIterator<>(
takeWhileIterator, entity ->
MessagesWorker.mapMessageBatchEntity(pageId, entity));
} finally {
lock.unlock();
}
return new ConvertingIterator<>(
takeWhileIterator, entity ->
MessagesWorker.mapMessageBatchEntity(pageId, entity));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import com.exactpro.cradle.cassandra.iterators.PagedIterator;
import com.exactpro.cradle.cassandra.resultset.IteratorProvider;
import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor;
import com.exactpro.cradle.cassandra.utils.ThreadSafeProvider;
import com.exactpro.cradle.cassandra.workers.MessagesWorker;
import com.exactpro.cradle.filters.FilterForGreater;
import com.exactpro.cradle.filters.FilterForLess;
Expand Down Expand Up @@ -68,7 +67,7 @@ public class GroupedMessageIteratorProvider extends IteratorProvider<StoredGroup
private final GroupedMessageFilter filter;
protected final FilterForGreater<Instant> leftBoundFilter;
protected final FilterForLess<Instant> rightBoundFilter;
private final ThreadSafeProvider<PageInfo> pageProvider;
private final Iterator<PageInfo> pageProvider;
private final Function<BoundStatementBuilder, BoundStatementBuilder> readAttrs;
/** limit must be strictly positive ( limit greater than 0 ) */
private final int limit;
Expand Down Expand Up @@ -96,12 +95,10 @@ public GroupedMessageIteratorProvider(String requestInfo,
this.returned = new AtomicInteger();
this.leftBoundFilter = createLeftBoundFilter(filter);
this.rightBoundFilter = createRightBoundFilter(filter);
this.pageProvider = new ThreadSafeProvider<>(
book.getPages(
leftBoundFilter.getValue(),
rightBoundFilter.getValue(),
order
)
this.pageProvider = book.getPages(
leftBoundFilter.getValue(),
rightBoundFilter.getValue(),
order
);
this.order = order;
}
Expand Down Expand Up @@ -174,10 +171,10 @@ private LocalTime getNearestBatchTime(Instant timestamp,

@Override
public CompletableFuture<Iterator<StoredGroupedMessageBatch>> nextIterator() {
PageInfo nextPage = pageProvider.next();
if (nextPage == null) {
if (!pageProvider.hasNext()) {
return CompletableFuture.completedFuture(null);
}
PageInfo nextPage = pageProvider.next();

if (limit > 0 && returned.get() >= limit) {
logger.debug("Filtering interrupted because limit for records to return ({}) is reached ({})", limit, returned);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import com.exactpro.cradle.cassandra.iterators.PagedIterator;
import com.exactpro.cradle.cassandra.resultset.IteratorProvider;
import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor;
import com.exactpro.cradle.cassandra.utils.ThreadSafeProvider;
import com.exactpro.cradle.counters.CounterSample;
import com.exactpro.cradle.iterators.ConvertingIterator;

Expand All @@ -45,7 +44,7 @@ public class EntityStatisticsIteratorProvider extends IteratorProvider<CounterSa
private final FrameType frameType;
private final FrameInterval frameInterval;
private final Function<BoundStatementBuilder, BoundStatementBuilder> readAttrs;
private final ThreadSafeProvider<PageInfo> pageProvider;
private final Iterator<PageInfo> pageProvider;
public EntityStatisticsIteratorProvider(String requestInfo, CassandraOperators operators, BookInfo book,
ExecutorService composingService, SelectQueryExecutor selectQueryExecutor,
EntityType entityType,
Expand All @@ -60,21 +59,19 @@ public EntityStatisticsIteratorProvider(String requestInfo, CassandraOperators o
this.frameType = frameType;
this.frameInterval = frameInterval;
this.readAttrs = readAttrs;
this.pageProvider = new ThreadSafeProvider<>(
book.getPages(
frameInterval.getInterval().getStart(),
frameInterval.getInterval().getEnd(),
DIRECT
)
this.pageProvider = book.getPages(
frameInterval.getInterval().getStart(),
frameInterval.getInterval().getEnd(),
DIRECT
);
}

@Override
public CompletableFuture<Iterator<CounterSample>> nextIterator() {
PageInfo pageInfo = pageProvider.next();
if(pageInfo == null){
if(!pageProvider.hasNext()){
return CompletableFuture.completedFuture(null);
}
PageInfo pageInfo = pageProvider.next();

Instant actualStart = frameInterval.getFrameType().getFrameStart(frameInterval.getInterval().getStart());
Instant actualEnd = frameInterval.getFrameType().getFrameEnd(frameInterval.getInterval().getEnd());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import com.exactpro.cradle.cassandra.iterators.PagedIterator;
import com.exactpro.cradle.cassandra.resultset.IteratorProvider;
import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor;
import com.exactpro.cradle.cassandra.utils.ThreadSafeProvider;
import com.exactpro.cradle.counters.CounterSample;
import com.exactpro.cradle.iterators.ConvertingIterator;

Expand All @@ -44,7 +43,7 @@ public class MessageStatisticsIteratorProvider extends IteratorProvider<CounterS
private final Direction direction;
private final FrameInterval frameInterval;
private final Function<BoundStatementBuilder, BoundStatementBuilder> readAttrs;
private final ThreadSafeProvider<PageInfo> pageProvider;
private final Iterator<PageInfo> pageProvider;

public MessageStatisticsIteratorProvider(String requestInfo, CassandraOperators operators, BookInfo book,
ExecutorService composingService, SelectQueryExecutor selectQueryExecutor,
Expand All @@ -59,22 +58,20 @@ public MessageStatisticsIteratorProvider(String requestInfo, CassandraOperators
this.direction = direction;
this.frameInterval = frameInterval;
this.readAttrs = readAttrs;
this.pageProvider = new ThreadSafeProvider<>(
book.getPages(
frameInterval.getInterval().getStart(),
frameInterval.getInterval().getEnd(),
DIRECT
)
this.pageProvider = book.getPages(
frameInterval.getInterval().getStart(),
frameInterval.getInterval().getEnd(),
DIRECT
);

}

@Override
public CompletableFuture<Iterator<CounterSample>> nextIterator() {
PageInfo pageInfo = pageProvider.next();
if(pageInfo == null){
if(!pageProvider.hasNext()){
return CompletableFuture.completedFuture(null);
}
PageInfo pageInfo = pageProvider.next();

Instant actualStart = frameInterval.getFrameType().getFrameStart(frameInterval.getInterval().getStart());
Instant actualEnd = frameInterval.getFrameType().getFrameEnd(frameInterval.getInterval().getEnd());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import com.exactpro.cradle.cassandra.iterators.PagedIterator;
import com.exactpro.cradle.cassandra.resultset.IteratorProvider;
import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor;
import com.exactpro.cradle.cassandra.utils.ThreadSafeProvider;
import com.exactpro.cradle.filters.ComparisonOperation;
import com.exactpro.cradle.filters.FilterForGreater;
import com.exactpro.cradle.iterators.ConvertingIterator;
Expand Down Expand Up @@ -71,7 +70,7 @@ public class TestEventIteratorProvider extends IteratorProvider<StoredTestEvent>
private final int limit;
private final AtomicInteger returned = new AtomicInteger();
private final Instant actualFrom;
private final ThreadSafeProvider<PageInfo> pageProvider;
private final Iterator<PageInfo> pageProvider;
private final TestEventFilter filter;
private final FilterForGreater<Instant> leftBoundFilter;

Expand All @@ -91,23 +90,21 @@ public TestEventIteratorProvider(String requestInfo, TestEventFilter filter, Cas
this.leftBoundFilter = createLeftBoundFilter(book, eventBatchDurationWorker, actualFrom, readAttrs, filter);
this.filter = filter;

this.pageProvider = new ThreadSafeProvider<>(
book.getPages(
// we must calculate the first timestamp by origin filter because
// left bound filter is calculated according to max duration of event batch on the first page
findFirstTimestamp(filter.getPageId(), filter.getStartTimestampFrom(), book),
findLastTimestamp(filter.getPageId(), filter.getStartTimestampTo(), book),
filter.getOrder()
)
this.pageProvider = book.getPages(
// we must calculate the first timestamp by origin filter because
// left bound filter is calculated according to max duration of event batch on the first page
findFirstTimestamp(filter.getPageId(), filter.getStartTimestampFrom(), book),
findLastTimestamp(filter.getPageId(), filter.getStartTimestampTo(), book),
filter.getOrder()
);
}

@Override
public CompletableFuture<Iterator<StoredTestEvent>> nextIterator() {
PageInfo nextPage = pageProvider.next();
if (nextPage == null) {
if (!pageProvider.hasNext()) {
return CompletableFuture.completedFuture(null);
}
PageInfo nextPage = pageProvider.next();

if (limit > 0 && returned.get() >= limit) {
LOGGER.debug("Filtering interrupted because limit for records to return ({}) is reached ({})", limit, returned);
Expand Down

This file was deleted.

Loading
Loading