From 1852b0ab3cfc366cc4e28adcb13219a98108b4e7 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Mon, 21 Oct 2024 13:34:29 -0500 Subject: [PATCH 01/35] Restore initial commit from grpc-java, plus a few local changes --- .../AsyncServletOutputStreamWriter.java | 110 ++++++++++++------ .../grpc/servlet/jakarta/ServletAdapter.java | 24 +--- .../servlet/jakarta/ServletServerBuilder.java | 41 +++++-- 3 files changed, 107 insertions(+), 68 deletions(-) diff --git a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java index 9a819c2426e..4370e9fceda 100644 --- a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java +++ b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java @@ -13,6 +13,7 @@ package io.grpc.servlet.jakarta; +import com.google.common.annotations.VisibleForTesting; import io.grpc.InternalLogId; import io.grpc.servlet.jakarta.ServletServerStream.ServletTransportState; import jakarta.servlet.AsyncContext; @@ -26,6 +27,8 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; +import java.util.function.BiFunction; +import java.util.function.BooleanSupplier; import java.util.logging.Logger; import static com.google.common.base.Preconditions.checkState; @@ -36,9 +39,6 @@ /** Handles write actions from the container thread and the application thread. */ final class AsyncServletOutputStreamWriter { - private static final Logger logger = - Logger.getLogger(AsyncServletOutputStreamWriter.class.getName()); - /** * Memory boundary for write actions. * @@ -61,11 +61,11 @@ final class AsyncServletOutputStreamWriter { */ private final AtomicReference writeState = new AtomicReference<>(WriteState.DEFAULT); - private final ServletOutputStream outputStream; - private final ServletTransportState transportState; - private final InternalLogId logId; + private final Log log; + private final BiFunction writeAction; private final ActionItem flushAction; private final ActionItem completeAction; + private final BooleanSupplier isReady; /** * New write actions will be buffered into this queue if the servlet output stream is not ready or the queue is not @@ -82,38 +82,68 @@ final class AsyncServletOutputStreamWriter { AsyncContext asyncContext, ServletTransportState transportState, InternalLogId logId) throws IOException { - this.outputStream = asyncContext.getResponse().getOutputStream(); - this.transportState = transportState; - this.logId = logId; + Logger logger = Logger.getLogger(AsyncServletOutputStreamWriter.class.getName()); + this.log = new Log() { + @Override + public void fine(String str, Object... params) { + if (logger.isLoggable(FINE)) { + logger.log(FINE, "[" + logId + "]" + str, params); + } + } + + @Override + public void finest(String str, Object... params) { + if (logger.isLoggable(FINEST)) { + logger.log(FINEST, "[" + logId + "] " + str, params); + } + } + }; + + ServletOutputStream outputStream = asyncContext.getResponse().getOutputStream(); + this.writeAction = (byte[] bytes, Integer numBytes) -> () -> { + outputStream.write(bytes, 0, numBytes); + transportState.runOnTransportThread(() -> transportState.onSentBytes(numBytes)); + log.finest("outbound data: length={0}, bytes={1}", numBytes, toHexString(bytes, numBytes)); + }; this.flushAction = () -> { - logger.log(FINEST, "[{0}] flushBuffer", logId); + log.finest("flushBuffer"); asyncContext.getResponse().flushBuffer(); }; this.completeAction = () -> { - logger.log(FINE, "[{0}] call is completing", logId); + log.fine("call is completing"); transportState.runOnTransportThread( () -> { transportState.complete(); asyncContext.complete(); - logger.log(FINE, "[{0}] call completed", logId); + log.fine("call completed"); }); }; + this.isReady = () -> outputStream.isReady(); + } + + /** + * Constructor without java.util.logging and jakarta.servlet.* dependency, so that Lincheck can run. + * + * @param writeAction Provides an {@link ActionItem} to write given bytes with specified length. + * @param isReady Indicates whether the writer can write bytes at the moment (asynchronously). + */ + @VisibleForTesting + AsyncServletOutputStreamWriter( + BiFunction writeAction, + ActionItem flushAction, + ActionItem completeAction, + BooleanSupplier isReady, + Log log) { + this.writeAction = writeAction; + this.flushAction = flushAction; + this.completeAction = completeAction; + this.isReady = isReady; + this.log = log; } /** Called from application thread. */ void writeBytes(byte[] bytes, int numBytes) throws IOException { - runOrBuffer( - // write bytes action - () -> { - outputStream.write(bytes, 0, numBytes); - transportState.runOnTransportThread(() -> transportState.onSentBytes(numBytes)); - if (logger.isLoggable(FINEST)) { - logger.log( - FINEST, - "[{0}] outbound data: length = {1}, bytes = {2}", - new Object[] {logId, numBytes, toHexString(bytes, numBytes)}); - } - }); + runOrBuffer(writeAction.apply(bytes, numBytes)); } /** Called from application thread. */ @@ -132,10 +162,9 @@ void complete() { /** Called from the container thread {@link jakarta.servlet.WriteListener#onWritePossible()}. */ void onWritePossible() throws IOException { - logger.log( - FINEST, "[{0}] onWritePossible: ENTRY. The servlet output stream becomes ready", logId); + log.finest("onWritePossible: ENTRY. The servlet output stream becomes ready"); assureReadyAndDrainedTurnsFalse(); - while (outputStream.isReady()) { + while (isReady.getAsBoolean()) { WriteState curState = writeState.get(); ActionItem actionItem = writeChain.poll(); @@ -146,18 +175,15 @@ void onWritePossible() throws IOException { if (writeState.compareAndSet(curState, curState.withReadyAndDrained(true))) { // state has not changed since. - logger.log( - FINEST, - "[{0}] onWritePossible: EXIT. All data available now is sent out and the servlet output" - + " stream is still ready", - logId); + log.finest( + "onWritePossible: EXIT. All data available now is sent out and the servlet output" + + " stream is still ready"); return; } // else, state changed by another thread (runOrBuffer()), need to drain the writeChain // again } - logger.log( - FINEST, "[{0}] onWritePossible: EXIT. The servlet output stream becomes not ready", logId); + log.finest("onWritePossible: EXIT. The servlet output stream becomes not ready"); } private void assureReadyAndDrainedTurnsFalse() { @@ -166,7 +192,7 @@ private void assureReadyAndDrainedTurnsFalse() { // being set to false by runOrBuffer() concurrently. while (writeState.get().readyAndDrained) { parkingThread = Thread.currentThread(); - LockSupport.parkNanos(Duration.ofHours(1).toNanos()); // should return immediately + LockSupport.parkNanos(Duration.ofMinutes(1).toNanos()); // should return immediately } parkingThread = null; } @@ -184,12 +210,12 @@ private void runOrBuffer(ActionItem actionItem) throws IOException { if (actionItem == completeAction) { return; } - if (!outputStream.isReady()) { + if (!isReady.getAsBoolean()) { boolean successful = writeState.compareAndSet(curState, curState.withReadyAndDrained(false)); LockSupport.unpark(parkingThread); checkState(successful, "Bug: curState is unexpectedly changed by another thread"); - logger.log(FINEST, "[{0}] the servlet output stream becomes not ready", logId); + log.finest("the servlet output stream becomes not ready"); } } else { // buffer to the writeChain writeChain.offer(actionItem); @@ -208,10 +234,18 @@ private void runOrBuffer(ActionItem actionItem) throws IOException { /** Write actions, e.g. writeBytes, flush, complete. */ @FunctionalInterface - private interface ActionItem { + @VisibleForTesting + interface ActionItem { void run() throws IOException; } + @VisibleForTesting // Lincheck test can not run with java.util.logging dependency. + interface Log { + default void fine(String str, Object...params) {} + + default void finest(String str, Object...params) {} + } + private static final class WriteState { static final WriteState DEFAULT = new WriteState(false); diff --git a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletAdapter.java b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletAdapter.java index 3fb2655b43b..5be92f0989e 100644 --- a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletAdapter.java +++ b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletAdapter.java @@ -45,7 +45,6 @@ import java.util.Enumeration; import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; import static com.google.common.base.Preconditions.checkArgument; @@ -116,7 +115,7 @@ public T otherAdapter(AdapterConstructor constructor) { * {@code resp.setBufferSize()} before invocation is allowed. */ public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { - // TODO(zdapeng) + resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED, "GET method not supported"); } /** @@ -172,10 +171,8 @@ public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOEx getAuthority(req), logId); - stream.transportState().runOnTransportThread(() -> { - transportListener.streamCreated(stream, method, headers); - stream.transportState().onStreamAllocated(); - }); + transportListener.streamCreated(stream, method, headers); + stream.transportState().runOnTransportThread(stream.transportState()::onStreamAllocated); asyncCtx.getRequest().getInputStream() .setReadListener(new GrpcReadListener(stream, asyncCtx, logId)); @@ -217,7 +214,7 @@ private static String getAuthority(HttpServletRequest req) { try { return new URI(req.getRequestURL().toString()).getAuthority(); } catch (URISyntaxException e) { - logger.log(FINE, "Error getting authority from the request URL {0}" + req.getRequestURL()); + logger.log(FINE, "Error getting authority from the request URL {0}", req.getRequestURL()); return req.getServerName() + ":" + req.getServerPort(); } } @@ -282,7 +279,6 @@ private static final class GrpcReadListener implements ReadListener { final AsyncContext asyncCtx; final ServletInputStream input; final InternalLogId logId; - private final AtomicBoolean closed = new AtomicBoolean(false); GrpcReadListener( ServletServerStream stream, @@ -325,16 +321,8 @@ public void onDataAvailable() throws IOException { @Override public void onAllDataRead() { logger.log(FINE, "[{0}] onAllDataRead", logId); - if (!closed.compareAndSet(false, true)) { - // https://github.com/eclipse/jetty.project/issues/8405 - // Note that while this can be mitigated by setting - // AbstractHTTP2ServerConnectionFactory.getStreamIdleTimeout to zero, we allow this to be customized, so - // this workaround is being left in place. - logger.log(FINE, "[{0}] onAllDataRead already called, skipping this one", logId); - return; - } - stream.transportState().runOnTransportThread( - () -> stream.transportState().inboundDataReceived(ReadableBuffers.empty(), true)); + stream.transportState().runOnTransportThread(() -> + stream.transportState().inboundDataReceived(ReadableBuffers.empty(), true)); } @Override diff --git a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerBuilder.java b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerBuilder.java index e149cb5eb56..113b4f0cd10 100644 --- a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerBuilder.java +++ b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerBuilder.java @@ -15,12 +15,14 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ListenableFuture; +import io.grpc.Attributes; import io.grpc.ExperimentalApi; import io.grpc.ForwardingServerBuilder; import io.grpc.Internal; import io.grpc.InternalChannelz.SocketStats; import io.grpc.InternalInstrumented; import io.grpc.InternalLogId; +import io.grpc.Metadata; import io.grpc.Server; import io.grpc.ServerBuilder; import io.grpc.ServerStreamTracer; @@ -29,6 +31,7 @@ import io.grpc.internal.InternalServer; import io.grpc.internal.ServerImplBuilder; import io.grpc.internal.ServerListener; +import io.grpc.internal.ServerStream; import io.grpc.internal.ServerTransport; import io.grpc.internal.ServerTransportListener; import io.grpc.internal.SharedResourceHolder; @@ -98,9 +101,10 @@ public ServletAdapter buildServletAdapter() { } private ServerTransportListener buildAndStart() { + Server server; try { internalCaller = true; - build().start(); + server = build().start(); } catch (IOException e) { // actually this should never happen throw new RuntimeException(e); @@ -115,9 +119,29 @@ private ServerTransportListener buildAndStart() { // Create only one "transport" for all requests because it has no knowledge of which request is // associated with which client socket. This "transport" does not do socket connection, the // container does. - ServerTransportImpl serverTransport = - new ServerTransportImpl(scheduler, usingCustomScheduler); - return internalServer.serverListener.transportCreated(serverTransport); + ServerTransportImpl serverTransport = new ServerTransportImpl(scheduler); + ServerTransportListener delegate = + internalServer.serverListener.transportCreated(serverTransport); + return new ServerTransportListener() { + @Override + public void streamCreated(ServerStream stream, String method, Metadata headers) { + delegate.streamCreated(stream, method, headers); + } + + @Override + public Attributes transportReady(Attributes attributes) { + return delegate.transportReady(attributes); + } + + @Override + public void transportTerminated() { + server.shutdown(); + delegate.transportTerminated(); + if (!usingCustomScheduler) { + SharedResourceHolder.release(GrpcUtil.TIMER_SERVICE, scheduler); + } + } + }; } @VisibleForTesting @@ -211,24 +235,17 @@ static final class ServerTransportImpl implements ServerTransport { private final InternalLogId logId = InternalLogId.allocate(ServerTransportImpl.class, null); private final ScheduledExecutorService scheduler; - private final boolean usingCustomScheduler; - ServerTransportImpl( - ScheduledExecutorService scheduler, boolean usingCustomScheduler) { + ServerTransportImpl(ScheduledExecutorService scheduler) { this.scheduler = checkNotNull(scheduler, "scheduler"); - this.usingCustomScheduler = usingCustomScheduler; } @Override public void shutdown() { - if (!usingCustomScheduler) { - SharedResourceHolder.release(GrpcUtil.TIMER_SERVICE, scheduler); - } } @Override public void shutdownNow(Status reason) { - shutdown(); } @Override From 68b925a0d0f627d712254fa59080dcdac48ddc71 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Mon, 21 Oct 2024 13:36:24 -0500 Subject: [PATCH 02/35] Guard writing payload as hex if FINEST is enabled --- .../jakarta/AsyncServletOutputStreamWriter.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java index 4370e9fceda..d31fcfd19b1 100644 --- a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java +++ b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java @@ -97,13 +97,20 @@ public void finest(String str, Object... params) { logger.log(FINEST, "[" + logId + "] " + str, params); } } + + @Override + public boolean isFinestEnabled() { + return logger.isLoggable(FINEST); + } }; ServletOutputStream outputStream = asyncContext.getResponse().getOutputStream(); this.writeAction = (byte[] bytes, Integer numBytes) -> () -> { outputStream.write(bytes, 0, numBytes); transportState.runOnTransportThread(() -> transportState.onSentBytes(numBytes)); - log.finest("outbound data: length={0}, bytes={1}", numBytes, toHexString(bytes, numBytes)); + if (log.isFinestEnabled()) { + log.finest("outbound data: length={0}, bytes={1}", numBytes, toHexString(bytes, numBytes)); + } }; this.flushAction = () -> { log.finest("flushBuffer"); @@ -244,6 +251,10 @@ interface Log { default void fine(String str, Object...params) {} default void finest(String str, Object...params) {} + + default boolean isFinestEnabled() { + return false; + }; } private static final class WriteState { From e88c47eb50ba88967fd657600cc7f4a244d77665 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Mon, 21 Oct 2024 13:39:09 -0500 Subject: [PATCH 03/35] Apply upstream "Fix AsyncServletOutputStreamWriterConcurrencyTest flakiness (#9948)" --- .../grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java index d31fcfd19b1..8f692fd2fe5 100644 --- a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java +++ b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/AsyncServletOutputStreamWriter.java @@ -199,7 +199,9 @@ private void assureReadyAndDrainedTurnsFalse() { // being set to false by runOrBuffer() concurrently. while (writeState.get().readyAndDrained) { parkingThread = Thread.currentThread(); - LockSupport.parkNanos(Duration.ofMinutes(1).toNanos()); // should return immediately + // Try to sleep for an extremely long time to avoid writeState being changed at exactly + // the time when sleep time expires (in extreme scenario, such as #9917). + LockSupport.parkNanos(Duration.ofHours(1).toNanos()); // should return immediately } parkingThread = null; } From f9a19fc96d56b57542208e00590dabd88a5f10c3 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Mon, 21 Oct 2024 13:46:51 -0500 Subject: [PATCH 04/35] Apply upstream "Avoid flushing headers when the server returns a single response (#9314)" --- .../main/java/io/grpc/servlet/jakarta/ServletServerStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerStream.java b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerStream.java index cd04cf3b41e..bf65f3f3c75 100644 --- a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerStream.java +++ b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerStream.java @@ -223,7 +223,7 @@ private final class Sink implements AbstractServerStream.Sink { final TrailerSupplier trailerSupplier = new TrailerSupplier(); @Override - public void writeHeaders(Metadata headers) { + public void writeHeaders(Metadata headers, boolean flush) { writeHeadersToServletResponse(headers); try { resp.setTrailerFields(trailerSupplier); From 4733524dbfd6ffce52b9438efd0f7c3698bb0ffe Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Mon, 21 Oct 2024 13:50:58 -0500 Subject: [PATCH 05/35] Apply upstream "servlet: introduce ServletServerBuilder.buildServlet()" --- .../src/main/java/io/grpc/servlet/jakarta/GrpcServlet.java | 2 -- .../java/io/grpc/servlet/jakarta/ServletServerBuilder.java | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/GrpcServlet.java b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/GrpcServlet.java index 89b518112a9..933c1dcf4b4 100644 --- a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/GrpcServlet.java +++ b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/GrpcServlet.java @@ -13,7 +13,6 @@ package io.grpc.servlet.jakarta; -import com.google.common.annotations.VisibleForTesting; import io.grpc.Attributes; import io.grpc.BindableService; import io.grpc.ExperimentalApi; @@ -44,7 +43,6 @@ public class GrpcServlet extends HttpServlet { private final ServletAdapter servletAdapter; - @VisibleForTesting GrpcServlet(ServletAdapter servletAdapter) { this.servletAdapter = servletAdapter; } diff --git a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerBuilder.java b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerBuilder.java index 113b4f0cd10..25c7b40c498 100644 --- a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerBuilder.java +++ b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerBuilder.java @@ -100,6 +100,13 @@ public ServletAdapter buildServletAdapter() { return new ServletAdapter(buildAndStart(), streamTracerFactories, maxInboundMessageSize); } + /** + * Creates a {@link GrpcServlet}. + */ + public GrpcServlet buildServlet() { + return new GrpcServlet(buildServletAdapter()); + } + private ServerTransportListener buildAndStart() { Server server; try { From 06e63ec498d36726586de3b7a1adaa3c9217c6e1 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Mon, 21 Oct 2024 14:04:58 -0500 Subject: [PATCH 06/35] Bump grpc vers, add inprocess dep for tests --- gradle/libs.versions.toml | 3 ++- .../servlet/web/websocket/MultiplexedWebsocketStreamImpl.java | 2 +- .../io/grpc/servlet/web/websocket/WebsocketStreamImpl.java | 2 +- java-client/session/build.gradle | 1 + server/test-utils/build.gradle | 1 + 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f342d603109..dd3a9bc9674 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -31,7 +31,7 @@ google-findbugs = "3.0.2" google-java-allocation-instrumenter = "3.3.4" groovy = "3.0.22" # Only bump this in concert with boringssl -grpc = "1.58.0" +grpc = "1.63.2" guava = "33.2.0-jre" gwt = "2.11.0" # used by GwtTools @@ -174,6 +174,7 @@ grpc-protobuf = { module = "io.grpc:grpc-protobuf" } grpc-services = { module = "io.grpc:grpc-services" } grpc-stub = { module = "io.grpc:grpc-services" } grpc-testing = { module = "io.grpc:grpc-testing" } +grpc-inprocess = { module = "io.grpc:grpc-inprocess" } grpc-util = { module = "io.grpc:grpc-util" } guava = { module = "com.google.guava:guava", version.ref = "guava" } diff --git a/grpc-java/grpc-servlet-websocket-jakarta/src/main/java/io/grpc/servlet/web/websocket/MultiplexedWebsocketStreamImpl.java b/grpc-java/grpc-servlet-websocket-jakarta/src/main/java/io/grpc/servlet/web/websocket/MultiplexedWebsocketStreamImpl.java index e2aa3b59901..f8f58d19ae5 100644 --- a/grpc-java/grpc-servlet-websocket-jakarta/src/main/java/io/grpc/servlet/web/websocket/MultiplexedWebsocketStreamImpl.java +++ b/grpc-java/grpc-servlet-websocket-jakarta/src/main/java/io/grpc/servlet/web/websocket/MultiplexedWebsocketStreamImpl.java @@ -56,7 +56,7 @@ protected AbstractServerStream.Sink abstractServerStreamSink() { private final class Sink implements AbstractServerStream.Sink { @Override - public void writeHeaders(Metadata headers) { + public void writeHeaders(Metadata headers, boolean flush) { writeMetadataToStream(headers, false); } diff --git a/grpc-java/grpc-servlet-websocket-jakarta/src/main/java/io/grpc/servlet/web/websocket/WebsocketStreamImpl.java b/grpc-java/grpc-servlet-websocket-jakarta/src/main/java/io/grpc/servlet/web/websocket/WebsocketStreamImpl.java index 259e230c171..04d953eb877 100644 --- a/grpc-java/grpc-servlet-websocket-jakarta/src/main/java/io/grpc/servlet/web/websocket/WebsocketStreamImpl.java +++ b/grpc-java/grpc-servlet-websocket-jakarta/src/main/java/io/grpc/servlet/web/websocket/WebsocketStreamImpl.java @@ -49,7 +49,7 @@ protected Sink abstractServerStreamSink() { private final class Sink implements AbstractServerStream.Sink { @Override - public void writeHeaders(Metadata headers) { + public void writeHeaders(Metadata headers, boolean flush) { // headers/trailers are always sent as asci, colon-delimited pairs, with \r\n separating them. The // trailer response must be prefixed with 0x80 (0r 0x81 if compressed), followed by the length of the // message diff --git a/java-client/session/build.gradle b/java-client/session/build.gradle index ef210ce3c64..b875569f9df 100644 --- a/java-client/session/build.gradle +++ b/java-client/session/build.gradle @@ -36,6 +36,7 @@ dependencies { testImplementation libs.junit4 testImplementation libs.grpc.testing + testImplementation libs.grpc.inprocess testImplementation libs.assertj diff --git a/server/test-utils/build.gradle b/server/test-utils/build.gradle index d9b0e11bb0a..febc3c603b7 100644 --- a/server/test-utils/build.gradle +++ b/server/test-utils/build.gradle @@ -18,6 +18,7 @@ dependencies { api platform(libs.grpc.bom) api libs.grpc.testing + api libs.grpc.inprocess compileOnly project(':util-immutables') annotationProcessor libs.immutables.value From c8af47c2d4f0dc2f738c9021326e58dc6121395c Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Mon, 28 Oct 2024 13:38:46 -0500 Subject: [PATCH 07/35] Apply https://github.com/deephaven/deephaven-core/pull/6301 --- .../java/io/grpc/servlet/jakarta/ServletServerStream.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerStream.java b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerStream.java index bf65f3f3c75..2f3bce41f9a 100644 --- a/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerStream.java +++ b/grpc-java/grpc-servlet-jakarta/src/main/java/io/grpc/servlet/jakarta/ServletServerStream.java @@ -152,8 +152,8 @@ public void bytesRead(int numBytes) { @Override public void deframeFailed(Throwable cause) { - if (logger.isLoggable(FINE)) { - logger.log(FINE, String.format("[{%s}] Exception processing message", logId), cause); + if (logger.isLoggable(WARNING)) { + logger.log(WARNING, String.format("[{%s}] Exception processing message", logId), cause); } cancel(Status.fromThrowable(cause)); } From 57c8008225a2759e1905832f1654142afd65b95f Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Fri, 1 Nov 2024 10:26:36 -0500 Subject: [PATCH 08/35] Bump to 1.65.1 to better match arrow 18 --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c093d96566e..befca41d282 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -31,7 +31,7 @@ google-findbugs = "3.0.2" google-java-allocation-instrumenter = "3.3.4" groovy = "3.0.22" # Only bump this in concert with boringssl -grpc = "1.63.2" +grpc = "1.65.1" guava = "33.3.1-jre" gwt = "2.11.0" # used by GwtTools From 85f604ffaeea869bec593da6fc0aa51aac310903 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Wed, 6 Nov 2024 16:48:39 -0700 Subject: [PATCH 09/35] Version Upgrades; MavenLocal --- buildSrc/build.gradle | 1 + ...io.deephaven.repository-conventions.gradle | 7 +++++ .../barrage/BarrageSnapshotOptions.java | 24 ++++++++++----- .../barrage/BarrageSubscriptionOptions.java | 28 ++++++++++++------ .../chunk/DefaultChunkReadingFactory.java | 11 ++----- .../barrage/util/StreamReaderOptions.java | 29 ++++++++++++++++++- gradle/libs.versions.toml | 11 ++++--- .../deephaven/client/examples/DoExchange.java | 3 +- java-client/flight/build.gradle | 1 - .../test/FlightMessageRoundTripTest.java | 4 +-- .../AbstractTableSubscription.java | 2 +- .../TableViewportSubscription.java | 2 +- 12 files changed, 85 insertions(+), 38 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 76254a824b1..4ad980da367 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -10,6 +10,7 @@ java { } repositories { + mavenLocal() mavenCentral() maven { url "https://plugins.gradle.org/m2/" diff --git a/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle b/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle index 1deccf352c0..7abd48a3ea6 100644 --- a/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle +++ b/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle @@ -1,5 +1,12 @@ repositories { mavenCentral() + mavenLocal() + maven { + url 'https://oss.sonatype.org/content/repositories/snapshots' + content { + includeGroup 'com.vertispan.flatbuffers' + } + } maven { url 'https://jitpack.io' content { diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java index 7993a5f663c..98dc864b8db 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java @@ -7,6 +7,7 @@ import io.deephaven.annotations.BuildableStyle; import io.deephaven.barrage.flatbuf.BarrageSnapshotRequest; import io.deephaven.extensions.barrage.util.StreamReaderOptions; +import io.deephaven.util.annotations.FinalDefault; import org.immutables.value.Value.Default; import org.immutables.value.Value.Immutable; @@ -21,12 +22,11 @@ public static BarrageSnapshotOptions of(final io.deephaven.barrage.flatbuf.Barra if (options == null) { return builder().build(); } - final byte mode = options.columnConversionMode(); return builder() .useDeephavenNulls(options.useDeephavenNulls()) - .columnConversionMode(ColumnConversionMode.conversionModeFbToEnum(mode)) .batchSize(options.batchSize()) .maxMessageSize(options.maxMessageSize()) + .previewListLengthLimit(options.previewListLengthLimit()) .build(); } @@ -65,27 +65,37 @@ public int maxMessageSize() { @Override @Default - public ColumnConversionMode columnConversionMode() { - return ColumnConversionMode.Stringify; + public int previewListLengthLimit() { + return 0; } public int appendTo(FlatBufferBuilder builder) { return io.deephaven.barrage.flatbuf.BarrageSnapshotOptions.createBarrageSnapshotOptions( - builder, ColumnConversionMode.conversionModeEnumToFb(columnConversionMode()), useDeephavenNulls(), + builder, useDeephavenNulls(), batchSize(), - maxMessageSize()); + maxMessageSize(), + previewListLengthLimit()); } public interface Builder { Builder useDeephavenNulls(boolean useDeephavenNulls); - Builder columnConversionMode(ColumnConversionMode columnConversionMode); + /** + * Deprecated since 0.37.0 and is marked for removal. (our GWT artifacts do not yet support the attributes) + */ + @FinalDefault + @Deprecated + default Builder columnConversionMode(ColumnConversionMode columnConversionMode) { + return this; + } Builder batchSize(int batchSize); Builder maxMessageSize(int messageSize); + Builder previewListLengthLimit(int previewListLengthLimit); + BarrageSnapshotOptions build(); } } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java index 64e5d13219c..f8bd47deded 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java @@ -7,6 +7,7 @@ import io.deephaven.annotations.BuildableStyle; import io.deephaven.barrage.flatbuf.BarrageSubscriptionRequest; import io.deephaven.extensions.barrage.util.StreamReaderOptions; +import io.deephaven.util.annotations.FinalDefault; import org.immutables.value.Value.Default; import org.immutables.value.Value.Immutable; @@ -22,14 +23,13 @@ public static BarrageSubscriptionOptions of(final io.deephaven.barrage.flatbuf.B if (options == null) { return builder().build(); } - final byte mode = options.columnConversionMode(); return builder() .useDeephavenNulls(options.useDeephavenNulls()) - .columnConversionMode(ColumnConversionMode.conversionModeFbToEnum(mode)) .minUpdateIntervalMs(options.minUpdateIntervalMs()) .batchSize(options.batchSize()) .maxMessageSize(options.maxMessageSize()) .columnsAsList(options.columnsAsList()) + .previewListLengthLimit(options.previewListLengthLimit()) .build(); } @@ -64,11 +64,11 @@ public boolean columnsAsList() { * needed to perform barrage-acrobatics for both of them. This greatly reduces the burden each client adds to the * server's workload. If a given table does want a shorter interval, consider using that shorter interval for all * subscriptions to that table. - * + *

* The default interval can be set on the server with the flag * {@code io.deephaven.server.arrow.ArrowFlightUtil#DEFAULT_UPDATE_INTERVAL_MS}, or * {@code -Dbarrage.minUpdateInterval=1000}. - * + *

* Related, when shortening the minUpdateInterval, you typically want to shorten the server's UGP cycle enough to * update at least as quickly. This can be done on the server with the flag * {@code io.deephaven.engine.updategraph.impl.PeriodicUpdateGraph#defaultTargetCycleTime}, or @@ -101,17 +101,18 @@ public int maxMessageSize() { @Override @Default - public ColumnConversionMode columnConversionMode() { - return ColumnConversionMode.Stringify; + public int previewListLengthLimit() { + return 0; } public int appendTo(FlatBufferBuilder builder) { return io.deephaven.barrage.flatbuf.BarrageSubscriptionOptions.createBarrageSubscriptionOptions( - builder, ColumnConversionMode.conversionModeEnumToFb(columnConversionMode()), useDeephavenNulls(), + builder, useDeephavenNulls(), minUpdateIntervalMs(), batchSize(), maxMessageSize(), - columnsAsList()); + columnsAsList(), + previewListLengthLimit()); } public interface Builder { @@ -120,7 +121,14 @@ public interface Builder { Builder columnsAsList(boolean columnsAsList); - Builder columnConversionMode(ColumnConversionMode columnConversionMode); + /** + * Deprecated since 0.37.0 and is marked for removal. (our GWT artifacts do not yet support the attributes) + */ + @FinalDefault + @Deprecated + default Builder columnConversionMode(ColumnConversionMode columnConversionMode) { + return this; + } Builder minUpdateIntervalMs(int minUpdateIntervalMs); @@ -128,6 +136,8 @@ public interface Builder { Builder maxMessageSize(int messageSize); + Builder previewListLengthLimit(int previewListLengthLimit); + BarrageSubscriptionOptions build(); } } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java index 18b96bbc9a4..d8945ded93d 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java @@ -170,17 +170,12 @@ public ChunkReader getReader(StreamReaderOptions options, int factor, return StringChunkReader.STRING_CHUNK_READER; } // TODO (core#58): add custom barrage serialization/deserialization support - // // Migrate Schema to custom format when available. if (typeInfo.type() == Schema.class) { + // Migrate Schema to custom format when available. return SchemaChunkReader.SCHEMA_CHUNK_READER; } - // Note: this Stringify check should come last - if (options.columnConversionMode().equals(ColumnConversionMode.Stringify)) { - return StringChunkReader.STRING_CHUNK_READER; - } - // TODO (core#936): support column conversion modes - throw new UnsupportedOperationException( - "Do not yet support column conversion mode: " + options.columnConversionMode()); + // Otherwise fall through to default of writing via toString. + return StringChunkReader.STRING_CHUNK_READER; default: throw new UnsupportedOperationException(); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java index e00d9f3c6cd..32a8640b36b 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java @@ -5,6 +5,7 @@ import io.deephaven.extensions.barrage.ColumnConversionMode; import io.deephaven.util.QueryConstants; +import io.deephaven.util.annotations.FinalDefault; public interface StreamReaderOptions { /** @@ -13,9 +14,15 @@ public interface StreamReaderOptions { boolean useDeephavenNulls(); /** + * Deprecated since 0.37.0 and is marked for removal. (our GWT artifacts do not yet support the attributes) + * * @return the conversion mode to use for object columns */ - ColumnConversionMode columnConversionMode(); + @FinalDefault + @Deprecated + default ColumnConversionMode columnConversionMode() { + return ColumnConversionMode.Stringify; + } /** * @return the ideal number of records to send per record batch @@ -36,4 +43,24 @@ public interface StreamReaderOptions { default boolean columnsAsList() { return false; } + + /** + * The maximum length of any list / array to encode. + *

    + *
  • If zero, list lengths will not be limited.
    • + *
  • If non-zero, the server will limit the length of any encoded list / array to n elements, where n is the + * absolute value of the specified value.
    • + *
  • If the column value has length less than zero, the server will encode the last n elements of the list / + * array.
    • + *
+ *

+ * Note that the server will append an arbitrary value to indicate truncation; this value may not be the actual last + * value in the list / array. + * + * @return the maximum length of any list / array to encode; zero means no limit; negative values indicate to treat + * the limit as a tail instead of a head + */ + default int previewListLengthLimit() { + return 0; + } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e2ba3b23be9..eaafb5cdfb1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,6 +1,6 @@ [versions] airlift = "2.0.2" -arrow = "13.0.0" +arrow = "18.0.0" autoservice = "1.1.1" avro = "1.12.0" awssdk = "2.24.5" @@ -17,7 +17,7 @@ commons-text = "1.12.0" confluent = "7.6.0" confluent-kafka-clients = "7.6.0-ccs" dagger = "2.52" -deephaven-barrage = "0.6.0" +deephaven-barrage = "0.7.0-SNAPSHOT" deephaven-csv = "0.14.0" deephaven-hash = "0.1.0" deephaven-suan-shu = "0.1.1" @@ -25,7 +25,7 @@ dev-dirs = "26" dsi = "8.5.15" elemental = "1.2.1" f4b6a3 = "6.0.0" -flatbuffers = "1.12.0" +flatbuffers = "24.3.25" freemarker = "2.3.33" google-findbugs = "3.0.2" google-java-allocation-instrumenter = "3.3.4" @@ -64,7 +64,7 @@ pac4j = "5.7.0" parquet = "1.14.3" picocli = "4.7.6" postgresql = "42.7.4" -protobuf = "3.25.3" +protobuf = "3.25.4" randelshofer = "2.0.0" rdblue = "0.1.1" selenium = "4.26.0" @@ -77,7 +77,7 @@ trove = "3.0.3" undercouch = "2.15.1" univocity = "2.9.1" vertispan-nio = "1.0-alpha-2" -vertispan-flatbuffers-gwt = "1.12.0-1" +vertispan-flatbuffers-gwt = "24.3.25-1-SNAPSHOT" vertispan-ts-defs = "1.0.0-RC4" xerial = "3.47.0.0" @@ -98,7 +98,6 @@ arrow-compression = { module = "org.apache.arrow:arrow-compression", version.ref arrow-format = { module = "org.apache.arrow:arrow-format", version.ref = "arrow" } arrow-vector = { module = "org.apache.arrow:arrow-vector", version.ref = "arrow" } arrow-flight-core = { module = "org.apache.arrow:flight-core", version.ref = "arrow" } -arrow-flight-grpc = { module = "org.apache.arrow:flight-grpc", version.ref = "arrow" } autoservice = { module = "com.google.auto.service:auto-service-annotations", version.ref = "autoservice" } autoservice-compiler = { module = "com.google.auto.service:auto-service", version.ref = "autoservice" } diff --git a/java-client/flight-examples/src/main/java/io/deephaven/client/examples/DoExchange.java b/java-client/flight-examples/src/main/java/io/deephaven/client/examples/DoExchange.java index 61ff7d96b58..a3527cbdbcc 100644 --- a/java-client/flight-examples/src/main/java/io/deephaven/client/examples/DoExchange.java +++ b/java-client/flight-examples/src/main/java/io/deephaven/client/examples/DoExchange.java @@ -45,8 +45,7 @@ protected void execute(FlightSession flight) throws Exception { // you can use 0 for batch size and max message size to use server-side defaults int optOffset = - BarrageSnapshotOptions.createBarrageSnapshotOptions(metadata, ColumnConversionMode.Stringify, - false, 0, 0); + BarrageSnapshotOptions.createBarrageSnapshotOptions(metadata, false, 0, 0, 0); final int ticOffset = BarrageSnapshotRequest.createTicketVector(metadata, diff --git a/java-client/flight/build.gradle b/java-client/flight/build.gradle index e88d8d58b5f..20f1a66aadd 100644 --- a/java-client/flight/build.gradle +++ b/java-client/flight/build.gradle @@ -10,7 +10,6 @@ dependencies { implementation project(':proto:proto-backplane-grpc-flight') api libs.arrow.flight.core - implementation libs.arrow.flight.grpc api libs.arrow.vector compileOnly libs.autoservice annotationProcessor libs.autoservice.compiler diff --git a/server/test-utils/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java b/server/test-utils/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java index 7b2a002f177..9a9b934a3b0 100644 --- a/server/test-utils/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java +++ b/server/test-utils/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java @@ -767,8 +767,8 @@ public void testDoExchangeSnapshot() throws Exception { // use 0 for batch size and max message size to use server-side defaults int optOffset = - BarrageSnapshotOptions.createBarrageSnapshotOptions(metadata, ColumnConversionMode.Stringify, - false, 0, 0); + BarrageSnapshotOptions.createBarrageSnapshotOptions(metadata, + false, 0, 0, 0); final int ticOffset = BarrageSnapshotRequest.createTicketVector(metadata, diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java index f690d3df66f..a834ad5ca22 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java @@ -180,10 +180,10 @@ protected void sendBarrageSubscriptionRequest(RangeSet viewport, JsArray this.options = BarrageSubscriptionOptions.builder() .batchSize(WebBarrageSubscription.BATCH_SIZE) .maxMessageSize(WebBarrageSubscription.MAX_MESSAGE_SIZE) - .columnConversionMode(ColumnConversionMode.Stringify) .minUpdateIntervalMs(updateIntervalMs == null ? 0 : (int) (double) updateIntervalMs) .columnsAsList(false)// TODO(deephaven-core#5927) flip this to true .useDeephavenNulls(true) + .previewListLengthLimit(0) .build(); FlatBufferBuilder request = subscriptionRequest( Js.uncheckedCast(state.getHandle().getTicket()), diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java index e745899cb5a..f9bcdd76b30 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java @@ -339,8 +339,8 @@ public Promise snapshot(JsRangeSet rows, Column[] columns) { BarrageSnapshotOptions options = BarrageSnapshotOptions.builder() .batchSize(WebBarrageSubscription.BATCH_SIZE) .maxMessageSize(WebBarrageSubscription.MAX_MESSAGE_SIZE) - .columnConversionMode(ColumnConversionMode.Stringify) .useDeephavenNulls(true) + .previewListLengthLimit(0) .build(); WebBarrageSubscription snapshot = From 70a020730a8a96954e795509a24fee1904d32074 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 8 Nov 2024 10:22:06 -0700 Subject: [PATCH 10/35] Implement Simplified Viewport Table Updates in BMP/BT --- .../engine/table/impl/FlattenOperation.java | 43 ++++-- .../barrage/BarrageStreamGenerator.java | 17 ++- .../barrage/BarrageStreamGeneratorImpl.java | 134 +++++++++++++----- .../barrage/table/BarrageBlinkTable.java | 2 +- .../barrage/table/BarrageRedirectedTable.java | 60 +++++--- .../barrage/table/BarrageTable.java | 18 ++- .../barrage/util/ArrowToTableConverter.java | 2 +- .../barrage/util/BarrageStreamReader.java | 14 +- .../client/impl/BarrageSnapshotImpl.java | 43 ++++-- .../client/impl/BarrageSubscriptionImpl.java | 50 +++++-- .../barrage/BarrageMessageProducer.java | 125 ++++++++++------ .../HierarchicalTableViewSubscription.java | 16 ++- .../server/barrage/BarrageBlinkTableTest.java | 2 +- .../barrage/BarrageMessageRoundTripTest.java | 2 +- 14 files changed, 383 insertions(+), 145 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/FlattenOperation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/FlattenOperation.java index 10059e035ca..c77674d346b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/FlattenOperation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/FlattenOperation.java @@ -12,6 +12,7 @@ import io.deephaven.engine.table.impl.sources.RedirectedColumnSource; import io.deephaven.engine.table.impl.util.*; import org.apache.commons.lang3.mutable.MutableObject; +import org.jetbrains.annotations.NotNull; import java.util.LinkedHashMap; import java.util.Map; @@ -98,6 +99,34 @@ private void onUpdate(final TableUpdate upstream) { try (final RowSet prevRowSet = rowSet.copyPrev()) { downstream.removed = prevRowSet.invert(upstream.removed()); } + + if (newSize < prevSize) { + resultTable.getRowSet().writableCast().removeRange(newSize, prevSize - 1); + } else if (newSize > prevSize) { + resultTable.getRowSet().writableCast().insertRange(prevSize, newSize - 1); + } + + downstream.shifted = computeFlattenedRowSetShiftData(downstream.removed(), downstream.added(), prevSize); + prevSize = newSize; + resultTable.notifyListeners(downstream); + } + + /** + * Compute the shift data for a flattened row set given which rows were removed and which were added. + * + * @param removed the rows that were removed in the flattened pre-update key-space + * @param added the rows that were added in the flattened post-update key-space + * @param prevSize the size of the table before the update + * @return the shift data + */ + public static RowSetShiftData computeFlattenedRowSetShiftData( + @NotNull final RowSet removed, + @NotNull final RowSet added, + final long prevSize) { + if (removed.isEmpty() && added.isEmpty()) { + return RowSetShiftData.EMPTY; + } + final RowSetShiftData.Builder outShifted = new RowSetShiftData.Builder(); // Helper to ensure that we can prime iterators and still detect the end. @@ -110,8 +139,8 @@ private void onUpdate(final TableUpdate upstream) { }; // Create our range iterators and prime them. - final MutableObject rmIt = new MutableObject<>(downstream.removed().rangeIterator()); - final MutableObject addIt = new MutableObject<>(downstream.added().rangeIterator()); + final MutableObject rmIt = new MutableObject<>(removed.rangeIterator()); + final MutableObject addIt = new MutableObject<>(added.rangeIterator()); updateIt.accept(rmIt); updateIt.accept(addIt); @@ -163,14 +192,6 @@ private void onUpdate(final TableUpdate upstream) { outShifted.shiftRange(currMarker, prevSize - 1, currDelta); } - if (newSize < prevSize) { - resultTable.getRowSet().writableCast().removeRange(newSize, prevSize - 1); - } else if (newSize > prevSize) { - resultTable.getRowSet().writableCast().insertRange(prevSize, newSize - 1); - } - - downstream.shifted = outShifted.build(); - prevSize = newSize; - resultTable.notifyListeners(downstream); + return outShifted.build(); } } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java index 2c0375235ae..730feaee99c 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java @@ -59,7 +59,7 @@ BarrageStreamGenerator newGenerator( * Obtain a Full-Subscription View of this StreamGenerator that can be sent to a single subscriber. * * @param options serialization options for this specific view - * @param isInitialSnapshot indicates whether or not this is the first snapshot for the listener + * @param isInitialSnapshot indicates whether this is the first snapshot for the listener * @return a MessageView filtered by the subscription properties that can be sent to that subscriber */ MessageView getSubView(BarrageSubscriptionOptions options, boolean isInitialSnapshot); @@ -68,15 +68,24 @@ BarrageStreamGenerator newGenerator( * Obtain a View of this StreamGenerator that can be sent to a single subscriber. * * @param options serialization options for this specific view - * @param isInitialSnapshot indicates whether or not this is the first snapshot for the listener + * @param isInitialSnapshot indicates whether this is the first snapshot for the listener + * @param isFullSubscription whether this is a full subscription (possibly a growing viewport) * @param viewport is the position-space viewport * @param reverseViewport is the viewport reversed (relative to end of table instead of beginning) + * @param keyspaceViewportPrev is the key-space viewport in prior to applying the update * @param keyspaceViewport is the key-space viewport * @param subscribedColumns are the columns subscribed for this view * @return a MessageView filtered by the subscription properties that can be sent to that subscriber */ - MessageView getSubView(BarrageSubscriptionOptions options, boolean isInitialSnapshot, @Nullable RowSet viewport, - boolean reverseViewport, @Nullable RowSet keyspaceViewport, BitSet subscribedColumns); + MessageView getSubView( + BarrageSubscriptionOptions options, + boolean isInitialSnapshot, + boolean isFullSubscription, + @Nullable RowSet viewport, + boolean reverseViewport, + @Nullable RowSet keyspaceViewportPrev, + @Nullable RowSet keyspaceViewport, + BitSet subscribedColumns); /** * Obtain a Full-Snapshot View of this StreamGenerator that can be sent to a single requestor. diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index c63a527104b..d02556049e8 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -56,8 +56,8 @@ import static io.deephaven.proto.flight.util.MessageHelper.toIpcBytes; public class BarrageStreamGeneratorImpl implements BarrageStreamGenerator { - private static final Logger log = LoggerFactory.getLogger(BarrageStreamGeneratorImpl.class); + // NB: This should likely be something smaller, such as 1<<16, but since the js api is not yet able // to receive multiple record batches we crank this up to MAX_INT. private static final int DEFAULT_BATCH_SIZE = Configuration.getInstance() @@ -175,6 +175,7 @@ public BarrageStreamGeneratorImpl(final BarrageMessage message, addColumnData = new ChunkListInputStreamGenerator[message.addColumnData.length]; for (int i = 0; i < message.addColumnData.length; ++i) { BarrageMessage.AddColumnData columnData = message.addColumnData[i]; + // noinspection resource addColumnData[i] = new ChunkListInputStreamGenerator(DefaultChunkInputStreamGeneratorFactory.INSTANCE, columnData.type, columnData.componentType, columnData.data, columnData.chunkType); @@ -182,6 +183,7 @@ public BarrageStreamGeneratorImpl(final BarrageMessage message, modColumnData = new ModColumnGenerator[message.modColumnData.length]; for (int i = 0; i < modColumnData.length; ++i) { + // noinspection resource modColumnData[i] = new ModColumnGenerator(DefaultChunkInputStreamGeneratorFactory.INSTANCE, message.modColumnData[i]); } @@ -217,65 +219,79 @@ public void close() { * Obtain a View of this StreamGenerator that can be sent to a single subscriber. * * @param options serialization options for this specific view - * @param isInitialSnapshot indicates whether or not this is the first snapshot for the listener + * @param isInitialSnapshot indicates whether this is the first snapshot for the listener + * @param isFullSubscription whether this is a full subscription (possibly a growing viewport) * @param viewport is the position-space viewport * @param reverseViewport is the viewport reversed (relative to end of table instead of beginning) + * @param keyspaceViewportPrev is the key-space viewport in prev key-space * @param keyspaceViewport is the key-space viewport * @param subscribedColumns are the columns subscribed for this view * @return a MessageView filtered by the subscription properties that can be sent to that subscriber */ @Override - public MessageView getSubView(final BarrageSubscriptionOptions options, + public MessageView getSubView( + final BarrageSubscriptionOptions options, final boolean isInitialSnapshot, + final boolean isFullSubscription, @Nullable final RowSet viewport, final boolean reverseViewport, + @Nullable final RowSet keyspaceViewportPrev, @Nullable final RowSet keyspaceViewport, @Nullable final BitSet subscribedColumns) { - return new SubView(options, isInitialSnapshot, viewport, reverseViewport, keyspaceViewport, - subscribedColumns); + return new SubView(options, isInitialSnapshot, isFullSubscription, viewport, reverseViewport, + keyspaceViewportPrev, keyspaceViewport, subscribedColumns); } /** * Obtain a Full-Subscription View of this StreamGenerator that can be sent to a single subscriber. * * @param options serialization options for this specific view - * @param isInitialSnapshot indicates whether or not this is the first snapshot for the listener + * @param isInitialSnapshot indicates whether this is the first snapshot for the listener * @return a MessageView filtered by the subscription properties that can be sent to that subscriber */ @Override public MessageView getSubView(BarrageSubscriptionOptions options, boolean isInitialSnapshot) { - return getSubView(options, isInitialSnapshot, null, false, null, null); + return getSubView(options, isInitialSnapshot, true, null, false, null, null, null); } private final class SubView implements RecordBatchMessageView { private final BarrageSubscriptionOptions options; private final boolean isInitialSnapshot; + private final boolean isFullSubscription; private final RowSet viewport; private final boolean reverseViewport; + private final RowSet keyspaceViewportPrev; private final RowSet keyspaceViewport; private final BitSet subscribedColumns; private final long numAddRows; private final long numModRows; - private final RowSet addRowOffsets; private final RowSet addRowKeys; + private final RowSet addRowOffsets; + private final RowSet[] modRowKeys; private final RowSet[] modRowOffsets; public SubView(final BarrageSubscriptionOptions options, final boolean isInitialSnapshot, + final boolean isFullSubscription, @Nullable final RowSet viewport, final boolean reverseViewport, + @Nullable final RowSet keyspaceViewportPrev, @Nullable final RowSet keyspaceViewport, @Nullable final BitSet subscribedColumns) { this.options = options; this.isInitialSnapshot = isInitialSnapshot; + this.isFullSubscription = isFullSubscription; this.viewport = viewport; this.reverseViewport = reverseViewport; + this.keyspaceViewportPrev = keyspaceViewportPrev; this.keyspaceViewport = keyspaceViewport; this.subscribedColumns = subscribedColumns; if (keyspaceViewport != null) { + this.modRowKeys = new WritableRowSet[modColumnData.length]; this.modRowOffsets = new WritableRowSet[modColumnData.length]; } else { + this.modRowKeys = null; this.modRowOffsets = null; } @@ -286,7 +302,12 @@ public SubView(final BarrageSubscriptionOptions options, if (keyspaceViewport != null) { try (WritableRowSet intersect = keyspaceViewport.intersect(mcd.rowsModified.original)) { - this.modRowOffsets[ii] = mcd.rowsModified.original.invert(intersect); + if (isFullSubscription) { + modRowKeys[ii] = intersect.copy(); + } else { + modRowKeys[ii] = keyspaceViewport.invert(intersect); + } + modRowOffsets[ii] = mcd.rowsModified.original.invert(intersect); numModRows = Math.max(numModRows, intersect.size()); } } else { @@ -296,8 +317,14 @@ public SubView(final BarrageSubscriptionOptions options, this.numModRows = numModRows; if (keyspaceViewport != null) { - addRowKeys = keyspaceViewport.intersect(rowsIncluded.original); - addRowOffsets = rowsIncluded.original.invert(addRowKeys); + try (WritableRowSet intersect = keyspaceViewport.intersect(rowsIncluded.original)) { + if (isFullSubscription) { + addRowKeys = intersect.copy(); + } else { + addRowKeys = keyspaceViewport.invert(intersect); + } + addRowOffsets = rowsIncluded.original.invert(intersect); + } } else if (!rowsAdded.original.equals(rowsIncluded.original)) { // there are scoped rows included in the chunks that need to be removed addRowKeys = rowsAdded.original.copy(); @@ -332,22 +359,23 @@ public void forEachStream(Consumer visitor) throws IOExcepti } // send the add batches (if any) - processBatches(visitor, this, numAddRows, maxBatchSize, metadata, - BarrageStreamGeneratorImpl.this::appendAddColumns, bytesWritten); - - // send the mod batches (if any) but don't send metadata twice - processBatches(visitor, this, numModRows, maxBatchSize, numAddRows > 0 ? null : metadata, - BarrageStreamGeneratorImpl.this::appendModColumns, bytesWritten); + try { + processBatches(visitor, this, numAddRows, maxBatchSize, metadata, + BarrageStreamGeneratorImpl.this::appendAddColumns, bytesWritten); - // clean up the helper indexes - addRowOffsets.close(); - addRowKeys.close(); - if (modRowOffsets != null) { - for (final RowSet modViewport : modRowOffsets) { - modViewport.close(); + // send the mod batches (if any) but don't send metadata twice + processBatches(visitor, this, numModRows, maxBatchSize, numAddRows > 0 ? null : metadata, + BarrageStreamGeneratorImpl.this::appendModColumns, bytesWritten); + } finally { + // clean up the helper indexes + addRowOffsets.close(); + addRowKeys.close(); + if (modRowOffsets != null) { + SafeCloseable.closeAll(modRowKeys); + SafeCloseable.closeAll(modRowOffsets); } + writeConsumer.onWrite(bytesWritten.get(), System.nanoTime() - startTm); } - writeConsumer.onWrite(bytesWritten.get(), System.nanoTime() - startTm); } private int batchSize() { @@ -397,32 +425,61 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { } final int rowsAddedOffset; - if (isSnapshot && !isInitialSnapshot) { - // client's don't need/want to receive the full RowSet on every snapshot + if (!isFullSubscription) { + try (final RowSetGenerator addRowsGen = new RowSetGenerator(addRowKeys)) { + rowsAddedOffset = addRowsGen.addToFlatBuffer(metadata); + } + } else if (isSnapshot && !isInitialSnapshot) { + // full subscription clients don't need/want to receive the full RowSet on every snapshot rowsAddedOffset = EmptyRowSetGenerator.INSTANCE.addToFlatBuffer(metadata); } else { rowsAddedOffset = rowsAdded.addToFlatBuffer(metadata); } - final int rowsRemovedOffset = rowsRemoved.addToFlatBuffer(metadata); - final int shiftDataOffset = shifted.addToFlatBuffer(metadata); + final int rowsRemovedOffset; + if (isViewport() && !isFullSubscription) { + try (final WritableRowSet removedInViewport = keyspaceViewportPrev.intersect(rowsRemoved.original); + final WritableRowSet removedInPositionSpace = keyspaceViewportPrev.invert(removedInViewport); + final RowSetGenerator rmRowsGen = new RowSetGenerator(removedInPositionSpace)) { + rowsRemovedOffset = rmRowsGen.addToFlatBuffer(metadata); + } + } else { + rowsRemovedOffset = rowsRemoved.addToFlatBuffer(metadata); + } + final int shiftDataOffset; + if (isViewport() && !isFullSubscription) { + // never send shifts to a viewport subscriber + shiftDataOffset = 0; + } else { + shiftDataOffset = shifted.addToFlatBuffer(metadata); + } // Added Chunk Data: int addedRowsIncludedOffset = 0; // don't send `rowsIncluded` when identical to `rowsAdded`, client will infer they are the same if (isSnapshot || !addRowKeys.equals(rowsAdded.original)) { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(addRowKeys, metadata); + if (isViewport() && !isFullSubscription) { + try (final WritableRowSet inclInViewport = keyspaceViewport.intersect(rowsIncluded.original); + final WritableRowSet inclInPositionSpace = keyspaceViewport.invert(inclInViewport); + final RowSetGenerator inclRowsGen = new RowSetGenerator(inclInPositionSpace)) { + addedRowsIncludedOffset = inclRowsGen.addToFlatBuffer(metadata); + } + } else { + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(addRowKeys, metadata); + } } // now add mod-column streams, and write the mod column indexes TIntArrayList modOffsets = new TIntArrayList(modColumnData.length); - for (final ModColumnGenerator mcd : modColumnData) { + for (int ii = 0; ii < modColumnData.length; ++ii) { final int myModRowOffset; if (keyspaceViewport != null) { - myModRowOffset = mcd.rowsModified.addToFlatBuffer(keyspaceViewport, metadata); + try (final RowSetGenerator modRowsGen = new RowSetGenerator(modRowKeys[ii])) { + myModRowOffset = modRowsGen.addToFlatBuffer(metadata); + } } else { - myModRowOffset = mcd.rowsModified.addToFlatBuffer(metadata); + myModRowOffset = modColumnData[ii].rowsModified.addToFlatBuffer(metadata); } modOffsets.add(BarrageModColumnMetadata.createBarrageModColumnMetadata(metadata, myModRowOffset)); } @@ -663,9 +720,13 @@ int visit(final RecordBatchMessageView view, final long startRange, final int ta * @param columnVisitor the helper method responsible for appending the payload columns to the RecordBatch * @return an InputStream ready to be drained by GRPC */ - private DefensiveDrainable getInputStream(final RecordBatchMessageView view, final long offset, + private DefensiveDrainable getInputStream( + final RecordBatchMessageView view, + final long offset, final int targetBatchSize, - final MutableInt actualBatchSize, final ByteBuffer metadata, final ColumnVisitor columnVisitor) + final MutableInt actualBatchSize, + final ByteBuffer metadata, + final ColumnVisitor columnVisitor) throws IOException { final ArrayDeque streams = new ArrayDeque<>(); final MutableInt size = new MutableInt(); @@ -703,12 +764,14 @@ private DefensiveDrainable getInputStream(final RecordBatchMessageView view, fin nodeOffsets.ensureCapacity(addColumnData.length); nodeOffsets.get().setSize(0); bufferInfos.ensureCapacity(addColumnData.length * 3); + // noinspection DataFlowIssue bufferInfos.get().setSize(0); final MutableLong totalBufferLength = new MutableLong(); final ChunkInputStreamGenerator.FieldNodeListener fieldNodeListener = (numElements, nullCount) -> { nodeOffsets.ensureCapacityPreserve(nodeOffsets.get().size() + 1); + // noinspection resource nodeOffsets.get().asWritableObjectChunk() .add(new ChunkInputStreamGenerator.FieldNodeInfo(numElements, nullCount)); }; @@ -1066,7 +1129,6 @@ public static class RowSetGenerator extends ByteArrayGenerator implements SafeCl public RowSetGenerator(final RowSet rowSet) throws IOException { this.original = rowSet.copy(); - // noinspection UnstableApiUsage try (final ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream(); final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos)) { ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, rowSet); @@ -1099,7 +1161,6 @@ protected int addToFlatBuffer(final RowSet viewport, final FlatBufferBuilder bui final int nlen; final byte[] nraw; - // noinspection UnstableApiUsage try (final ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream(); final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos); final RowSet viewOfOriginal = original.intersect(viewport)) { @@ -1143,7 +1204,6 @@ public RowSetShiftDataGenerator(final RowSetShiftData shifted) throws IOExceptio } } - // noinspection UnstableApiUsage try (final RowSet sRange = sRangeBuilder.build(); final RowSet eRange = eRangeBuilder.build(); final RowSet dest = destBuilder.build(); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageBlinkTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageBlinkTable.java index 2c9896af2d9..c4ecc6004dd 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageBlinkTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageBlinkTable.java @@ -47,7 +47,7 @@ protected BarrageBlinkTable( final WritableColumnSource[] writableSources, final Map attributes, @Nullable final ViewportChangedCallback vpCallback) { - super(registrar, notificationQueue, executorService, columns, writableSources, attributes, vpCallback); + super(registrar, notificationQueue, executorService, columns, writableSources, attributes, true, vpCallback); setFlat(); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java index fe735a20f5b..a4c3906959a 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java @@ -10,6 +10,7 @@ import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.rowset.RowSetShiftData; import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; import io.deephaven.engine.rowset.chunkattributes.RowKeys; @@ -20,6 +21,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableUpdate; import io.deephaven.engine.table.WritableColumnSource; +import io.deephaven.engine.table.impl.FlattenOperation; import io.deephaven.engine.table.impl.TableUpdateImpl; import io.deephaven.engine.table.impl.util.BarrageMessage; import io.deephaven.engine.table.impl.util.UpdateCoalescer; @@ -30,7 +32,6 @@ import org.jetbrains.annotations.Nullable; import java.util.ArrayDeque; -import java.util.BitSet; import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.ScheduledExecutorService; @@ -55,10 +56,12 @@ protected BarrageRedirectedTable(final UpdateSourceRegistrar registrar, final WritableRowRedirection rowRedirection, final Map attributes, final boolean isFlat, + final boolean isFullSubscription, @Nullable final ViewportChangedCallback vpCallback) { - super(registrar, notificationQueue, executorService, columns, writableSources, attributes, vpCallback); + super(registrar, notificationQueue, executorService, columns, writableSources, attributes, isFullSubscription, + vpCallback); this.rowRedirection = rowRedirection; - if (isFlat) { + if (!isFullSubscription || isFlat) { setFlat(); } } @@ -105,26 +108,47 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC final boolean serverReverseViewport = getServerReverseViewport(); try (final RowSet currRowsFromPrev = currentRowSet.copy(); - final WritableRowSet populatedRows = - (serverViewport != null - ? currentRowSet.subSetForPositions(serverViewport, serverReverseViewport) - : null)) { + final WritableRowSet populatedRows = serverViewport != null && isFullSubscription + ? currentRowSet.subSetForPositions(serverViewport, serverReverseViewport) + : null) { // removes - currentRowSet.remove(update.rowsRemoved); - try (final RowSet removed = serverViewport != null ? populatedRows.extract(update.rowsRemoved) : null) { + final long prevSize = currentRowSet.size(); + if (isFullSubscription) { + currentRowSet.remove(update.rowsRemoved); + } + try (final RowSet removed = populatedRows != null ? populatedRows.extract(update.rowsRemoved) : null) { freeRows(removed != null ? removed : update.rowsRemoved); } + final RowSetShiftData updateShiftData; + if (isFullSubscription) { + updateShiftData = update.shifted; + } else { + updateShiftData = FlattenOperation.computeFlattenedRowSetShiftData( + update.rowsRemoved, update.rowsAdded, prevSize); + } + // shifts - if (update.shifted.nonempty()) { - rowRedirection.applyShift(currentRowSet, update.shifted); - update.shifted.apply(currentRowSet); + if (updateShiftData.nonempty()) { + rowRedirection.applyShift(currentRowSet, updateShiftData); + if (isFullSubscription) { + updateShiftData.apply(currentRowSet); + } if (populatedRows != null) { - update.shifted.apply(populatedRows); + updateShiftData.apply(populatedRows); + } + } + if (isFullSubscription) { + currentRowSet.insert(update.rowsAdded); + } else { + final long newSize = prevSize - update.rowsRemoved.size() + update.rowsAdded.size(); + if (newSize < prevSize) { + currentRowSet.removeRange(newSize, prevSize - 1); + } else if (newSize > prevSize) { + currentRowSet.insertRange(prevSize, newSize - 1); } } - currentRowSet.insert(update.rowsAdded); final WritableRowSet totalMods = RowSetFactory.empty(); for (int i = 0; i < update.modColumnData.length; ++i) { @@ -231,12 +255,16 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC } // remove all data outside of our viewport - if (serverViewport != null) { + if (populatedRows != null) { try (final RowSet newPopulated = currentRowSet.subSetForPositions(serverViewport, serverReverseViewport)) { populatedRows.remove(newPopulated); freeRows(populatedRows); } + } else if (!isFullSubscription && prevSize > currentRowSet.size()) { + try (final RowSet toFree = RowSetFactory.fromRange(currentRowSet.size(), prevSize - 1)) { + freeRows(toFree); + } } if (update.isSnapshot && !mightBeInitialSnapshot) { @@ -247,7 +275,7 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC } final TableUpdate downstream = new TableUpdateImpl( - update.rowsAdded.copy(), update.rowsRemoved.copy(), totalMods, update.shifted, modifiedColumnSet); + update.rowsAdded.copy(), update.rowsRemoved.copy(), totalMods, updateShiftData, modifiedColumnSet); return (coalescer == null) ? new UpdateCoalescer(currRowsFromPrev, downstream) : coalescer.update(downstream); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index 514f76f05d6..6248d63a398 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -101,7 +101,7 @@ public interface ViewportChangedCallback { * Due to the asynchronous aspect of this protocol, the client may have multiple requests in-flight and the server * may choose to honor the most recent request and assumes that the client no longer wants earlier but unacked * viewport changes. - * + *

* The server notifies the client which viewport it is respecting by including it inside of each snapshot. Note that * the server assumes that the client has maintained its state prior to these server-side viewport acks and will not * re-send data that the client should already have within the existing viewport. @@ -110,6 +110,13 @@ public interface ViewportChangedCallback { private BitSet serverColumns; private boolean serverReverseViewport; + /** + * A full subscription is where the server sends all data to the client. The server is allowed to initially send + * growing viewports to the client to avoid contention on the update graph lock. Once the server has sent a full + * subscription, it will not send any more snapshots and serverViewport will be set to null. + */ + protected final boolean isFullSubscription; + /** * A batch of updates may change the viewport more than once, but we cannot deliver until the updates have been * propagated to this BarrageTable and its last notification step has been updated. @@ -151,6 +158,7 @@ protected BarrageTable(final UpdateSourceRegistrar registrar, final LinkedHashMap> columns, final WritableColumnSource[] writableSources, final Map attributes, + final boolean isFullSubscription, @Nullable final ViewportChangedCallback viewportChangedCallback) { super(RowSetFactory.empty().toTracking(), columns); attributes.entrySet().stream() @@ -160,6 +168,7 @@ protected BarrageTable(final UpdateSourceRegistrar registrar, this.registrar = registrar; this.notificationQueue = notificationQueue; this.executorService = executorService; + this.isFullSubscription = isFullSubscription; final String tableKey = BarragePerformanceLog.getKeyFor(this); if (executorService == null || tableKey == null) { @@ -423,6 +432,7 @@ private void enqueueError(final Throwable e) { * @param executorService an executor service used to flush stats * @param tableDefinition the table definition * @param attributes Key-Value pairs of attributes to forward to the QueryTable's metadata + * @param isFullSubscription whether this table is a full subscription * * @return a properly initialized {@link BarrageTable} */ @@ -431,9 +441,10 @@ public static BarrageTable make( @Nullable final ScheduledExecutorService executorService, final TableDefinition tableDefinition, final Map attributes, + final boolean isFullSubscription, @Nullable final ViewportChangedCallback vpCallback) { final UpdateGraph ug = ExecutionContext.getContext().getUpdateGraph(); - return make(ug, ug, executorService, tableDefinition, attributes, vpCallback); + return make(ug, ug, executorService, tableDefinition, attributes, isFullSubscription, vpCallback); } @VisibleForTesting @@ -443,6 +454,7 @@ public static BarrageTable make( @Nullable final ScheduledExecutorService executor, final TableDefinition tableDefinition, final Map attributes, + final boolean isFullSubscription, @Nullable final ViewportChangedCallback vpCallback) { final List> columns = tableDefinition.getColumns(); final WritableColumnSource[] writableSources = new WritableColumnSource[columns.size()]; @@ -471,7 +483,7 @@ public static BarrageTable make( makeColumns(columns, writableSources, rowRedirection); table = new BarrageRedirectedTable( registrar, queue, executor, finalColumns, writableSources, rowRedirection, attributes, isFlat, - vpCallback); + isFullSubscription, vpCallback); } return table; diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/ArrowToTableConverter.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/ArrowToTableConverter.java index 2c8388ad9d1..5e83dea4b63 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/ArrowToTableConverter.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/ArrowToTableConverter.java @@ -174,7 +174,7 @@ protected void configureWithSchema(final Schema schema) { } final BarrageUtil.ConvertedArrowSchema result = BarrageUtil.convertArrowSchema(schema); - resultTable = BarrageTable.make(null, result.tableDef, result.attributes, null); + resultTable = BarrageTable.make(null, result.tableDef, result.attributes, true, null); resultTable.setFlat(); ChunkType[] columnChunkTypes = result.computeWireChunkTypes(); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index 496de4ed31d..0f206a7d8e4 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -51,7 +51,7 @@ public class BarrageStreamReader implements StreamReader { // We would like to use jdk.internal.util.ArraysSupport.MAX_ARRAY_LENGTH, but it is not exported private static final int MAX_CHUNK_SIZE = ArrayUtil.MAX_ARRAY_SIZE; - private final LongConsumer deserializeTmConsumer; + private volatile LongConsumer deserializeTmConsumer; private long numAddRowsRead = 0; private long numAddRowsTotal = 0; @@ -63,10 +63,19 @@ public class BarrageStreamReader implements StreamReader { private final ChunkReader.Factory chunkReaderFactory = DefaultChunkReadingFactory.INSTANCE; private final List readers = new ArrayList<>(); + public BarrageStreamReader() { + this(tm -> { + }); + } + public BarrageStreamReader(final LongConsumer deserializeTmConsumer) { this.deserializeTmConsumer = deserializeTmConsumer; } + public void setDeserializeTmConsumer(final LongConsumer deserializeTmConsumer) { + this.deserializeTmConsumer = deserializeTmConsumer; + } + @Override public BarrageMessage safelyParseFrom(final StreamReaderOptions options, final ChunkType[] columnChunkTypes, @@ -126,7 +135,8 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.lastSeq = metadata.lastSeq(); msg.rowsAdded = extractIndex(metadata.addedRowsAsByteBuffer()); msg.rowsRemoved = extractIndex(metadata.removedRowsAsByteBuffer()); - msg.shifted = extractIndexShiftData(metadata.shiftDataAsByteBuffer()); + ByteBuffer shiftData = metadata.shiftDataAsByteBuffer(); + msg.shifted = shiftData != null ? extractIndexShiftData(shiftData) : RowSetShiftData.EMPTY; final ByteBuffer rowsIncluded = metadata.addedRowsIncludedAsByteBuffer(); msg.rowsIncluded = rowsIncluded != null ? extractIndex(rowsIncluded) : msg.rowsAdded.copy(); diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java index f34382297e0..133c72fa06a 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java @@ -5,6 +5,7 @@ import com.google.flatbuffers.FlatBufferBuilder; import com.google.protobuf.ByteStringAccess; +import com.google.rpc.Code; import io.deephaven.UncheckedDeephavenException; import io.deephaven.barrage.flatbuf.*; import io.deephaven.base.log.LogOutput; @@ -14,7 +15,6 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; -import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.util.BarrageMessage; import io.deephaven.extensions.barrage.BarrageSnapshotOptions; @@ -22,10 +22,12 @@ import io.deephaven.extensions.barrage.util.*; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; +import io.deephaven.proto.util.Exceptions; import io.grpc.CallOptions; import io.grpc.ClientCall; import io.grpc.Context; import io.grpc.MethodDescriptor; +import io.grpc.StatusRuntimeException; import io.grpc.protobuf.ProtoUtils; import io.grpc.stub.ClientCallStreamObserver; import io.grpc.stub.ClientCalls; @@ -54,11 +56,14 @@ public class BarrageSnapshotImpl extends ReferenceCountedLivenessNode implements private static final Logger log = LoggerFactory.getLogger(BarrageSnapshotImpl.class); private final String logName; + private final ScheduledExecutorService executorService; private final TableHandle tableHandle; private final BarrageSnapshotOptions options; private final ClientCallStreamObserver observer; + private final BarrageUtil.ConvertedArrowSchema schema; + private final BarrageStreamReader barrageStreamReader; - private final BarrageTable resultTable; + private volatile BarrageTable resultTable; private final CompletableFuture future; private volatile int connected = 1; @@ -82,18 +87,17 @@ public class BarrageSnapshotImpl extends ReferenceCountedLivenessNode implements super(false); this.logName = tableHandle.exportId().toString(); + this.executorService = executorService; this.options = options; this.tableHandle = tableHandle; - final BarrageUtil.ConvertedArrowSchema schema = BarrageUtil.convertArrowSchema(tableHandle.response()); - final TableDefinition tableDefinition = schema.tableDef; - resultTable = BarrageTable.make(executorService, tableDefinition, schema.attributes, new CheckForCompletion()); + schema = BarrageUtil.convertArrowSchema(tableHandle.response()); future = new SnapshotCompletableFuture(); + barrageStreamReader = new BarrageStreamReader(); final MethodDescriptor snapshotDescriptor = getClientDoExchangeDescriptor(options, schema.computeWireChunkTypes(), schema.computeWireTypes(), - schema.computeWireComponentTypes(), - new BarrageStreamReader(resultTable.getDeserializationTmConsumer())); + schema.computeWireComponentTypes(), barrageStreamReader); // We need to ensure that the DoExchange RPC does not get attached to the server RPC when this is being called // from a Deephaven server RPC thread. If we need to generalize this in the future, we may wrap this logic in a @@ -145,6 +149,15 @@ public void onNext(final BarrageMessage barrageMessage) { rowsReceived += resultSize; + if (resultTable == null) { + log.error().append(BarrageSnapshotImpl.this) + .append(": Received data before snapshot was requested").endl(); + final StatusRuntimeException sre = Exceptions.statusRuntimeException( + Code.FAILED_PRECONDITION, "Received data before snapshot was requested"); + GrpcUtil.safelyError(observer, sre); + future.completeExceptionally(sre); + return; + } resultTable.handleBarrageMessage(barrageMessage); } } @@ -160,9 +173,14 @@ public void onError(final Throwable t) { .append(t).endl(); final String label = TableSpecLabeler.of(tableHandle.export().table()); - // this error will always be propagated to our CheckForCompletion#onError callback - resultTable.handleBarrageError(new TableDataException( - String.format("Barrage snapshot error for %s (%s)", logName, label), t)); + final TableDataException tde = new TableDataException( + String.format("Barrage snapshot error for %s (%s)", logName, label), t); + if (resultTable != null) { + // this error will always be propagated to our CheckForCompletion#onError callback + resultTable.handleBarrageError(tde); + } else { + future.completeExceptionally(t); + } cleanup(); } @@ -200,6 +218,11 @@ public Future
partialTable( alreadyUsed = true; } + final boolean isFullSubscription = viewport == null; + resultTable = BarrageTable.make(executorService, schema.tableDef, schema.attributes, isFullSubscription, + new CheckForCompletion()); + barrageStreamReader.setDeserializeTmConsumer(resultTable.getDeserializationTmConsumer()); + // Send the snapshot request: observer.onNext(FlightData.newBuilder() .setAppMetadata(ByteStringAccess.wrap(makeRequestInternal(viewport, columns, reverseViewport, options))) diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java index 26c0672e649..2dfdc9fc36f 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java @@ -5,6 +5,7 @@ import com.google.flatbuffers.FlatBufferBuilder; import com.google.protobuf.ByteStringAccess; +import com.google.rpc.Code; import io.deephaven.UncheckedDeephavenException; import io.deephaven.barrage.flatbuf.BarrageMessageType; import io.deephaven.barrage.flatbuf.BarrageMessageWrapper; @@ -16,7 +17,6 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.Table; -import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.util.BarrageMessage; import io.deephaven.engine.updategraph.DynamicNode; @@ -27,12 +27,14 @@ import io.deephaven.extensions.barrage.util.*; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; +import io.deephaven.proto.util.Exceptions; import io.deephaven.util.annotations.FinalDefault; import io.deephaven.util.annotations.VisibleForTesting; import io.grpc.CallOptions; import io.grpc.ClientCall; import io.grpc.Context; import io.grpc.MethodDescriptor; +import io.grpc.StatusRuntimeException; import io.grpc.protobuf.ProtoUtils; import io.grpc.stub.ClientCallStreamObserver; import io.grpc.stub.ClientCalls; @@ -63,7 +65,10 @@ public class BarrageSubscriptionImpl extends ReferenceCountedLivenessNode implem private final BarrageSubscriptionOptions options; private final ClientCallStreamObserver observer; private final CheckForCompletion checkForCompletion; - private final BarrageTable resultTable; + private final BarrageUtil.ConvertedArrowSchema schema; + private final ScheduledExecutorService executorService; + private final BarrageStreamReader barrageStreamReader; + private volatile BarrageTable resultTable; private LivenessScope constructionScope; private volatile FutureAdapter future; @@ -94,17 +99,16 @@ public class BarrageSubscriptionImpl extends ReferenceCountedLivenessNode implem this.logName = tableHandle.exportId().toString(); this.tableHandle = tableHandle; this.options = options; + this.executorService = executorService; this.constructionScope = constructionScope; - final BarrageUtil.ConvertedArrowSchema schema = BarrageUtil.convertArrowSchema(tableHandle.response()); - final TableDefinition tableDefinition = schema.tableDef; + schema = BarrageUtil.convertArrowSchema(tableHandle.response()); checkForCompletion = new CheckForCompletion(); - resultTable = BarrageTable.make(executorService, tableDefinition, schema.attributes, checkForCompletion); + barrageStreamReader = new BarrageStreamReader(); final MethodDescriptor subscribeDescriptor = getClientDoExchangeDescriptor(options, schema.computeWireChunkTypes(), schema.computeWireTypes(), - schema.computeWireComponentTypes(), - new BarrageStreamReader(resultTable.getDeserializationTmConsumer())); + schema.computeWireComponentTypes(), barrageStreamReader); // We need to ensure that the DoExchange RPC does not get attached to the server RPC when this is being called // from a Deephaven server RPC thread. If we need to generalize this in the future, we may wrap this logic in a @@ -141,6 +145,15 @@ public void onNext(final BarrageMessage barrageMessage) { return; } + if (resultTable == null) { + log.error().append(BarrageSubscriptionImpl.this) + .append(": Received data before subscription was requested").endl(); + final StatusRuntimeException sre = Exceptions.statusRuntimeException( + Code.FAILED_PRECONDITION, "Received data before subscription was requested"); + GrpcUtil.safelyError(observer, sre); + checkForCompletion.onError(sre); + return; + } resultTable.handleBarrageMessage(barrageMessage); } } @@ -156,8 +169,14 @@ public void onError(final Throwable t) { .append(t).endl(); final String label = TableSpecLabeler.of(tableHandle.export().table()); - resultTable.handleBarrageError(new TableDataException( - String.format("Barrage subscription error for %s (%s)", logName, label), t)); + final TableDataException tde = new TableDataException( + String.format("Barrage subscription error for %s (%s)", logName, label), t); + if (resultTable != null) { + // this error will always be propagated to our CheckForCompletion#onError callback + resultTable.handleBarrageError(tde); + } else { + checkForCompletion.onError(tde); + } cleanup(); } @@ -168,7 +187,13 @@ public void onCompleted() { } log.error().append(BarrageSubscriptionImpl.this).append(": unexpectedly closed by other host").endl(); - resultTable.handleBarrageError(new RequestCancelledException("Barrage subscription closed by server")); + final RequestCancelledException cancelErr = + new RequestCancelledException("Barrage subscription closed by server"); + if (resultTable != null) { + resultTable.handleBarrageError(cancelErr); + } else { + checkForCompletion.onError(cancelErr); + } cleanup(); } } @@ -225,6 +250,11 @@ public Future
partialTable(RowSet viewport, BitSet columns, boolean rever columns == null ? null : (BitSet) (columns.clone()), reverseViewport); + boolean isFullSubscription = viewport == null; + resultTable = BarrageTable.make(executorService, schema.tableDef, schema.attributes, isFullSubscription, + checkForCompletion); + barrageStreamReader.setDeserializeTmConsumer(resultTable.getDeserializationTmConsumer()); + if (!isSnapshot) { resultTable.addSourceToRegistrar(); resultTable.addParentReference(this); diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index e0420105d05..9e34f0e797e 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -410,37 +410,56 @@ public void setOnGetSnapshot(Runnable onGetSnapshot, boolean isPreSnap) { * job is run we clean up deleted subscriptions and rebuild any state that is used to filter recorded updates. * */ - private class Subscription { - final BarrageSubscriptionOptions options; - final StreamObserver listener; - final String logPrefix; - - RowSet viewport; // active viewport - BitSet subscribedColumns; // active subscription columns - boolean reverseViewport; // is the active viewport reversed (indexed from end of table) - - boolean isActive = false; // is this subscription in our active list? - boolean pendingDelete = false; // is this subscription deleted as far as the client is concerned? - boolean hasPendingUpdate = false; // is this subscription in our pending list? - boolean pendingInitialSnapshot = true; // do we need to send the initial snapshot? - - RowSet pendingViewport; // if an update is pending this is our new viewport - boolean pendingReverseViewport; // is the pending viewport reversed (indexed from end of table) - BitSet pendingColumns; // if an update is pending this is our new column subscription set - - WritableRowSet snapshotViewport = null; // promoted to `active` viewport by the snapshot process - BitSet snapshotColumns = null; // promoted to `active` columns by the snapshot process - boolean snapshotReverseViewport = false; // promoted to `active` viewport direction by the snapshot process - - RowSet targetViewport = null; // the final viewport for a changed (new or updated) subscription - BitSet targetColumns; // the final set of columns for a changed subscription - boolean targetReverseViewport; // the final viewport direction for a changed subscription - - boolean isGrowingViewport; // is this subscription actively growing - WritableRowSet growingRemainingViewport = null; // rows still needed to satisfy this subscription target - // viewport - WritableRowSet growingIncrementalViewport = null; // rows to be sent to the client from the current snapshot - boolean isFirstSnapshot; // is this the first snapshot after a change to a subscriptions + private static class Subscription { + private final BarrageSubscriptionOptions options; + private final StreamObserver listener; + private final String logPrefix; + + /** active viewport **/ + private RowSet viewport; + /** active subscription columns */ + private BitSet subscribedColumns; + /** is the active viewport reversed (indexed from end of table) */ + private boolean reverseViewport; + + /** is this subscription in our active list? */ + private boolean isActive = false; + /** is this subscription deleted as far as the client is concerned? */ + private boolean pendingDelete = false; + /** is this subscription in our pending list? */ + private boolean hasPendingUpdate = false; + /** do we need to send the initial snapshot? */ + private boolean pendingInitialSnapshot = true; + + /** if an update is pending this is our new viewport */ + private RowSet pendingViewport; + /** is the pending viewport reversed (indexed from end of table) */ + private boolean pendingReverseViewport; + /** if an update is pending this is our new column subscription set */ + private BitSet pendingColumns; + + /** promoted to `active` viewport by the snapshot process */ + private WritableRowSet snapshotViewport = null; + /** promoted to `active` columns by the snapshot process */ + private BitSet snapshotColumns = null; + /** promoted to `active` viewport direction by the snapshot process */ + private boolean snapshotReverseViewport = false; + + /** the final viewport for a changed (new or updated) subscription */ + private RowSet targetViewport = null; + /** the final set of columns for a changed subscription */ + private BitSet targetColumns; + /** the final viewport direction for a changed subscription */ + private boolean targetReverseViewport; + + /** is this subscription actively growing */ + private boolean isGrowingViewport; + /** rows still needed to satisfy this subscription target viewport */ + private WritableRowSet growingRemainingViewport = null; + /** rows to be sent to the client from the current snapshot */ + private WritableRowSet growingIncrementalViewport = null; + /** is this the first snapshot after a change to a subscriptions */ + private boolean isFirstSnapshot; private Subscription(final StreamObserver listener, final BarrageSubscriptionOptions options, @@ -460,6 +479,12 @@ private Subscription(final StreamObserver li public boolean isViewport() { return viewport != null; } + + public boolean isFullSubscription() { + return !isViewport() + || (hasPendingUpdate && pendingViewport == null) + || (isGrowingViewport && targetViewport == null); + } } /** @@ -558,6 +583,12 @@ public boolean updateSubscription( if (sub.pendingViewport != null) { sub.pendingViewport.close(); } + if (sub.isFullSubscription() != (newViewport == null)) { + GrpcUtil.safelyError(listener, Code.INVALID_ARGUMENT, + "cannot change from full subscription to viewport or vice versa"); + removeSubscription(listener); + return; + } sub.pendingViewport = newViewport != null ? newViewport.copy() : null; sub.pendingReverseViewport = newReverseViewport; if (isBlinkTable && newReverseViewport) { @@ -1187,7 +1218,9 @@ private void updateSubscriptionsSnapshotAndPropagate() { BarrageMessage preSnapshot = null; BarrageMessage blinkTableFlushPreSnapshot = null; + RowSet preSnapRowSetPrev = null; RowSet preSnapRowSet = null; + RowSet postSnapRowSetPrev = null; BarrageMessage snapshot = null; BarrageMessage postSnapshot = null; @@ -1394,6 +1427,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { if (!firstSubscription && deltaSplitIdx > 0) { final long startTm = System.nanoTime(); + preSnapRowSetPrev = propagationRowSet.copy(); preSnapshot = aggregateUpdatesInRange(0, deltaSplitIdx); recordMetric(stats -> stats.aggregate, System.nanoTime() - startTm); preSnapRowSet = propagationRowSet.copy(); @@ -1419,6 +1453,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { if (deltaSplitIdx < pendingDeltas.size()) { final long startTm = System.nanoTime(); + postSnapRowSetPrev = propagationRowSet.copy(); postSnapshot = aggregateUpdatesInRange(deltaSplitIdx, pendingDeltas.size()); recordMetric(stats -> stats.aggregate, System.nanoTime() - startTm); } @@ -1442,8 +1477,9 @@ private void updateSubscriptionsSnapshotAndPropagate() { // now, propagate updates if (preSnapshot != null) { final long startTm = System.nanoTime(); - propagateToSubscribers(preSnapshot, preSnapRowSet); + propagateToSubscribers(preSnapshot, preSnapRowSetPrev, preSnapRowSet); recordMetric(stats -> stats.propagate, System.nanoTime() - startTm); + preSnapRowSetPrev.close(); preSnapRowSet.close(); } @@ -1451,7 +1487,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { final long startTm = System.nanoTime(); try (final RowSet fakeTableRowSet = RowSetFactory.empty()) { // the method expects the post-update RowSet; which is empty after the flush - propagateToSubscribers(blinkTableFlushPreSnapshot, fakeTableRowSet); + propagateToSubscribers(blinkTableFlushPreSnapshot, fakeTableRowSet, fakeTableRowSet); } recordMetric(stats -> stats.propagate, System.nanoTime() - startTm); } @@ -1477,8 +1513,9 @@ private void updateSubscriptionsSnapshotAndPropagate() { if (postSnapshot != null) { final long startTm = System.nanoTime(); - propagateToSubscribers(postSnapshot, propagationRowSet); + propagateToSubscribers(postSnapshot, postSnapRowSetPrev, propagationRowSet); recordMetric(stats -> stats.propagate, System.nanoTime() - startTm); + postSnapRowSetPrev.close(); } if (deletedSubscriptions != null) { @@ -1513,8 +1550,11 @@ private void updateSubscriptionsSnapshotAndPropagate() { } } - private void propagateToSubscribers(final BarrageMessage message, final RowSet propRowSetForMessage) { - // message is released via transfer to stream generator (as it must live until all view's are closed) + private void propagateToSubscribers( + final BarrageMessage message, + final RowSet propRowSetForMessagePrev, + final RowSet propRowSetForMessage) { + // message is released via transfer to stream generator (as it must live until all views are closed) try (final BarrageStreamGenerator generator = streamGeneratorFactory.newGenerator( message, this::recordWriteMetrics)) { for (final Subscription subscription : activeSubscriptions) { @@ -1537,10 +1577,13 @@ private void propagateToSubscribers(final BarrageMessage message, final RowSet p final boolean isReversed = isPreSnapshot ? subscription.snapshotReverseViewport : subscription.reverseViewport; - try (final RowSet clientView = - vp != null ? propRowSetForMessage.subSetForPositions(vp, isReversed) : null) { + try (final RowSet clientViewPrev = + vp != null ? propRowSetForMessagePrev.subSetForPositions(vp, isReversed) : null; + final RowSet clientView = + vp != null ? propRowSetForMessage.subSetForPositions(vp, isReversed) : null) { subscription.listener.onNext(generator.getSubView( - subscription.options, false, vp, subscription.reverseViewport, clientView, cols)); + subscription.options, false, subscription.isFullSubscription(), vp, + subscription.reverseViewport, clientViewPrev, clientView, cols)); } catch (final Exception e) { try { subscription.listener.onError(errorTransformer.transform(e)); @@ -1607,8 +1650,8 @@ private void propagateSnapshotForSubscription(final Subscription subscription, // some messages may be empty of rows, but we need to update the client viewport and column set subscription.listener .onNext(snapshotGenerator.getSubView(subscription.options, subscription.pendingInitialSnapshot, - subscription.viewport, subscription.reverseViewport, keySpaceViewport, - subscription.subscribedColumns)); + subscription.isFullSubscription(), subscription.viewport, subscription.reverseViewport, + keySpaceViewport.copy(), keySpaceViewport, subscription.subscribedColumns)); } catch (final Exception e) { GrpcUtil.safelyError(subscription.listener, errorTransformer.transform(e)); diff --git a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java index a0e18a46ad6..110cbad5d25 100644 --- a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java +++ b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java @@ -355,13 +355,15 @@ private static long buildAndSendSnapshot( barrageMessage.modColumnData = BarrageMessage.ZERO_MOD_COLUMNS; // 5. Send the BarrageMessage - final BarrageStreamGenerator streamGenerator = - streamGeneratorFactory.newGenerator(barrageMessage, writeMetricsConsumer); - // Note that we're always specifying "isInitialSnapshot=true". This is to provoke the subscription view to - // send the added rows on every snapshot, since (1) our added rows are flat, and thus cheap to send, and - // (2) we're relying on added rows to signal the full expanded size to the client. - GrpcUtil.safelyOnNext(listener, - streamGenerator.getSubView(subscriptionOptions, true, rows, false, rows, columns)); + try (final BarrageStreamGenerator streamGenerator = + streamGeneratorFactory.newGenerator(barrageMessage, writeMetricsConsumer)) { + // Note that we're always specifying "isInitialSnapshot=true". This is to provoke the subscription view to + // send the added rows on every snapshot, since (1) our added rows are flat, and thus cheap to send, and + // (2) we're relying on added rows to signal the full expanded size to the client. + GrpcUtil.safelyOnNext(listener, + streamGenerator.getSubView(subscriptionOptions, true, true, rows, false, rows.copy(), rows.copy(), + columns)); + } // 6. Let the caller know what the expanded size was return expandedSize; diff --git a/server/src/test/java/io/deephaven/server/barrage/BarrageBlinkTableTest.java b/server/src/test/java/io/deephaven/server/barrage/BarrageBlinkTableTest.java index dcb7445077e..703c6f6cc81 100644 --- a/server/src/test/java/io/deephaven/server/barrage/BarrageBlinkTableTest.java +++ b/server/src/test/java/io/deephaven/server/barrage/BarrageBlinkTableTest.java @@ -182,7 +182,7 @@ private class RemoteClient { final Schema flatbufSchema = SchemaHelper.flatbufSchema(schemaBytes.asReadOnlyByteBuffer()); final BarrageUtil.ConvertedArrowSchema schema = BarrageUtil.convertArrowSchema(flatbufSchema); this.barrageTable = BarrageTable.make(updateSourceCombiner, ExecutionContext.getContext().getUpdateGraph(), - null, schema.tableDef, schema.attributes, null); + null, schema.tableDef, schema.attributes, viewport == null, null); this.barrageTable.addSourceToRegistrar(); final BarrageSubscriptionOptions options = BarrageSubscriptionOptions.builder() diff --git a/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java b/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java index 77eab59b012..58cbaaea078 100644 --- a/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java +++ b/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java @@ -183,7 +183,7 @@ private class RemoteClient { } this.barrageTable = BarrageTable.make(updateSourceCombiner, ExecutionContext.getContext().getUpdateGraph(), - null, barrageMessageProducer.getTableDefinition(), attributes, null); + null, barrageMessageProducer.getTableDefinition(), attributes, viewport == null, null); this.barrageTable.addSourceToRegistrar(); final BarrageSubscriptionOptions options = BarrageSubscriptionOptions.builder() From 0089d629fa6278b0cb423c83b6f3bc9a362487c1 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 8 Nov 2024 17:10:05 -0700 Subject: [PATCH 11/35] Ryan's Synchronous Review --- ...io.deephaven.repository-conventions.gradle | 6 - .../barrage/BarrageSnapshotOptions.java | 48 ++++-- .../barrage/BarrageStreamGenerator.java | 2 +- .../barrage/BarrageStreamGeneratorImpl.java | 156 +++++++----------- .../barrage/BarrageSubscriptionOptions.java | 65 ++++++-- .../chunk/DefaultChunkReadingFactory.java | 2 +- .../barrage/table/BarrageBlinkTable.java | 2 +- .../barrage/table/BarrageRedirectedTable.java | 28 +++- .../barrage/table/BarrageTable.java | 9 - .../barrage/util/BarrageStreamReader.java | 2 +- .../barrage/util/StreamReaderOptions.java | 16 +- gradle/libs.versions.toml | 2 +- .../client/impl/BarrageSnapshotImpl.java | 31 +++- .../client/impl/BarrageSubscriptionImpl.java | 45 ++--- .../barrage/BarrageMessageProducer.java | 12 +- .../HierarchicalTableViewSubscription.java | 3 +- .../barrage/BarrageMessageRoundTripTest.java | 33 ++-- 17 files changed, 251 insertions(+), 211 deletions(-) diff --git a/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle b/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle index 7abd48a3ea6..ea0c0abeb8d 100644 --- a/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle +++ b/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle @@ -1,12 +1,6 @@ repositories { mavenCentral() mavenLocal() - maven { - url 'https://oss.sonatype.org/content/repositories/snapshots' - content { - includeGroup 'com.vertispan.flatbuffers' - } - } maven { url 'https://jitpack.io' content { diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java index 98dc864b8db..005601ba355 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java @@ -34,29 +34,18 @@ public static BarrageSnapshotOptions of(final BarrageSnapshotRequest snapshotReq return of(snapshotRequest.snapshotOptions()); } - /** - * By default, prefer to communicate null values using the arrow-compatible validity structure. - * - * @return whether to use deephaven nulls - */ @Override @Default public boolean useDeephavenNulls() { return false; } - /** - * @return the preferred batch size if specified - */ @Override @Default public int batchSize() { return 0; } - /** - * @return the maximum GRPC message size if specified - */ @Override @Default public int maxMessageSize() { @@ -70,8 +59,8 @@ public int previewListLengthLimit() { } public int appendTo(FlatBufferBuilder builder) { - return io.deephaven.barrage.flatbuf.BarrageSnapshotOptions.createBarrageSnapshotOptions( - builder, useDeephavenNulls(), + return io.deephaven.barrage.flatbuf.BarrageSnapshotOptions.createBarrageSnapshotOptions(builder, + useDeephavenNulls(), batchSize(), maxMessageSize(), previewListLengthLimit()); @@ -79,10 +68,20 @@ builder, useDeephavenNulls(), public interface Builder { + /** + * See {@link StreamReaderOptions#useDeephavenNulls()} for details. + * + * @param useDeephavenNulls whether to use deephaven nulls + * @return this builder + */ Builder useDeephavenNulls(boolean useDeephavenNulls); /** - * Deprecated since 0.37.0 and is marked for removal. (our GWT artifacts do not yet support the attributes) + * The default conversion mode is to Stringify all objects that do not have a registered encoding. Column + * conversion modes are no longer supported. + * + * @deprecated Since 0.37.0 and is marked for removal. (Note, GWT does not support encoding this context via + * annotation values.) */ @FinalDefault @Deprecated @@ -90,12 +89,33 @@ default Builder columnConversionMode(ColumnConversionMode columnConversionMode) return this; } + /** + * See {@link StreamReaderOptions#batchSize()} for details. + * + * @param batchSize the ideal number of records to send per record batch + * @return this builder + */ Builder batchSize(int batchSize); + /** + * See {@link StreamReaderOptions#maxMessageSize()} for details. + * + * @param messageSize the maximum size of a GRPC message in bytes + * @return this builder + */ Builder maxMessageSize(int messageSize); + /** + * See {@link StreamReaderOptions#previewListLengthLimit()} for details. + * + * @param previewListLengthLimit the magnitude of the number of elements to include in a preview list + * @return this builder + */ Builder previewListLengthLimit(int previewListLengthLimit); + /** + * @return a new BarrageSnapshotOptions instance + */ BarrageSnapshotOptions build(); } } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java index 730feaee99c..a571c7ba183 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java @@ -72,7 +72,7 @@ BarrageStreamGenerator newGenerator( * @param isFullSubscription whether this is a full subscription (possibly a growing viewport) * @param viewport is the position-space viewport * @param reverseViewport is the viewport reversed (relative to end of table instead of beginning) - * @param keyspaceViewportPrev is the key-space viewport in prior to applying the update + * @param keyspaceViewportPrev is the key-space viewport prior to applying the update * @param keyspaceViewport is the key-space viewport * @param subscribedColumns are the columns subscribed for this view * @return a MessageView filtered by the subscription properties that can be sent to that subscriber diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index d02556049e8..620e08b9e7d 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -215,19 +215,6 @@ public void close() { } } - /** - * Obtain a View of this StreamGenerator that can be sent to a single subscriber. - * - * @param options serialization options for this specific view - * @param isInitialSnapshot indicates whether this is the first snapshot for the listener - * @param isFullSubscription whether this is a full subscription (possibly a growing viewport) - * @param viewport is the position-space viewport - * @param reverseViewport is the viewport reversed (relative to end of table instead of beginning) - * @param keyspaceViewportPrev is the key-space viewport in prev key-space - * @param keyspaceViewport is the key-space viewport - * @param subscribedColumns are the columns subscribed for this view - * @return a MessageView filtered by the subscription properties that can be sent to that subscriber - */ @Override public MessageView getSubView( final BarrageSubscriptionOptions options, @@ -242,13 +229,6 @@ public MessageView getSubView( keyspaceViewportPrev, keyspaceViewport, subscribedColumns); } - /** - * Obtain a Full-Subscription View of this StreamGenerator that can be sent to a single subscriber. - * - * @param options serialization options for this specific view - * @param isInitialSnapshot indicates whether this is the first snapshot for the listener - * @return a MessageView filtered by the subscription properties that can be sent to that subscriber - */ @Override public MessageView getSubView(BarrageSubscriptionOptions options, boolean isInitialSnapshot) { return getSubView(options, isInitialSnapshot, true, null, false, null, null, null); @@ -263,12 +243,12 @@ private final class SubView implements RecordBatchMessageView { private final RowSet keyspaceViewportPrev; private final RowSet keyspaceViewport; private final BitSet subscribedColumns; - private final long numAddRows; - private final long numModRows; - private final RowSet addRowKeys; - private final RowSet addRowOffsets; - private final RowSet[] modRowKeys; - private final RowSet[] modRowOffsets; + private final long numClientAddRows; + private final long numClientModRows; + private final RowSet clientAddedRows; + private final RowSet clientAddedRowOffsets; + private final RowSet[] clientModdedRows; + private final RowSet[] clientModdedRowOffsets; public SubView(final BarrageSubscriptionOptions options, final boolean isInitialSnapshot, @@ -288,11 +268,11 @@ public SubView(final BarrageSubscriptionOptions options, this.subscribedColumns = subscribedColumns; if (keyspaceViewport != null) { - this.modRowKeys = new WritableRowSet[modColumnData.length]; - this.modRowOffsets = new WritableRowSet[modColumnData.length]; + this.clientModdedRows = new WritableRowSet[modColumnData.length]; + this.clientModdedRowOffsets = new WritableRowSet[modColumnData.length]; } else { - this.modRowKeys = null; - this.modRowOffsets = null; + this.clientModdedRows = null; + this.clientModdedRowOffsets = null; } // precompute the modified column indexes, and calculate total rows needed @@ -303,38 +283,38 @@ public SubView(final BarrageSubscriptionOptions options, if (keyspaceViewport != null) { try (WritableRowSet intersect = keyspaceViewport.intersect(mcd.rowsModified.original)) { if (isFullSubscription) { - modRowKeys[ii] = intersect.copy(); + clientModdedRows[ii] = intersect.copy(); } else { - modRowKeys[ii] = keyspaceViewport.invert(intersect); + clientModdedRows[ii] = keyspaceViewport.invert(intersect); } - modRowOffsets[ii] = mcd.rowsModified.original.invert(intersect); + clientModdedRowOffsets[ii] = mcd.rowsModified.original.invert(intersect); numModRows = Math.max(numModRows, intersect.size()); } } else { numModRows = Math.max(numModRows, mcd.rowsModified.original.size()); } } - this.numModRows = numModRows; + this.numClientModRows = numModRows; if (keyspaceViewport != null) { try (WritableRowSet intersect = keyspaceViewport.intersect(rowsIncluded.original)) { if (isFullSubscription) { - addRowKeys = intersect.copy(); + clientAddedRows = intersect.copy(); } else { - addRowKeys = keyspaceViewport.invert(intersect); + clientAddedRows = keyspaceViewport.invert(intersect); } - addRowOffsets = rowsIncluded.original.invert(intersect); + clientAddedRowOffsets = rowsIncluded.original.invert(intersect); } } else if (!rowsAdded.original.equals(rowsIncluded.original)) { // there are scoped rows included in the chunks that need to be removed - addRowKeys = rowsAdded.original.copy(); - addRowOffsets = rowsIncluded.original.invert(addRowKeys); + clientAddedRows = rowsAdded.original.copy(); + clientAddedRowOffsets = rowsIncluded.original.invert(clientAddedRows); } else { - addRowKeys = rowsAdded.original.copy(); - addRowOffsets = RowSetFactory.flat(rowsAdded.original.size()); + clientAddedRows = rowsAdded.original.copy(); + clientAddedRowOffsets = RowSetFactory.flat(rowsAdded.original.size()); } - this.numAddRows = addRowOffsets.size(); + this.numClientAddRows = clientAddedRowOffsets.size(); } @Override @@ -348,7 +328,7 @@ public void forEachStream(Consumer visitor) throws IOExcepti final MutableInt actualBatchSize = new MutableInt(); - if (numAddRows == 0 && numModRows == 0) { + if (numClientAddRows == 0 && numClientModRows == 0) { // we still need to send a message containing metadata when there are no rows final DefensiveDrainable is = getInputStream(this, 0, 0, actualBatchSize, metadata, BarrageStreamGeneratorImpl.this::appendAddColumns); @@ -360,22 +340,21 @@ public void forEachStream(Consumer visitor) throws IOExcepti // send the add batches (if any) try { - processBatches(visitor, this, numAddRows, maxBatchSize, metadata, + processBatches(visitor, this, numClientAddRows, maxBatchSize, metadata, BarrageStreamGeneratorImpl.this::appendAddColumns, bytesWritten); // send the mod batches (if any) but don't send metadata twice - processBatches(visitor, this, numModRows, maxBatchSize, numAddRows > 0 ? null : metadata, + processBatches(visitor, this, numClientModRows, maxBatchSize, numClientAddRows > 0 ? null : metadata, BarrageStreamGeneratorImpl.this::appendModColumns, bytesWritten); } finally { - // clean up the helper indexes - addRowOffsets.close(); - addRowKeys.close(); - if (modRowOffsets != null) { - SafeCloseable.closeAll(modRowKeys); - SafeCloseable.closeAll(modRowOffsets); + clientAddedRowOffsets.close(); + clientAddedRows.close(); + if (clientModdedRowOffsets != null) { + SafeCloseable.closeAll(clientModdedRows); + SafeCloseable.closeAll(clientModdedRowOffsets); } - writeConsumer.onWrite(bytesWritten.get(), System.nanoTime() - startTm); } + writeConsumer.onWrite(bytesWritten.get(), System.nanoTime() - startTm); } private int batchSize() { @@ -398,15 +377,15 @@ public StreamReaderOptions options() { @Override public RowSet addRowOffsets() { - return addRowOffsets; + return clientAddedRowOffsets; } @Override public RowSet modRowOffsets(int col) { - if (modRowOffsets == null) { + if (clientModdedRowOffsets == null) { return null; } - return modRowOffsets[col]; + return clientModdedRowOffsets[col]; } private ByteBuffer getSubscriptionMetadata() throws IOException { @@ -426,11 +405,11 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { final int rowsAddedOffset; if (!isFullSubscription) { - try (final RowSetGenerator addRowsGen = new RowSetGenerator(addRowKeys)) { - rowsAddedOffset = addRowsGen.addToFlatBuffer(metadata); + try (final RowSetGenerator clientAddedRowsGen = new RowSetGenerator(clientAddedRows)) { + rowsAddedOffset = clientAddedRowsGen.addToFlatBuffer(metadata); } } else if (isSnapshot && !isInitialSnapshot) { - // full subscription clients don't need/want to receive the full RowSet on every snapshot + // Growing viewport clients don't need/want to receive the full RowSet on every snapshot rowsAddedOffset = EmptyRowSetGenerator.INSTANCE.addToFlatBuffer(metadata); } else { rowsAddedOffset = rowsAdded.addToFlatBuffer(metadata); @@ -438,10 +417,11 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { final int rowsRemovedOffset; if (isViewport() && !isFullSubscription) { - try (final WritableRowSet removedInViewport = keyspaceViewportPrev.intersect(rowsRemoved.original); - final WritableRowSet removedInPositionSpace = keyspaceViewportPrev.invert(removedInViewport); - final RowSetGenerator rmRowsGen = new RowSetGenerator(removedInPositionSpace)) { - rowsRemovedOffset = rmRowsGen.addToFlatBuffer(metadata); + try (final WritableRowSet clientRemovedKeySpace = keyspaceViewportPrev.intersect(rowsRemoved.original); + final WritableRowSet clientRemovedPositionSpace = + keyspaceViewportPrev.invert(clientRemovedKeySpace); + final RowSetGenerator clientRemovedRowsGen = new RowSetGenerator(clientRemovedPositionSpace)) { + rowsRemovedOffset = clientRemovedRowsGen.addToFlatBuffer(metadata); } } else { rowsRemovedOffset = rowsRemoved.addToFlatBuffer(metadata); @@ -458,7 +438,7 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { int addedRowsIncludedOffset = 0; // don't send `rowsIncluded` when identical to `rowsAdded`, client will infer they are the same - if (isSnapshot || !addRowKeys.equals(rowsAdded.original)) { + if (isSnapshot || !clientAddedRows.equals(rowsAdded.original)) { if (isViewport() && !isFullSubscription) { try (final WritableRowSet inclInViewport = keyspaceViewport.intersect(rowsIncluded.original); final WritableRowSet inclInPositionSpace = keyspaceViewport.invert(inclInViewport); @@ -466,7 +446,7 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { addedRowsIncludedOffset = inclRowsGen.addToFlatBuffer(metadata); } } else { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(addRowKeys, metadata); + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(clientAddedRows, metadata); } } @@ -475,7 +455,7 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { for (int ii = 0; ii < modColumnData.length; ++ii) { final int myModRowOffset; if (keyspaceViewport != null) { - try (final RowSetGenerator modRowsGen = new RowSetGenerator(modRowKeys[ii])) { + try (final RowSetGenerator modRowsGen = new RowSetGenerator(clientModdedRows[ii])) { myModRowOffset = modRowsGen.addToFlatBuffer(metadata); } } else { @@ -517,16 +497,6 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { } } - /** - * Obtain a View of this StreamGenerator that can be sent to a single snapshot requestor. - * - * @param options serialization options for this specific view - * @param viewport is the position-space viewport - * @param reverseViewport is the viewport reversed (relative to end of table instead of beginning) - * @param keyspaceViewport is the key-space viewport - * @param snapshotColumns are the columns subscribed for this view - * @return a MessageView filtered by the snapshot properties that can be sent to that subscriber - */ @Override public MessageView getSnapshotView(final BarrageSnapshotOptions options, @Nullable final RowSet viewport, @@ -536,12 +506,6 @@ public MessageView getSnapshotView(final BarrageSnapshotOptions options, return new SnapshotView(options, viewport, reverseViewport, keyspaceViewport, snapshotColumns); } - /** - * Obtain a Full-Snapshot View of this StreamGenerator that can be sent to a single snapshot requestor. - * - * @param options serialization options for this specific view - * @return a MessageView filtered by the snapshot properties that can be sent to that subscriber - */ @Override public MessageView getSnapshotView(BarrageSnapshotOptions options) { return getSnapshotView(options, null, false, null, null); @@ -552,9 +516,9 @@ private final class SnapshotView implements RecordBatchMessageView { private final RowSet viewport; private final boolean reverseViewport; private final BitSet subscribedColumns; - private final long numAddRows; - private final RowSet addRowKeys; - private final RowSet addRowOffsets; + private final long numClientAddRows; + private final RowSet clientAddedRows; + private final RowSet clientAddedRowOffsets; public SnapshotView(final BarrageSnapshotOptions options, @Nullable final RowSet viewport, @@ -569,14 +533,14 @@ public SnapshotView(final BarrageSnapshotOptions options, // precompute add row offsets if (keyspaceViewport != null) { - addRowKeys = keyspaceViewport.intersect(rowsIncluded.original); - addRowOffsets = rowsIncluded.original.invert(addRowKeys); + clientAddedRows = keyspaceViewport.intersect(rowsIncluded.original); + clientAddedRowOffsets = rowsIncluded.original.invert(clientAddedRows); } else { - addRowKeys = rowsAdded.original.copy(); - addRowOffsets = RowSetFactory.flat(addRowKeys.size()); + clientAddedRows = rowsAdded.original.copy(); + clientAddedRowOffsets = RowSetFactory.flat(clientAddedRows.size()); } - numAddRows = addRowOffsets.size(); + numClientAddRows = clientAddedRowOffsets.size(); } @Override @@ -588,17 +552,17 @@ public void forEachStream(Consumer visitor) throws IOExcepti // batch size is maximum, will write fewer rows when needed int maxBatchSize = batchSize(); final MutableInt actualBatchSize = new MutableInt(); - if (numAddRows == 0) { + if (numClientAddRows == 0) { // we still need to send a message containing metadata when there are no rows visitor.accept(getInputStream(this, 0, 0, actualBatchSize, metadata, BarrageStreamGeneratorImpl.this::appendAddColumns)); } else { // send the add batches - processBatches(visitor, this, numAddRows, maxBatchSize, metadata, + processBatches(visitor, this, numClientAddRows, maxBatchSize, metadata, BarrageStreamGeneratorImpl.this::appendAddColumns, bytesWritten); } - addRowOffsets.close(); - addRowKeys.close(); + clientAddedRowOffsets.close(); + clientAddedRows.close(); writeConsumer.onWrite(bytesWritten.get(), System.nanoTime() - startTm); } @@ -622,7 +586,7 @@ public StreamReaderOptions options() { @Override public RowSet addRowOffsets() { - return addRowOffsets; + return clientAddedRowOffsets; } @Override @@ -653,8 +617,8 @@ private ByteBuffer getSnapshotMetadata() throws IOException { // Added Chunk Data: int addedRowsIncludedOffset = 0; // don't send `rowsIncluded` when identical to `rowsAdded`, client will infer they are the same - if (isSnapshot || !addRowKeys.equals(rowsAdded.original)) { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(addRowKeys, metadata); + if (isSnapshot || !clientAddedRows.equals(rowsAdded.original)) { + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(clientAddedRows, metadata); } BarrageUpdateMetadata.startBarrageUpdateMetadata(metadata); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java index f8bd47deded..8c45da1e10b 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java @@ -37,21 +37,12 @@ public static BarrageSubscriptionOptions of(final BarrageSubscriptionRequest sub return of(subscriptionRequest.subscriptionOptions()); } - /** - * By default, prefer to communicate null values using the arrow-compatible validity structure. - * - * @return whether to use deephaven nulls - */ @Override @Default public boolean useDeephavenNulls() { return false; } - /** - * Requesting clients can specify whether they want columns to be returned wrapped in a list. This enables easier - * support in some official arrow clients, but is not the default. - */ @Override @Default public boolean columnsAsList() { @@ -81,18 +72,12 @@ public int minUpdateIntervalMs() { return 0; } - /** - * @return the preferred batch size if specified - */ @Override @Default public int batchSize() { return 0; } - /** - * @return the preferred maximum GRPC message size if specified - */ @Override @Default public int maxMessageSize() { @@ -106,8 +91,8 @@ public int previewListLengthLimit() { } public int appendTo(FlatBufferBuilder builder) { - return io.deephaven.barrage.flatbuf.BarrageSubscriptionOptions.createBarrageSubscriptionOptions( - builder, useDeephavenNulls(), + return io.deephaven.barrage.flatbuf.BarrageSubscriptionOptions.createBarrageSubscriptionOptions(builder, + useDeephavenNulls(), minUpdateIntervalMs(), batchSize(), maxMessageSize(), @@ -117,12 +102,29 @@ builder, useDeephavenNulls(), public interface Builder { + /** + * See {@link StreamReaderOptions#useDeephavenNulls()} for details. + * + * @param useDeephavenNulls whether to use deephaven nulls + * @return this builder + */ Builder useDeephavenNulls(boolean useDeephavenNulls); + /** + * See {@link StreamReaderOptions#columnsAsList() } for details. + * + * @param columnsAsList whether to wrap columns in a list to be compatible with native Flight clients + * @return this builder + */ Builder columnsAsList(boolean columnsAsList); /** - * Deprecated since 0.37.0 and is marked for removal. (our GWT artifacts do not yet support the attributes) + * The default conversion mode is to Stringify all objects that do not have a registered encoding. Column + * conversion modes are no longer supported. + * + * @deprecated Since 0.37.0 and is marked for removal. (Note, GWT does not support encoding this context via + * annotation values.) + * @return this builder */ @FinalDefault @Deprecated @@ -130,14 +132,41 @@ default Builder columnConversionMode(ColumnConversionMode columnConversionMode) return this; } + /** + * See {@link BarrageSubscriptionOptions#minUpdateIntervalMs()} for details. + * + * @param minUpdateIntervalMs the update interval used to limit barrage message frequency + * @return this builder + */ Builder minUpdateIntervalMs(int minUpdateIntervalMs); + /** + * See {@link StreamReaderOptions#batchSize()} for details. + * + * @param batchSize the ideal number of records to send per record batch + * @return this builder + */ Builder batchSize(int batchSize); + /** + * See {@link StreamReaderOptions#maxMessageSize()} for details. + * + * @param messageSize the maximum size of a GRPC message in bytes + * @return this builder + */ Builder maxMessageSize(int messageSize); + /** + * See {@link StreamReaderOptions#previewListLengthLimit()} for details. + * + * @param previewListLengthLimit the magnitude of the number of elements to include in a preview list + * @return this builder + */ Builder previewListLengthLimit(int previewListLengthLimit); + /** + * @return a new BarrageSubscriptionOptions instance + */ BarrageSubscriptionOptions build(); } } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java index d8945ded93d..ddecdb09ca8 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java @@ -174,7 +174,7 @@ public ChunkReader getReader(StreamReaderOptions options, int factor, // Migrate Schema to custom format when available. return SchemaChunkReader.SCHEMA_CHUNK_READER; } - // Otherwise fall through to default of writing via toString. + // All other object types are sent from the server as strings return StringChunkReader.STRING_CHUNK_READER; default: throw new UnsupportedOperationException(); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageBlinkTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageBlinkTable.java index c4ecc6004dd..2c9896af2d9 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageBlinkTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageBlinkTable.java @@ -47,7 +47,7 @@ protected BarrageBlinkTable( final WritableColumnSource[] writableSources, final Map attributes, @Nullable final ViewportChangedCallback vpCallback) { - super(registrar, notificationQueue, executorService, columns, writableSources, attributes, true, vpCallback); + super(registrar, notificationQueue, executorService, columns, writableSources, attributes, vpCallback); setFlat(); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java index a4c3906959a..31a4bf2fba5 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java @@ -48,6 +48,14 @@ public class BarrageRedirectedTable extends BarrageTable { /** represents which rows in writable source exist but are not mapped to any parent rows */ private WritableRowSet freeset = RowSetFactory.empty(); + /** + * A full subscription is where the server sends all data to the client. The server is allowed to initially send + * growing viewports to the client to avoid contention on the update graph lock. Once the server has grown the + * viewport to match the entire table as of any particular consistent state, it will not send any more snapshots and + * {@code serverViewport} will be set to {@code null}. + */ + protected final boolean isFullSubscription; + protected BarrageRedirectedTable(final UpdateSourceRegistrar registrar, final NotificationQueue notificationQueue, @Nullable final ScheduledExecutorService executorService, @@ -58,9 +66,9 @@ protected BarrageRedirectedTable(final UpdateSourceRegistrar registrar, final boolean isFlat, final boolean isFullSubscription, @Nullable final ViewportChangedCallback vpCallback) { - super(registrar, notificationQueue, executorService, columns, writableSources, attributes, isFullSubscription, - vpCallback); + super(registrar, notificationQueue, executorService, columns, writableSources, attributes, vpCallback); this.rowRedirection = rowRedirection; + this.isFullSubscription = isFullSubscription; if (!isFullSubscription || isFlat) { setFlat(); } @@ -131,7 +139,12 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC // shifts if (updateShiftData.nonempty()) { - rowRedirection.applyShift(currentRowSet, updateShiftData); + try (final WritableRowSet postRemoveRowSet = isFullSubscription + ? null + : currentRowSet.minus(update.rowsRemoved)) { + rowRedirection.applyShift( + postRemoveRowSet != null ? postRemoveRowSet : currentRowSet, updateShiftData); + } if (isFullSubscription) { updateShiftData.apply(currentRowSet); } @@ -142,7 +155,14 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC if (isFullSubscription) { currentRowSet.insert(update.rowsAdded); } else { - final long newSize = prevSize - update.rowsRemoved.size() + update.rowsAdded.size(); + final long newSize; + if (update.isSnapshot) { + newSize = update.rowsAdded.size(); + } else { + // note that we are not told about rows that fall off the end of our respected viewport + newSize = Math.min(serverViewport.size(), + prevSize - update.rowsRemoved.size() + update.rowsIncluded.size()); + } if (newSize < prevSize) { currentRowSet.removeRange(newSize, prevSize - 1); } else if (newSize > prevSize) { diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index 6248d63a398..a9c2c093744 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -110,13 +110,6 @@ public interface ViewportChangedCallback { private BitSet serverColumns; private boolean serverReverseViewport; - /** - * A full subscription is where the server sends all data to the client. The server is allowed to initially send - * growing viewports to the client to avoid contention on the update graph lock. Once the server has sent a full - * subscription, it will not send any more snapshots and serverViewport will be set to null. - */ - protected final boolean isFullSubscription; - /** * A batch of updates may change the viewport more than once, but we cannot deliver until the updates have been * propagated to this BarrageTable and its last notification step has been updated. @@ -158,7 +151,6 @@ protected BarrageTable(final UpdateSourceRegistrar registrar, final LinkedHashMap> columns, final WritableColumnSource[] writableSources, final Map attributes, - final boolean isFullSubscription, @Nullable final ViewportChangedCallback viewportChangedCallback) { super(RowSetFactory.empty().toTracking(), columns); attributes.entrySet().stream() @@ -168,7 +160,6 @@ protected BarrageTable(final UpdateSourceRegistrar registrar, this.registrar = registrar; this.notificationQueue = notificationQueue; this.executorService = executorService; - this.isFullSubscription = isFullSubscription; final String tableKey = BarragePerformanceLog.getKeyFor(this); if (executorService == null || tableKey == null) { diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index 0f206a7d8e4..f78ce861495 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -135,7 +135,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.lastSeq = metadata.lastSeq(); msg.rowsAdded = extractIndex(metadata.addedRowsAsByteBuffer()); msg.rowsRemoved = extractIndex(metadata.removedRowsAsByteBuffer()); - ByteBuffer shiftData = metadata.shiftDataAsByteBuffer(); + final ByteBuffer shiftData = metadata.shiftDataAsByteBuffer(); msg.shifted = shiftData != null ? extractIndexShiftData(shiftData) : RowSetShiftData.EMPTY; final ByteBuffer rowsIncluded = metadata.addedRowsIncludedAsByteBuffer(); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java index 32a8640b36b..45f6bf49ed3 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java @@ -14,8 +14,8 @@ public interface StreamReaderOptions { boolean useDeephavenNulls(); /** - * Deprecated since 0.37.0 and is marked for removal. (our GWT artifacts do not yet support the attributes) - * + * @deprecated Since 0.37.0 and is marked for removal. (Note, GWT does not support encoding this context via + * annotation values.) * @return the conversion mode to use for object columns */ @FinalDefault @@ -48,14 +48,14 @@ default boolean columnsAsList() { * The maximum length of any list / array to encode. *
    *
  • If zero, list lengths will not be limited.
    • - *
  • If non-zero, the server will limit the length of any encoded list / array to n elements, where n is the - * absolute value of the specified value.
    • - *
  • If the column value has length less than zero, the server will encode the last n elements of the list / - * array.
    • + *
  • If non-zero, the server will limit the length of any encoded list / array to the absolute value of the + * returned length.
    • + *
  • If less than zero, the server will encode elements from the end of the list / array, rather than rom the + * beginning.
    • *
*

- * Note that the server will append an arbitrary value to indicate truncation; this value may not be the actual last - * value in the list / array. + * Note: The server is unable to indicate when truncation occurs. To detect truncation request one more element than + * the maximum number you wish to display. * * @return the maximum length of any list / array to encode; zero means no limit; negative values indicate to treat * the limit as a tail instead of a head diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index eaafb5cdfb1..673d23e8f5a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -77,7 +77,7 @@ trove = "3.0.3" undercouch = "2.15.1" univocity = "2.9.1" vertispan-nio = "1.0-alpha-2" -vertispan-flatbuffers-gwt = "24.3.25-1-SNAPSHOT" +vertispan-flatbuffers-gwt = "24.3.25-1" vertispan-ts-defs = "1.0.0-RC4" xerial = "3.47.0.0" diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java index 133c72fa06a..4968921a2c2 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java @@ -149,7 +149,8 @@ public void onNext(final BarrageMessage barrageMessage) { rowsReceived += resultSize; - if (resultTable == null) { + final BarrageTable localResultTable = resultTable; + if (localResultTable == null) { log.error().append(BarrageSnapshotImpl.this) .append(": Received data before snapshot was requested").endl(); final StatusRuntimeException sre = Exceptions.statusRuntimeException( @@ -158,7 +159,7 @@ public void onNext(final BarrageMessage barrageMessage) { future.completeExceptionally(sre); return; } - resultTable.handleBarrageMessage(barrageMessage); + localResultTable.handleBarrageMessage(barrageMessage); } } @@ -175,11 +176,12 @@ public void onError(final Throwable t) { final String label = TableSpecLabeler.of(tableHandle.export().table()); final TableDataException tde = new TableDataException( String.format("Barrage snapshot error for %s (%s)", logName, label), t); - if (resultTable != null) { + final BarrageTable localResultTable = resultTable; + if (localResultTable != null) { // this error will always be propagated to our CheckForCompletion#onError callback - resultTable.handleBarrageError(tde); + localResultTable.handleBarrageError(tde); } else { - future.completeExceptionally(t); + future.completeExceptionally(tde); } cleanup(); } @@ -190,7 +192,17 @@ public void onCompleted() { return; } - future.complete(resultTable); + final BarrageTable localResultTable = resultTable; + if (localResultTable == null) { + log.error().append(BarrageSnapshotImpl.this) + .append(": Received onComplete before snapshot was requested").endl(); + final StatusRuntimeException sre = Exceptions.statusRuntimeException( + Code.FAILED_PRECONDITION, "Received onComplete before snapshot was requested"); + GrpcUtil.safelyError(observer, sre); + future.completeExceptionally(sre); + return; + } + future.complete(localResultTable); cleanup(); } } @@ -219,9 +231,10 @@ public Future

partialTable( } final boolean isFullSubscription = viewport == null; - resultTable = BarrageTable.make(executorService, schema.tableDef, schema.attributes, isFullSubscription, - new CheckForCompletion()); - barrageStreamReader.setDeserializeTmConsumer(resultTable.getDeserializationTmConsumer()); + final BarrageTable localResultTable = BarrageTable.make( + executorService, schema.tableDef, schema.attributes, isFullSubscription, new CheckForCompletion()); + resultTable = localResultTable; + barrageStreamReader.setDeserializeTmConsumer(localResultTable.getDeserializationTmConsumer()); // Send the snapshot request: observer.onNext(FlightData.newBuilder() diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java index 2dfdc9fc36f..43bb20d0b1b 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java @@ -145,7 +145,8 @@ public void onNext(final BarrageMessage barrageMessage) { return; } - if (resultTable == null) { + final BarrageTable localResultTable = resultTable; + if (localResultTable == null) { log.error().append(BarrageSubscriptionImpl.this) .append(": Received data before subscription was requested").endl(); final StatusRuntimeException sre = Exceptions.statusRuntimeException( @@ -154,7 +155,7 @@ public void onNext(final BarrageMessage barrageMessage) { checkForCompletion.onError(sre); return; } - resultTable.handleBarrageMessage(barrageMessage); + localResultTable.handleBarrageMessage(barrageMessage); } } @@ -171,9 +172,10 @@ public void onError(final Throwable t) { final String label = TableSpecLabeler.of(tableHandle.export().table()); final TableDataException tde = new TableDataException( String.format("Barrage subscription error for %s (%s)", logName, label), t); - if (resultTable != null) { + final BarrageTable localResultTable = resultTable; + if (localResultTable != null) { // this error will always be propagated to our CheckForCompletion#onError callback - resultTable.handleBarrageError(tde); + localResultTable.handleBarrageError(tde); } else { checkForCompletion.onError(tde); } @@ -189,8 +191,9 @@ public void onCompleted() { log.error().append(BarrageSubscriptionImpl.this).append(": unexpectedly closed by other host").endl(); final RequestCancelledException cancelErr = new RequestCancelledException("Barrage subscription closed by server"); - if (resultTable != null) { - resultTable.handleBarrageError(cancelErr); + final BarrageTable localResultTable = resultTable; + if (localResultTable != null) { + localResultTable.handleBarrageError(cancelErr); } else { checkForCompletion.onError(cancelErr); } @@ -233,11 +236,16 @@ public Future
partialTable(RowSet viewport, BitSet columns, boolean rever subscribed = true; } + boolean isFullSubscription = viewport == null; + final BarrageTable localResultTable = BarrageTable.make( + executorService, schema.tableDef, schema.attributes, isFullSubscription, checkForCompletion); + resultTable = localResultTable; + // we must create the future before checking `isConnected` to guarantee `future` visibility in `destroy` if (isSnapshot) { future = new CompletableFutureAdapter(); } else { - future = new UpdateGraphAwareFutureAdapter(resultTable.getUpdateGraph()); + future = new UpdateGraphAwareFutureAdapter(localResultTable.getUpdateGraph()); } if (!isConnected()) { @@ -250,14 +258,11 @@ public Future
partialTable(RowSet viewport, BitSet columns, boolean rever columns == null ? null : (BitSet) (columns.clone()), reverseViewport); - boolean isFullSubscription = viewport == null; - resultTable = BarrageTable.make(executorService, schema.tableDef, schema.attributes, isFullSubscription, - checkForCompletion); - barrageStreamReader.setDeserializeTmConsumer(resultTable.getDeserializationTmConsumer()); + barrageStreamReader.setDeserializeTmConsumer(localResultTable.getDeserializationTmConsumer()); if (!isSnapshot) { - resultTable.addSourceToRegistrar(); - resultTable.addParentReference(this); + localResultTable.addSourceToRegistrar(); + localResultTable.addParentReference(this); } // Send the initial subscription: @@ -299,9 +304,10 @@ private void cancel(final String reason) { return; } - if (!isSnapshot) { + final BarrageTable localResultTable = resultTable; + if (!isSnapshot && localResultTable != null) { // Stop our result table from processing any more data. - resultTable.forceReferenceCountToZero(); + localResultTable.forceReferenceCountToZero(); } GrpcUtil.safelyCancel(observer, "Barrage subscription is " + reason, new RequestCancelledException("Barrage subscription is " + reason)); @@ -444,11 +450,12 @@ public synchronized boolean viewportChanged( return false; } + final BarrageTable localResultTable = resultTable; // @formatter:off final boolean correctColumns = // all columns are expected (expectedColumns == null - && (serverColumns == null || serverColumns.cardinality() == resultTable.numColumns())) + && (serverColumns == null || serverColumns.cardinality() == localResultTable.numColumns())) // only specific set of columns are expected || (expectedColumns != null && expectedColumns.equals(serverColumns)); @@ -457,7 +464,7 @@ public synchronized boolean viewportChanged( (correctColumns && expectedViewport == null && serverViewport == null) // Viewport subscription is completed || (correctColumns && expectedViewport != null - && expectedReverseViewport == resultTable.getServerReverseViewport() + && expectedReverseViewport == localResultTable.getServerReverseViewport() && expectedViewport.equals(serverViewport)); // @formatter:on @@ -465,14 +472,14 @@ public synchronized boolean viewportChanged( // remove all unpopulated rows from viewport snapshots if (isSnapshot && serverViewport != null) { // noinspection resource - WritableRowSet currentRowSet = resultTable.getRowSet().writableCast(); + final WritableRowSet currentRowSet = localResultTable.getRowSet().writableCast(); try (final RowSet populated = currentRowSet.subSetForPositions(serverViewport, serverReverseViewport)) { currentRowSet.retain(populated); } } - if (future.complete(resultTable)) { + if (future.complete(localResultTable)) { onFutureComplete(); } } diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 9e34f0e797e..81d607def9f 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -580,15 +580,17 @@ public boolean updateSubscription( @Nullable final BitSet columnsToSubscribe, final boolean newReverseViewport) { return findAndUpdateSubscription(listener, sub -> { - if (sub.pendingViewport != null) { - sub.pendingViewport.close(); - } - if (sub.isFullSubscription() != (newViewport == null)) { + if (sub.isFullSubscription()) { + // never allow changes to a full subscription GrpcUtil.safelyError(listener, Code.INVALID_ARGUMENT, "cannot change from full subscription to viewport or vice versa"); removeSubscription(listener); return; } + + if (sub.pendingViewport != null) { + sub.pendingViewport.close(); + } sub.pendingViewport = newViewport != null ? newViewport.copy() : null; sub.pendingReverseViewport = newReverseViewport; if (isBlinkTable && newReverseViewport) { @@ -1651,7 +1653,7 @@ private void propagateSnapshotForSubscription(final Subscription subscription, subscription.listener .onNext(snapshotGenerator.getSubView(subscription.options, subscription.pendingInitialSnapshot, subscription.isFullSubscription(), subscription.viewport, subscription.reverseViewport, - keySpaceViewport.copy(), keySpaceViewport, subscription.subscribedColumns)); + keySpaceViewport, keySpaceViewport, subscription.subscribedColumns)); } catch (final Exception e) { GrpcUtil.safelyError(subscription.listener, errorTransformer.transform(e)); diff --git a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java index 110cbad5d25..fb631c8fbbc 100644 --- a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java +++ b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java @@ -361,8 +361,7 @@ private static long buildAndSendSnapshot( // send the added rows on every snapshot, since (1) our added rows are flat, and thus cheap to send, and // (2) we're relying on added rows to signal the full expanded size to the client. GrpcUtil.safelyOnNext(listener, - streamGenerator.getSubView(subscriptionOptions, true, true, rows, false, rows.copy(), rows.copy(), - columns)); + streamGenerator.getSubView(subscriptionOptions, true, true, rows, false, rows, rows, columns)); } // 6. Let the caller know what the expanded size was diff --git a/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java b/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java index 58cbaaea078..954d7a519c2 100644 --- a/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java +++ b/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java @@ -232,8 +232,6 @@ public void validate(final String msg, QueryTable expected) { if (viewport != null) { expected = expected .getSubTable(expected.getRowSet().subSetForPositions(viewport, reverseViewport).toTracking()); - toCheck = toCheck - .getSubTable(toCheck.getRowSet().subSetForPositions(viewport, reverseViewport).toTracking()); } if (subscribedColumns.cardinality() != expected.numColumns()) { final List columns = new ArrayList<>(); @@ -246,10 +244,15 @@ public void validate(final String msg, QueryTable expected) { // Data should be identical and in-order. TstUtils.assertTableEquals(expected, toCheck); - // Since key-space needs to be kept the same, the RowSets should also be identical between producer and - // consumer (not the RowSets between expected and consumer; as the consumer maintains the entire RowSet). - Assert.equals(barrageMessageProducer.getRowSet(), "barrageMessageProducer.build()", - barrageTable.getRowSet(), ".build()"); + if (viewport == null) { + // Since key-space needs to be kept the same, the RowSets should also be identical between producer and + // consumer (not RowSets between expected and consumer; as the consumer maintains the entire RowSet). + Assert.equals(barrageMessageProducer.getRowSet(), "barrageMessageProducer.build()", + barrageTable.getRowSet(), ".build()"); + } else { + // otherwise, the RowSet should represent a flattened view of the viewport + Assert.eqTrue(barrageTable.getRowSet().isFlat(), "barrageTable.getRowSet().isFlat()"); + } } private void showResult(final String label, final Table table) { @@ -487,15 +490,15 @@ private class OneProducerPerClient extends TestHelper { } void createNuggetsForTableMaker(final Supplier
makeTable) { - nuggets.add(new RemoteNugget(makeTable)); final BitSet subscribedColumns = new BitSet(); - subscribedColumns.set(0, nuggets.get(nuggets.size() - 1).originalTable.numColumns()); + subscribedColumns.set(0, makeTable.get().numColumns()); + + nuggets.add(new RemoteNugget(makeTable)); nuggets.get(nuggets.size() - 1).newClient(null, subscribedColumns, "full"); nuggets.add(new RemoteNugget(makeTable)); nuggets.get(nuggets.size() - 1).newClient(RowSetFactory.fromRange(0, size / 10), - subscribedColumns, - "header"); + subscribedColumns, "header"); nuggets.add(new RemoteNugget(makeTable)); nuggets.get(nuggets.size() - 1).newClient( RowSetFactory.fromRange(size / 2, size * 3L / 4), @@ -534,17 +537,15 @@ private class SharedProducerForAllClients extends TestHelper { } void createNuggetsForTableMaker(final Supplier
makeTable) { - final RemoteNugget nugget = new RemoteNugget(makeTable); - nuggets.add(nugget); - final BitSet subscribedColumns = new BitSet(); - subscribedColumns.set(0, nugget.originalTable.numColumns()); + subscribedColumns.set(0, makeTable.get().numColumns()); + final RemoteNugget nugget = new RemoteNugget(makeTable); + nuggets.add(nugget); nugget.newClient(null, subscribedColumns, "full"); nugget.newClient(RowSetFactory.fromRange(0, size / 10), subscribedColumns, "header"); - nugget.newClient(RowSetFactory.fromRange(size / 2, size * 3L / 4), subscribedColumns, - "floating"); + nugget.newClient(RowSetFactory.fromRange(size / 2, size * 3L / 4), subscribedColumns, "floating"); nugget.newClient(RowSetFactory.fromRange(0, size / 10), subscribedColumns, true, "footer"); nugget.newClient(RowSetFactory.fromRange(size / 2, size * 3L / 4), subscribedColumns, true, From ad8de736a86591fa5074aac28a7d2a48ada9352e Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 11 Nov 2024 13:43:02 -0700 Subject: [PATCH 12/35] Remove SNAPSHOT version and mavenLocal references --- buildSrc/build.gradle | 1 - .../src/main/groovy/io.deephaven.repository-conventions.gradle | 1 - gradle/libs.versions.toml | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 4ad980da367..76254a824b1 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -10,7 +10,6 @@ java { } repositories { - mavenLocal() mavenCentral() maven { url "https://plugins.gradle.org/m2/" diff --git a/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle b/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle index ea0c0abeb8d..1deccf352c0 100644 --- a/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle +++ b/buildSrc/src/main/groovy/io.deephaven.repository-conventions.gradle @@ -1,6 +1,5 @@ repositories { mavenCentral() - mavenLocal() maven { url 'https://jitpack.io' content { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 2d57b26b1be..6c91cb8fc32 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,7 +17,7 @@ commons-text = "1.12.0" confluent = "7.6.0" confluent-kafka-clients = "7.6.0-ccs" dagger = "2.52" -deephaven-barrage = "0.7.0-SNAPSHOT" +deephaven-barrage = "0.7.0" deephaven-csv = "0.15.0" deephaven-hash = "0.1.0" deephaven-suan-shu = "0.1.1" From 02ce2ad5f717b00d1538e7f7dc763745d51ec153 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 12 Nov 2024 11:28:26 -0700 Subject: [PATCH 13/35] Fixes removed/added rows in most VP cases --- .../barrage/BarrageStreamGeneratorImpl.java | 60 +++++++++++-------- .../barrage/table/BarrageRedirectedTable.java | 46 ++++---------- .../barrage/BarrageMessageProducer.java | 9 ++- 3 files changed, 52 insertions(+), 63 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 620e08b9e7d..c97b79f33a1 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -14,6 +14,7 @@ import io.deephaven.barrage.flatbuf.BarrageMessageWrapper; import io.deephaven.barrage.flatbuf.BarrageModColumnMetadata; import io.deephaven.barrage.flatbuf.BarrageUpdateMetadata; +import io.deephaven.base.verify.Assert; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.WritableLongChunk; @@ -35,6 +36,7 @@ import io.deephaven.io.logger.Logger; import io.deephaven.proto.flight.util.MessageHelper; import io.deephaven.util.SafeCloseable; +import io.deephaven.util.SafeCloseableList; import io.deephaven.util.datastructures.LongSizedDataStructure; import io.deephaven.util.datastructures.SizeException; import io.deephaven.util.mutable.MutableInt; @@ -297,13 +299,20 @@ public SubView(final BarrageSubscriptionOptions options, this.numClientModRows = numModRows; if (keyspaceViewport != null) { - try (WritableRowSet intersect = keyspaceViewport.intersect(rowsIncluded.original)) { - if (isFullSubscription) { - clientAddedRows = intersect.copy(); - } else { - clientAddedRows = keyspaceViewport.invert(intersect); + Assert.neqNull(keyspaceViewportPrev, "keyspaceViewportPrev"); + try (final WritableRowSet existingRows = keyspaceViewportPrev.minus(rowsRemoved.original)) { + shifted.original.apply(existingRows); + try (final WritableRowSet toInclude = keyspaceViewport.minus(existingRows)) { + if (!toInclude.subsetOf(rowsIncluded.original)) { + throw new IllegalStateException("did not record row data needed for client"); + } + if (isFullSubscription) { + clientAddedRows = toInclude.copy(); + } else { + clientAddedRows = keyspaceViewport.invert(toInclude); + } + clientAddedRowOffsets = rowsIncluded.original.invert(toInclude); } - clientAddedRowOffsets = rowsIncluded.original.invert(intersect); } } else if (!rowsAdded.original.equals(rowsIncluded.original)) { // there are scoped rows included in the chunks that need to be removed @@ -416,19 +425,26 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { } final int rowsRemovedOffset; - if (isViewport() && !isFullSubscription) { - try (final WritableRowSet clientRemovedKeySpace = keyspaceViewportPrev.intersect(rowsRemoved.original); - final WritableRowSet clientRemovedPositionSpace = - keyspaceViewportPrev.invert(clientRemovedKeySpace); - final RowSetGenerator clientRemovedRowsGen = new RowSetGenerator(clientRemovedPositionSpace)) { - rowsRemovedOffset = clientRemovedRowsGen.addToFlatBuffer(metadata); + if (!isFullSubscription) { + // while rowsIncluded knows about rows that were scoped into view, rowsRemoved does not, and we need to + // infer them by comparing the previous keyspace viewport with the current keyspace viewport + try (final SafeCloseableList toClose = new SafeCloseableList()) { + + final WritableRowSet existingRows = toClose.add(keyspaceViewport.minus(rowsAdded.original)); + shifted.original.unapply(existingRows); + final WritableRowSet noLongerExistingRows = toClose.add(keyspaceViewportPrev.minus(existingRows)); + final WritableRowSet removedInPosSpace = + toClose.add(keyspaceViewportPrev.invert(noLongerExistingRows)); + try (final RowSetGenerator clientRemovedRowsGen = new RowSetGenerator(removedInPosSpace)) { + rowsRemovedOffset = clientRemovedRowsGen.addToFlatBuffer(metadata); + } } } else { rowsRemovedOffset = rowsRemoved.addToFlatBuffer(metadata); } final int shiftDataOffset; - if (isViewport() && !isFullSubscription) { - // never send shifts to a viewport subscriber + if (!isFullSubscription) { + // we only send shifts to full table subscriptions shiftDataOffset = 0; } else { shiftDataOffset = shifted.addToFlatBuffer(metadata); @@ -438,16 +454,8 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { int addedRowsIncludedOffset = 0; // don't send `rowsIncluded` when identical to `rowsAdded`, client will infer they are the same - if (isSnapshot || !clientAddedRows.equals(rowsAdded.original)) { - if (isViewport() && !isFullSubscription) { - try (final WritableRowSet inclInViewport = keyspaceViewport.intersect(rowsIncluded.original); - final WritableRowSet inclInPositionSpace = keyspaceViewport.invert(inclInViewport); - final RowSetGenerator inclRowsGen = new RowSetGenerator(inclInPositionSpace)) { - addedRowsIncludedOffset = inclRowsGen.addToFlatBuffer(metadata); - } - } else { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(clientAddedRows, metadata); - } + if (isFullSubscription && (isSnapshot || !clientAddedRows.equals(rowsAdded.original))) { + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(clientAddedRows, metadata); } // now add mod-column streams, and write the mod column indexes @@ -1148,7 +1156,11 @@ public BitSetGenerator(final BitSet bitset) { } public static class RowSetShiftDataGenerator extends ByteArrayGenerator { + private final RowSetShiftData original; + public RowSetShiftDataGenerator(final RowSetShiftData shifted) throws IOException { + original = shifted; + final RowSetBuilderSequential sRangeBuilder = RowSetFactory.builderSequential(); final RowSetBuilderSequential eRangeBuilder = RowSetFactory.builderSequential(); final RowSetBuilderSequential destBuilder = RowSetFactory.builderSequential(); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java index 31a4bf2fba5..1fd59114515 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java @@ -122,11 +122,11 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC // removes final long prevSize = currentRowSet.size(); - if (isFullSubscription) { - currentRowSet.remove(update.rowsRemoved); - } - try (final RowSet removed = populatedRows != null ? populatedRows.extract(update.rowsRemoved) : null) { - freeRows(removed != null ? removed : update.rowsRemoved); + currentRowSet.remove(update.rowsRemoved); + try (final RowSet removed = populatedRows != null + ? populatedRows.extract(update.rowsRemoved) + : currentRowSet.extract(update.rowsRemoved)) { + freeRows(removed); } final RowSetShiftData updateShiftData; @@ -139,36 +139,13 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC // shifts if (updateShiftData.nonempty()) { - try (final WritableRowSet postRemoveRowSet = isFullSubscription - ? null - : currentRowSet.minus(update.rowsRemoved)) { - rowRedirection.applyShift( - postRemoveRowSet != null ? postRemoveRowSet : currentRowSet, updateShiftData); - } - if (isFullSubscription) { - updateShiftData.apply(currentRowSet); - } + rowRedirection.applyShift(currentRowSet, updateShiftData); + updateShiftData.apply(currentRowSet); if (populatedRows != null) { updateShiftData.apply(populatedRows); } } - if (isFullSubscription) { - currentRowSet.insert(update.rowsAdded); - } else { - final long newSize; - if (update.isSnapshot) { - newSize = update.rowsAdded.size(); - } else { - // note that we are not told about rows that fall off the end of our respected viewport - newSize = Math.min(serverViewport.size(), - prevSize - update.rowsRemoved.size() + update.rowsIncluded.size()); - } - if (newSize < prevSize) { - currentRowSet.removeRange(newSize, prevSize - 1); - } else if (newSize > prevSize) { - currentRowSet.insertRange(prevSize, newSize - 1); - } - } + currentRowSet.insert(update.rowsAdded); final WritableRowSet totalMods = RowSetFactory.empty(); for (int i = 0; i < update.modColumnData.length; ++i) { @@ -281,10 +258,6 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC populatedRows.remove(newPopulated); freeRows(populatedRows); } - } else if (!isFullSubscription && prevSize > currentRowSet.size()) { - try (final RowSet toFree = RowSetFactory.fromRange(currentRowSet.size(), prevSize - 1)) { - freeRows(toFree); - } } if (update.isSnapshot && !mightBeInitialSnapshot) { @@ -295,7 +268,8 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC } final TableUpdate downstream = new TableUpdateImpl( - update.rowsAdded.copy(), update.rowsRemoved.copy(), totalMods, updateShiftData, modifiedColumnSet); + isFullSubscription ? update.rowsAdded.copy() : update.rowsIncluded.copy(), + update.rowsRemoved.copy(), totalMods, updateShiftData, modifiedColumnSet); return (coalescer == null) ? new UpdateCoalescer(currRowsFromPrev, downstream) : coalescer.update(downstream); } diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 81d607def9f..d7589713864 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -1621,8 +1621,6 @@ private void propagateSnapshotForSubscription(final Subscription subscription, // the parent table listener needs to be recording data as if we've already sent the successful snapshot. if (subscription.snapshotViewport != null) { - subscription.snapshotViewport.close(); - subscription.snapshotViewport = null; needsSnapshot = true; } @@ -1653,7 +1651,7 @@ private void propagateSnapshotForSubscription(final Subscription subscription, subscription.listener .onNext(snapshotGenerator.getSubView(subscription.options, subscription.pendingInitialSnapshot, subscription.isFullSubscription(), subscription.viewport, subscription.reverseViewport, - keySpaceViewport, keySpaceViewport, subscription.subscribedColumns)); + subscription.snapshotViewport, keySpaceViewport, subscription.subscribedColumns)); } catch (final Exception e) { GrpcUtil.safelyError(subscription.listener, errorTransformer.transform(e)); @@ -1661,6 +1659,11 @@ private void propagateSnapshotForSubscription(final Subscription subscription, } } + if (subscription.snapshotViewport != null) { + subscription.snapshotViewport.close(); + subscription.snapshotViewport = null; + } + if (subscription.growingIncrementalViewport != null) { subscription.growingIncrementalViewport.close(); subscription.growingIncrementalViewport = null; From da23e2b824a22515d32165dbc872100a57241533 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 12 Nov 2024 12:31:15 -0700 Subject: [PATCH 14/35] Bug fixes around viewport snapshot rowsRemoved and rowsAdded --- .../barrage/BarrageStreamGeneratorImpl.java | 12 ++++++++++-- .../barrage/table/BarrageRedirectedTable.java | 6 ++---- .../server/barrage/BarrageMessageProducer.java | 6 ++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index c97b79f33a1..25bdc59d76a 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -429,10 +429,18 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { // while rowsIncluded knows about rows that were scoped into view, rowsRemoved does not, and we need to // infer them by comparing the previous keyspace viewport with the current keyspace viewport try (final SafeCloseableList toClose = new SafeCloseableList()) { - - final WritableRowSet existingRows = toClose.add(keyspaceViewport.minus(rowsAdded.original)); + final WritableRowSet existingRows; + if (isSnapshot) { + existingRows = toClose.add(keyspaceViewport.copy()); + } else { + existingRows = toClose.add(keyspaceViewport.minus(rowsAdded.original)); + } shifted.original.unapply(existingRows); final WritableRowSet noLongerExistingRows = toClose.add(keyspaceViewportPrev.minus(existingRows)); + if (isSnapshot) { + // then we must filter noLongerExistingRows to only include rows in the table + noLongerExistingRows.retain(rowsAdded.original); + } final WritableRowSet removedInPosSpace = toClose.add(keyspaceViewportPrev.invert(noLongerExistingRows)); try (final RowSetGenerator clientRemovedRowsGen = new RowSetGenerator(removedInPosSpace)) { diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java index 1fd59114515..52ecc4f9410 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java @@ -123,10 +123,8 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC // removes final long prevSize = currentRowSet.size(); currentRowSet.remove(update.rowsRemoved); - try (final RowSet removed = populatedRows != null - ? populatedRows.extract(update.rowsRemoved) - : currentRowSet.extract(update.rowsRemoved)) { - freeRows(removed); + try (final RowSet removed = populatedRows != null ? populatedRows.extract(update.rowsRemoved) : null) { + freeRows(removed != null ? removed : update.rowsRemoved); } final RowSetShiftData updateShiftData; diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index d7589713864..2538851af4d 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -1638,7 +1638,9 @@ private void propagateSnapshotForSubscription(final Subscription subscription, // limit the rows included by this message to the subset of rows in this snapshot that this subscription // requested (exclude rows needed by other subscribers but not this one) try (final RowSet keySpaceViewport = snapshotGenerator.getMessage().rowsAdded - .subSetForPositions(subscription.growingIncrementalViewport, subscription.reverseViewport)) { + .subSetForPositions(subscription.viewport, subscription.reverseViewport); + final RowSet keySpaceViewportPrev = snapshotGenerator.getMessage().rowsAdded + .subSetForPositions(subscription.snapshotViewport, subscription.snapshotReverseViewport)) { if (subscription.pendingInitialSnapshot) { // Send schema metadata to this new client. @@ -1651,7 +1653,7 @@ private void propagateSnapshotForSubscription(final Subscription subscription, subscription.listener .onNext(snapshotGenerator.getSubView(subscription.options, subscription.pendingInitialSnapshot, subscription.isFullSubscription(), subscription.viewport, subscription.reverseViewport, - subscription.snapshotViewport, keySpaceViewport, subscription.subscribedColumns)); + keySpaceViewportPrev, keySpaceViewport, subscription.subscribedColumns)); } catch (final Exception e) { GrpcUtil.safelyError(subscription.listener, errorTransformer.transform(e)); From 299f56e5c8b8e3a5e27eabe9acc933d10eff640e Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 12 Nov 2024 12:46:29 -0700 Subject: [PATCH 15/35] Bugfix for correct growing VP logic --- .../barrage/BarrageStreamGeneratorImpl.java | 23 ++++++++++--------- .../barrage/BarrageMessageProducer.java | 15 ++++++++---- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 25bdc59d76a..bc59a88c5d8 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -299,19 +299,20 @@ public SubView(final BarrageSubscriptionOptions options, this.numClientModRows = numModRows; if (keyspaceViewport != null) { - Assert.neqNull(keyspaceViewportPrev, "keyspaceViewportPrev"); - try (final WritableRowSet existingRows = keyspaceViewportPrev.minus(rowsRemoved.original)) { - shifted.original.apply(existingRows); - try (final WritableRowSet toInclude = keyspaceViewport.minus(existingRows)) { - if (!toInclude.subsetOf(rowsIncluded.original)) { - throw new IllegalStateException("did not record row data needed for client"); - } - if (isFullSubscription) { - clientAddedRows = toInclude.copy(); - } else { + if (isFullSubscription) { + clientAddedRows = keyspaceViewport.intersect(rowsAdded.original); + clientAddedRowOffsets = rowsIncluded.original.invert(rowsAdded.original); + } else { + Assert.neqNull(keyspaceViewportPrev, "keyspaceViewportPrev"); + try (final WritableRowSet existingRows = keyspaceViewportPrev.minus(rowsRemoved.original)) { + shifted.original.apply(existingRows); + try (final WritableRowSet toInclude = keyspaceViewport.minus(existingRows)) { + if (!toInclude.subsetOf(rowsIncluded.original)) { + throw new IllegalStateException("did not record row data needed for client"); + } clientAddedRows = keyspaceViewport.invert(toInclude); + clientAddedRowOffsets = rowsIncluded.original.invert(toInclude); } - clientAddedRowOffsets = rowsIncluded.original.invert(toInclude); } } } else if (!rowsAdded.original.equals(rowsIncluded.original)) { diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 2538851af4d..ecf5d73e289 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -1637,10 +1637,17 @@ private void propagateSnapshotForSubscription(final Subscription subscription, // limit the rows included by this message to the subset of rows in this snapshot that this subscription // requested (exclude rows needed by other subscribers but not this one) + boolean fullSubscription = subscription.isFullSubscription(); try (final RowSet keySpaceViewport = snapshotGenerator.getMessage().rowsAdded - .subSetForPositions(subscription.viewport, subscription.reverseViewport); - final RowSet keySpaceViewportPrev = snapshotGenerator.getMessage().rowsAdded - .subSetForPositions(subscription.snapshotViewport, subscription.snapshotReverseViewport)) { + .subSetForPositions(fullSubscription + ? subscription.growingIncrementalViewport + : subscription.viewport, + subscription.reverseViewport); + final RowSet keySpaceViewportPrev = fullSubscription + ? null + : snapshotGenerator.getMessage().rowsAdded + .subSetForPositions(subscription.snapshotViewport, + subscription.snapshotReverseViewport)) { if (subscription.pendingInitialSnapshot) { // Send schema metadata to this new client. @@ -1652,7 +1659,7 @@ private void propagateSnapshotForSubscription(final Subscription subscription, // some messages may be empty of rows, but we need to update the client viewport and column set subscription.listener .onNext(snapshotGenerator.getSubView(subscription.options, subscription.pendingInitialSnapshot, - subscription.isFullSubscription(), subscription.viewport, subscription.reverseViewport, + fullSubscription, subscription.viewport, subscription.reverseViewport, keySpaceViewportPrev, keySpaceViewport, subscription.subscribedColumns)); } catch (final Exception e) { From 9d6f3890ba09d2b76d31ee3a1a41af6dd0054da9 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Wed, 13 Nov 2024 00:29:50 -0700 Subject: [PATCH 16/35] remaining java side fixes --- .../barrage/BarrageStreamGeneratorImpl.java | 30 +++++++++---- .../barrage/table/BarrageRedirectedTable.java | 44 ++++++++++++++----- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index bc59a88c5d8..4d4248dd0b1 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -251,6 +251,7 @@ private final class SubView implements RecordBatchMessageView { private final RowSet clientAddedRowOffsets; private final RowSet[] clientModdedRows; private final RowSet[] clientModdedRowOffsets; + private final RowSet clientAddOrScopedRows; public SubView(final BarrageSubscriptionOptions options, final boolean isInitialSnapshot, @@ -300,18 +301,21 @@ public SubView(final BarrageSubscriptionOptions options, if (keyspaceViewport != null) { if (isFullSubscription) { - clientAddedRows = keyspaceViewport.intersect(rowsAdded.original); - clientAddedRowOffsets = rowsIncluded.original.invert(rowsAdded.original); + clientAddOrScopedRows = RowSetFactory.empty(); + clientAddedRows = keyspaceViewport.intersect(rowsIncluded.original); + clientAddedRowOffsets = rowsIncluded.original.invert(clientAddedRows); } else { Assert.neqNull(keyspaceViewportPrev, "keyspaceViewportPrev"); - try (final WritableRowSet existingRows = keyspaceViewportPrev.minus(rowsRemoved.original)) { + try (final WritableRowSet clientIncludedRows = keyspaceViewport.intersect(rowsIncluded.original); + final WritableRowSet existingRows = keyspaceViewportPrev.minus(rowsRemoved.original)) { + clientAddedRows = keyspaceViewport.invert(clientIncludedRows); + clientAddedRowOffsets = rowsIncluded.original.invert(clientIncludedRows); shifted.original.apply(existingRows); try (final WritableRowSet toInclude = keyspaceViewport.minus(existingRows)) { if (!toInclude.subsetOf(rowsIncluded.original)) { throw new IllegalStateException("did not record row data needed for client"); } - clientAddedRows = keyspaceViewport.invert(toInclude); - clientAddedRowOffsets = rowsIncluded.original.invert(toInclude); + clientAddOrScopedRows = keyspaceViewport.invert(toInclude); } } } @@ -319,9 +323,11 @@ public SubView(final BarrageSubscriptionOptions options, // there are scoped rows included in the chunks that need to be removed clientAddedRows = rowsAdded.original.copy(); clientAddedRowOffsets = rowsIncluded.original.invert(clientAddedRows); + clientAddOrScopedRows = RowSetFactory.empty(); } else { clientAddedRows = rowsAdded.original.copy(); clientAddedRowOffsets = RowSetFactory.flat(rowsAdded.original.size()); + clientAddOrScopedRows = RowSetFactory.empty(); } this.numClientAddRows = clientAddedRowOffsets.size(); @@ -415,7 +421,7 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { final int rowsAddedOffset; if (!isFullSubscription) { - try (final RowSetGenerator clientAddedRowsGen = new RowSetGenerator(clientAddedRows)) { + try (final RowSetGenerator clientAddedRowsGen = new RowSetGenerator(clientAddOrScopedRows)) { rowsAddedOffset = clientAddedRowsGen.addToFlatBuffer(metadata); } } else if (isSnapshot && !isInitialSnapshot) { @@ -463,8 +469,16 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { int addedRowsIncludedOffset = 0; // don't send `rowsIncluded` when identical to `rowsAdded`, client will infer they are the same - if (isFullSubscription && (isSnapshot || !clientAddedRows.equals(rowsAdded.original))) { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(clientAddedRows, metadata); + if (isFullSubscription) { + if (isSnapshot || !clientAddedRows.equals(rowsAdded.original)) { + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(clientAddedRows, metadata); + } + } else if (!clientAddedRows.equals(clientAddOrScopedRows)) { + // the clientAddedRows on a viewport are all rows sent with the message; including rows that scoped out + // of view, were modified, and then scoped back into view within the same coalesced source message + try (final RowSetGenerator clientAddOrScopedRowsGen = new RowSetGenerator(clientAddedRows)) { + addedRowsIncludedOffset = clientAddOrScopedRowsGen.addToFlatBuffer(metadata); + } } // now add mod-column streams, and write the mod column indexes diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java index 52ecc4f9410..48f6192fa89 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java @@ -115,24 +115,39 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC final RowSet serverViewport = getServerViewport(); final boolean serverReverseViewport = getServerReverseViewport(); + final RowSet scopedRows; + if (isFullSubscription) { + scopedRows = RowSetFactory.empty(); + } else { + scopedRows = update.rowsIncluded.minus(update.rowsAdded); + } + try (final RowSet currRowsFromPrev = currentRowSet.copy(); final WritableRowSet populatedRows = serverViewport != null && isFullSubscription ? currentRowSet.subSetForPositions(serverViewport, serverReverseViewport) : null) { - // removes - final long prevSize = currentRowSet.size(); - currentRowSet.remove(update.rowsRemoved); - try (final RowSet removed = populatedRows != null ? populatedRows.extract(update.rowsRemoved) : null) { - freeRows(removed != null ? removed : update.rowsRemoved); - } - final RowSetShiftData updateShiftData; if (isFullSubscription) { updateShiftData = update.shifted; } else { updateShiftData = FlattenOperation.computeFlattenedRowSetShiftData( - update.rowsRemoved, update.rowsAdded, prevSize); + update.rowsRemoved, update.rowsAdded, currentRowSet.size()); + } + + // removes + currentRowSet.remove(update.rowsRemoved); + try (final RowSet removed = populatedRows != null ? populatedRows.extract(update.rowsRemoved) : null) { + freeRows(removed != null ? removed : update.rowsRemoved); + } + if (scopedRows.isNonempty()) { + try (final RowSet prevScopedRows = updateShiftData.unapply(scopedRows.copy()); + final RowSet removed = currentRowSet.extract(prevScopedRows)) { + freeRows(removed); + if (populatedRows != null) { + populatedRows.remove(removed); + } + } } // shifts @@ -144,6 +159,9 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC } } currentRowSet.insert(update.rowsAdded); + if (scopedRows.isNonempty()) { + currentRowSet.insert(scopedRows); + } final WritableRowSet totalMods = RowSetFactory.empty(); for (int i = 0; i < update.modColumnData.length; ++i) { @@ -265,9 +283,15 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC return coalescer; } + final WritableRowSet totalRowsAdded = update.rowsAdded.union(scopedRows); + if (!isFullSubscription) { + totalMods.remove(totalRowsAdded); + } final TableUpdate downstream = new TableUpdateImpl( - isFullSubscription ? update.rowsAdded.copy() : update.rowsIncluded.copy(), - update.rowsRemoved.copy(), totalMods, updateShiftData, modifiedColumnSet); + totalRowsAdded, update.rowsRemoved.union(scopedRows), totalMods, updateShiftData, + modifiedColumnSet); + scopedRows.close(); + return (coalescer == null) ? new UpdateCoalescer(currRowsFromPrev, downstream) : coalescer.update(downstream); } From fd5aced957252376757bffdc38823503213e2ce3 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Thu, 14 Nov 2024 09:59:56 -0700 Subject: [PATCH 17/35] Ryan's feedback on javaserver/client impls --- .../engine/rowset/RowSetShiftData.java | 4 +- .../barrage/BarrageStreamGeneratorImpl.java | 133 +++++++++--------- .../barrage/table/BarrageRedirectedTable.java | 30 +--- .../AbstractTableSubscription.java | 1 - 4 files changed, 73 insertions(+), 95 deletions(-) diff --git a/engine/rowset/src/main/java/io/deephaven/engine/rowset/RowSetShiftData.java b/engine/rowset/src/main/java/io/deephaven/engine/rowset/RowSetShiftData.java index 8db68e8e0fe..c723a94c07c 100644 --- a/engine/rowset/src/main/java/io/deephaven/engine/rowset/RowSetShiftData.java +++ b/engine/rowset/src/main/java/io/deephaven/engine/rowset/RowSetShiftData.java @@ -280,7 +280,7 @@ public void unapply(final RowKeyRangeShiftCallback shiftCallback) { * @param rowSet The {@link WritableRowSet} to shift * @return {@code rowSet} */ - public boolean apply(final WritableRowSet rowSet) { + public WritableRowSet apply(final WritableRowSet rowSet) { final RowSetBuilderSequential toRemove = RowSetFactory.builderSequential(); final RowSetBuilderSequential toInsert = RowSetFactory.builderSequential(); try (final RowSequence.Iterator rsIt = rowSet.getRowSequenceIterator()) { @@ -315,7 +315,7 @@ public boolean apply(final WritableRowSet rowSet) { rowSet.remove(remove); rowSet.insert(insert); - return remove.isNonempty() || insert.isNonempty(); + return rowSet; } } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 4d4248dd0b1..5fb2aebe9d4 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -245,13 +245,14 @@ private final class SubView implements RecordBatchMessageView { private final RowSet keyspaceViewportPrev; private final RowSet keyspaceViewport; private final BitSet subscribedColumns; + private final long numClientAddRows; private final long numClientModRows; - private final RowSet clientAddedRows; - private final RowSet clientAddedRowOffsets; + private final RowSet clientIncludedRows; + private final RowSet clientIncludedRowOffsets; private final RowSet[] clientModdedRows; private final RowSet[] clientModdedRowOffsets; - private final RowSet clientAddOrScopedRows; + private final RowSet clientRemovedRows; public SubView(final BarrageSubscriptionOptions options, final boolean isInitialSnapshot, @@ -299,38 +300,58 @@ public SubView(final BarrageSubscriptionOptions options, } this.numClientModRows = numModRows; - if (keyspaceViewport != null) { - if (isFullSubscription) { - clientAddOrScopedRows = RowSetFactory.empty(); - clientAddedRows = keyspaceViewport.intersect(rowsIncluded.original); - clientAddedRowOffsets = rowsIncluded.original.invert(clientAddedRows); + if (isFullSubscription) { + clientRemovedRows = null; // we'll send full subscriptions the full removed set + + if (keyspaceViewport != null) { + // growing viewport clients need to know about all rows, including those that were scoped into view + clientIncludedRows = keyspaceViewport.intersect(rowsIncluded.original); + clientIncludedRowOffsets = rowsIncluded.original.invert(clientIncludedRows); + } else if (!rowsAdded.original.equals(rowsIncluded.original)) { + // there are scoped rows that need to be removed from the data sent to the client + clientIncludedRows = rowsAdded.original.copy(); + clientIncludedRowOffsets = rowsIncluded.original.invert(clientIncludedRows); } else { - Assert.neqNull(keyspaceViewportPrev, "keyspaceViewportPrev"); - try (final WritableRowSet clientIncludedRows = keyspaceViewport.intersect(rowsIncluded.original); - final WritableRowSet existingRows = keyspaceViewportPrev.minus(rowsRemoved.original)) { - clientAddedRows = keyspaceViewport.invert(clientIncludedRows); - clientAddedRowOffsets = rowsIncluded.original.invert(clientIncludedRows); - shifted.original.apply(existingRows); - try (final WritableRowSet toInclude = keyspaceViewport.minus(existingRows)) { - if (!toInclude.subsetOf(rowsIncluded.original)) { - throw new IllegalStateException("did not record row data needed for client"); - } - clientAddOrScopedRows = keyspaceViewport.invert(toInclude); - } - } + clientIncludedRows = rowsAdded.original.copy(); + clientIncludedRowOffsets = RowSetFactory.flat(rowsAdded.original.size()); } - } else if (!rowsAdded.original.equals(rowsIncluded.original)) { - // there are scoped rows included in the chunks that need to be removed - clientAddedRows = rowsAdded.original.copy(); - clientAddedRowOffsets = rowsIncluded.original.invert(clientAddedRows); - clientAddOrScopedRows = RowSetFactory.empty(); } else { - clientAddedRows = rowsAdded.original.copy(); - clientAddedRowOffsets = RowSetFactory.flat(rowsAdded.original.size()); - clientAddOrScopedRows = RowSetFactory.empty(); + Assert.neqNull(keyspaceViewportPrev, "keyspaceViewportPrev"); + try (final SafeCloseableList toClose = new SafeCloseableList()) { + final WritableRowSet clientIncludedRows = + toClose.add(keyspaceViewport.intersect(rowsIncluded.original)); + // all included rows are sent to viewport clients as adds (already includes repainted rows) + this.clientIncludedRows = keyspaceViewport.invert(clientIncludedRows); + clientIncludedRowOffsets = rowsIncluded.original.invert(clientIncludedRows); + + // A row may slide out of the viewport and back into the viewport within the same coalesced message. + // The coalesced adds/removes will not contain this row, but the server has recorded it as needing + // to be sent to the client in its entirety. The client will process this row as both removed and + // added. + final WritableRowSet clientRepaintedRows = toClose.add(clientIncludedRows.copy()); + if (!isSnapshot) { + // note that snapshot rowsAdded contain all rows; we "repaint" only rows shared between prev and + // new viewports. + clientRepaintedRows.remove(rowsAdded.original); + shifted.original.unapply(clientRepaintedRows); + } + clientRepaintedRows.retain(keyspaceViewportPrev); + + // any pre-existing rows that are no longer in the viewport also need to be removed + final WritableRowSet existing; + if (isSnapshot) { + existing = toClose.add(keyspaceViewport.copy()); + } else { + existing = toClose.add(keyspaceViewport.minus(rowsAdded.original)); + } + shifted.original.unapply(existing); + final WritableRowSet noLongerExistingRows = toClose.add(keyspaceViewportPrev.minus(existing)); + noLongerExistingRows.insert(clientRepaintedRows); + clientRemovedRows = keyspaceViewportPrev.invert(noLongerExistingRows); + } } - this.numClientAddRows = clientAddedRowOffsets.size(); + this.numClientAddRows = clientIncludedRowOffsets.size(); } @Override @@ -363,12 +384,15 @@ public void forEachStream(Consumer visitor) throws IOExcepti processBatches(visitor, this, numClientModRows, maxBatchSize, numClientAddRows > 0 ? null : metadata, BarrageStreamGeneratorImpl.this::appendModColumns, bytesWritten); } finally { - clientAddedRowOffsets.close(); - clientAddedRows.close(); + clientIncludedRows.close(); + clientIncludedRowOffsets.close(); if (clientModdedRowOffsets != null) { SafeCloseable.closeAll(clientModdedRows); SafeCloseable.closeAll(clientModdedRowOffsets); } + if (clientRemovedRows != null) { + clientRemovedRows.close(); + } } writeConsumer.onWrite(bytesWritten.get(), System.nanoTime() - startTm); } @@ -393,7 +417,7 @@ public StreamReaderOptions options() { @Override public RowSet addRowOffsets() { - return clientAddedRowOffsets; + return clientIncludedRowOffsets; } @Override @@ -421,8 +445,9 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { final int rowsAddedOffset; if (!isFullSubscription) { - try (final RowSetGenerator clientAddedRowsGen = new RowSetGenerator(clientAddOrScopedRows)) { - rowsAddedOffset = clientAddedRowsGen.addToFlatBuffer(metadata); + // viewport clients consider all rows as added; scoped rows will also appear in the removed set + try (final RowSetGenerator clientIncludedRowsGen = new RowSetGenerator(clientIncludedRows)) { + rowsAddedOffset = clientIncludedRowsGen.addToFlatBuffer(metadata); } } else if (isSnapshot && !isInitialSnapshot) { // Growing viewport clients don't need/want to receive the full RowSet on every snapshot @@ -433,30 +458,14 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { final int rowsRemovedOffset; if (!isFullSubscription) { - // while rowsIncluded knows about rows that were scoped into view, rowsRemoved does not, and we need to - // infer them by comparing the previous keyspace viewport with the current keyspace viewport - try (final SafeCloseableList toClose = new SafeCloseableList()) { - final WritableRowSet existingRows; - if (isSnapshot) { - existingRows = toClose.add(keyspaceViewport.copy()); - } else { - existingRows = toClose.add(keyspaceViewport.minus(rowsAdded.original)); - } - shifted.original.unapply(existingRows); - final WritableRowSet noLongerExistingRows = toClose.add(keyspaceViewportPrev.minus(existingRows)); - if (isSnapshot) { - // then we must filter noLongerExistingRows to only include rows in the table - noLongerExistingRows.retain(rowsAdded.original); - } - final WritableRowSet removedInPosSpace = - toClose.add(keyspaceViewportPrev.invert(noLongerExistingRows)); - try (final RowSetGenerator clientRemovedRowsGen = new RowSetGenerator(removedInPosSpace)) { - rowsRemovedOffset = clientRemovedRowsGen.addToFlatBuffer(metadata); - } + // viewport clients need to also remove rows that were scoped out of view; computed in the constructor + try (final RowSetGenerator clientRemovedRowsGen = new RowSetGenerator(clientRemovedRows)) { + rowsRemovedOffset = clientRemovedRowsGen.addToFlatBuffer(metadata); } } else { rowsRemovedOffset = rowsRemoved.addToFlatBuffer(metadata); } + final int shiftDataOffset; if (!isFullSubscription) { // we only send shifts to full table subscriptions @@ -468,17 +477,9 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { // Added Chunk Data: int addedRowsIncludedOffset = 0; - // don't send `rowsIncluded` when identical to `rowsAdded`, client will infer they are the same - if (isFullSubscription) { - if (isSnapshot || !clientAddedRows.equals(rowsAdded.original)) { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(clientAddedRows, metadata); - } - } else if (!clientAddedRows.equals(clientAddOrScopedRows)) { - // the clientAddedRows on a viewport are all rows sent with the message; including rows that scoped out - // of view, were modified, and then scoped back into view within the same coalesced source message - try (final RowSetGenerator clientAddOrScopedRowsGen = new RowSetGenerator(clientAddedRows)) { - addedRowsIncludedOffset = clientAddOrScopedRowsGen.addToFlatBuffer(metadata); - } + // don't send `rowsIncluded` to viewport clients or if identical to `rowsAdded` + if (isFullSubscription && (isSnapshot || !clientIncludedRows.equals(rowsAdded.original))) { + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(clientIncludedRows, metadata); } // now add mod-column streams, and write the mod column indexes diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java index 48f6192fa89..8cdc77380ed 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java @@ -115,13 +115,6 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC final RowSet serverViewport = getServerViewport(); final boolean serverReverseViewport = getServerReverseViewport(); - final RowSet scopedRows; - if (isFullSubscription) { - scopedRows = RowSetFactory.empty(); - } else { - scopedRows = update.rowsIncluded.minus(update.rowsAdded); - } - try (final RowSet currRowsFromPrev = currentRowSet.copy(); final WritableRowSet populatedRows = serverViewport != null && isFullSubscription ? currentRowSet.subSetForPositions(serverViewport, serverReverseViewport) @@ -137,17 +130,8 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC // removes currentRowSet.remove(update.rowsRemoved); - try (final RowSet removed = populatedRows != null ? populatedRows.extract(update.rowsRemoved) : null) { - freeRows(removed != null ? removed : update.rowsRemoved); - } - if (scopedRows.isNonempty()) { - try (final RowSet prevScopedRows = updateShiftData.unapply(scopedRows.copy()); - final RowSet removed = currentRowSet.extract(prevScopedRows)) { - freeRows(removed); - if (populatedRows != null) { - populatedRows.remove(removed); - } - } + try (final RowSet populatedRowsRemoved = populatedRows != null ? populatedRows.extract(update.rowsRemoved) : null) { + freeRows(populatedRowsRemoved != null ? populatedRowsRemoved : update.rowsRemoved); } // shifts @@ -159,9 +143,6 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC } } currentRowSet.insert(update.rowsAdded); - if (scopedRows.isNonempty()) { - currentRowSet.insert(scopedRows); - } final WritableRowSet totalMods = RowSetFactory.empty(); for (int i = 0; i < update.modColumnData.length; ++i) { @@ -283,14 +264,11 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC return coalescer; } - final WritableRowSet totalRowsAdded = update.rowsAdded.union(scopedRows); if (!isFullSubscription) { - totalMods.remove(totalRowsAdded); + totalMods.remove(update.rowsIncluded); } final TableUpdate downstream = new TableUpdateImpl( - totalRowsAdded, update.rowsRemoved.union(scopedRows), totalMods, updateShiftData, - modifiedColumnSet); - scopedRows.close(); + update.rowsAdded, update.rowsRemoved, totalMods, updateShiftData, modifiedColumnSet); return (coalescer == null) ? new UpdateCoalescer(currRowsFromPrev, downstream) : coalescer.update(downstream); diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java index a834ad5ca22..e91ec5e4776 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java @@ -9,7 +9,6 @@ import io.deephaven.barrage.flatbuf.BarrageMessageType; import io.deephaven.barrage.flatbuf.BarrageSubscriptionRequest; import io.deephaven.extensions.barrage.BarrageSubscriptionOptions; -import io.deephaven.extensions.barrage.ColumnConversionMode; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightData; import io.deephaven.web.client.api.Column; import io.deephaven.web.client.api.Format; From 53b1eed9c7f25e25414eb688cbe26507f205f1e4 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Thu, 14 Nov 2024 13:02:13 -0700 Subject: [PATCH 18/35] Inline Feedback from VC w/Ryan --- .../barrage/BarrageStreamGeneratorImpl.java | 40 ++++++++++--------- .../barrage/table/BarrageRedirectedTable.java | 3 +- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 5fb2aebe9d4..002128200b4 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -246,7 +246,7 @@ private final class SubView implements RecordBatchMessageView { private final RowSet keyspaceViewport; private final BitSet subscribedColumns; - private final long numClientAddRows; + private final long numClientIncludedRows; private final long numClientModRows; private final RowSet clientIncludedRows; private final RowSet clientIncludedRowOffsets; @@ -318,40 +318,41 @@ public SubView(final BarrageSubscriptionOptions options, } else { Assert.neqNull(keyspaceViewportPrev, "keyspaceViewportPrev"); try (final SafeCloseableList toClose = new SafeCloseableList()) { - final WritableRowSet clientIncludedRows = + final WritableRowSet keyspaceClientIncludedRows = toClose.add(keyspaceViewport.intersect(rowsIncluded.original)); // all included rows are sent to viewport clients as adds (already includes repainted rows) - this.clientIncludedRows = keyspaceViewport.invert(clientIncludedRows); - clientIncludedRowOffsets = rowsIncluded.original.invert(clientIncludedRows); + clientIncludedRows = keyspaceViewport.invert(keyspaceClientIncludedRows); + clientIncludedRowOffsets = rowsIncluded.original.invert(keyspaceClientIncludedRows); // A row may slide out of the viewport and back into the viewport within the same coalesced message. // The coalesced adds/removes will not contain this row, but the server has recorded it as needing // to be sent to the client in its entirety. The client will process this row as both removed and // added. - final WritableRowSet clientRepaintedRows = toClose.add(clientIncludedRows.copy()); + final WritableRowSet keyspacePrevClientRepaintedRows = + toClose.add(keyspaceClientIncludedRows.copy()); if (!isSnapshot) { // note that snapshot rowsAdded contain all rows; we "repaint" only rows shared between prev and // new viewports. - clientRepaintedRows.remove(rowsAdded.original); - shifted.original.unapply(clientRepaintedRows); + keyspacePrevClientRepaintedRows.remove(rowsAdded.original); + shifted.original.unapply(keyspacePrevClientRepaintedRows); } - clientRepaintedRows.retain(keyspaceViewportPrev); + keyspacePrevClientRepaintedRows.retain(keyspaceViewportPrev); // any pre-existing rows that are no longer in the viewport also need to be removed - final WritableRowSet existing; + final WritableRowSet rowsToRetain; if (isSnapshot) { - existing = toClose.add(keyspaceViewport.copy()); + rowsToRetain = toClose.add(keyspaceViewport.copy()); } else { - existing = toClose.add(keyspaceViewport.minus(rowsAdded.original)); + rowsToRetain = toClose.add(keyspaceViewport.minus(rowsAdded.original)); + shifted.original.unapply(rowsToRetain); } - shifted.original.unapply(existing); - final WritableRowSet noLongerExistingRows = toClose.add(keyspaceViewportPrev.minus(existing)); - noLongerExistingRows.insert(clientRepaintedRows); + final WritableRowSet noLongerExistingRows = toClose.add(keyspaceViewportPrev.minus(rowsToRetain)); + noLongerExistingRows.insert(keyspacePrevClientRepaintedRows); clientRemovedRows = keyspaceViewportPrev.invert(noLongerExistingRows); } } - this.numClientAddRows = clientIncludedRowOffsets.size(); + this.numClientIncludedRows = clientIncludedRowOffsets.size(); } @Override @@ -365,7 +366,7 @@ public void forEachStream(Consumer visitor) throws IOExcepti final MutableInt actualBatchSize = new MutableInt(); - if (numClientAddRows == 0 && numClientModRows == 0) { + if (numClientIncludedRows == 0 && numClientModRows == 0) { // we still need to send a message containing metadata when there are no rows final DefensiveDrainable is = getInputStream(this, 0, 0, actualBatchSize, metadata, BarrageStreamGeneratorImpl.this::appendAddColumns); @@ -377,11 +378,12 @@ public void forEachStream(Consumer visitor) throws IOExcepti // send the add batches (if any) try { - processBatches(visitor, this, numClientAddRows, maxBatchSize, metadata, + processBatches(visitor, this, numClientIncludedRows, maxBatchSize, metadata, BarrageStreamGeneratorImpl.this::appendAddColumns, bytesWritten); // send the mod batches (if any) but don't send metadata twice - processBatches(visitor, this, numClientModRows, maxBatchSize, numClientAddRows > 0 ? null : metadata, + processBatches(visitor, this, numClientModRows, maxBatchSize, + numClientIncludedRows > 0 ? null : metadata, BarrageStreamGeneratorImpl.this::appendModColumns, bytesWritten); } finally { clientIncludedRows.close(); @@ -445,7 +447,7 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { final int rowsAddedOffset; if (!isFullSubscription) { - // viewport clients consider all rows as added; scoped rows will also appear in the removed set + // viewport clients consider all included rows as added; scoped rows will also appear in the removed set try (final RowSetGenerator clientIncludedRowsGen = new RowSetGenerator(clientIncludedRows)) { rowsAddedOffset = clientIncludedRowsGen.addToFlatBuffer(metadata); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java index 8cdc77380ed..fb29c9e36bb 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java @@ -130,7 +130,8 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC // removes currentRowSet.remove(update.rowsRemoved); - try (final RowSet populatedRowsRemoved = populatedRows != null ? populatedRows.extract(update.rowsRemoved) : null) { + try (final RowSet populatedRowsRemoved = + populatedRows != null ? populatedRows.extract(update.rowsRemoved) : null) { freeRows(populatedRowsRemoved != null ? populatedRowsRemoved : update.rowsRemoved); } From 6e7fe94f454af2f68ab652c53df355190de4fc35 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Thu, 14 Nov 2024 13:48:33 -0700 Subject: [PATCH 19/35] Do not propagate modifies for any repainted rows --- .../barrage/BarrageStreamGeneratorImpl.java | 76 ++++++++++--------- .../barrage/table/BarrageRedirectedTable.java | 3 - 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 002128200b4..1c59a38c054 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -242,17 +242,16 @@ private final class SubView implements RecordBatchMessageView { private final boolean isFullSubscription; private final RowSet viewport; private final boolean reverseViewport; - private final RowSet keyspaceViewportPrev; private final RowSet keyspaceViewport; private final BitSet subscribedColumns; private final long numClientIncludedRows; private final long numClientModRows; - private final RowSet clientIncludedRows; - private final RowSet clientIncludedRowOffsets; - private final RowSet[] clientModdedRows; - private final RowSet[] clientModdedRowOffsets; - private final RowSet clientRemovedRows; + private final WritableRowSet clientIncludedRows; + private final WritableRowSet clientIncludedRowOffsets; + private final WritableRowSet[] clientModdedRows; + private final WritableRowSet[] clientModdedRowOffsets; + private final WritableRowSet clientRemovedRows; public SubView(final BarrageSubscriptionOptions options, final boolean isInitialSnapshot, @@ -267,39 +266,10 @@ public SubView(final BarrageSubscriptionOptions options, this.isFullSubscription = isFullSubscription; this.viewport = viewport; this.reverseViewport = reverseViewport; - this.keyspaceViewportPrev = keyspaceViewportPrev; this.keyspaceViewport = keyspaceViewport; this.subscribedColumns = subscribedColumns; - if (keyspaceViewport != null) { - this.clientModdedRows = new WritableRowSet[modColumnData.length]; - this.clientModdedRowOffsets = new WritableRowSet[modColumnData.length]; - } else { - this.clientModdedRows = null; - this.clientModdedRowOffsets = null; - } - - // precompute the modified column indexes, and calculate total rows needed - long numModRows = 0; - for (int ii = 0; ii < modColumnData.length; ++ii) { - final ModColumnGenerator mcd = modColumnData[ii]; - - if (keyspaceViewport != null) { - try (WritableRowSet intersect = keyspaceViewport.intersect(mcd.rowsModified.original)) { - if (isFullSubscription) { - clientModdedRows[ii] = intersect.copy(); - } else { - clientModdedRows[ii] = keyspaceViewport.invert(intersect); - } - clientModdedRowOffsets[ii] = mcd.rowsModified.original.invert(intersect); - numModRows = Math.max(numModRows, intersect.size()); - } - } else { - numModRows = Math.max(numModRows, mcd.rowsModified.original.size()); - } - } - this.numClientModRows = numModRows; - + // precompute the included rows / offsets and viewport removed rows if (isFullSubscription) { clientRemovedRows = null; // we'll send full subscriptions the full removed set @@ -341,6 +311,7 @@ public SubView(final BarrageSubscriptionOptions options, // any pre-existing rows that are no longer in the viewport also need to be removed final WritableRowSet rowsToRetain; if (isSnapshot) { + // for a snapshot, the goal is to calculate which rows to remove due to viewport changes rowsToRetain = toClose.add(keyspaceViewport.copy()); } else { rowsToRetain = toClose.add(keyspaceViewport.minus(rowsAdded.original)); @@ -351,8 +322,39 @@ public SubView(final BarrageSubscriptionOptions options, clientRemovedRows = keyspaceViewportPrev.invert(noLongerExistingRows); } } - this.numClientIncludedRows = clientIncludedRowOffsets.size(); + + // precompute the modified column indexes, and calculate total rows needed + if (keyspaceViewport != null) { + this.clientModdedRows = new WritableRowSet[modColumnData.length]; + this.clientModdedRowOffsets = new WritableRowSet[modColumnData.length]; + } else { + this.clientModdedRows = null; + this.clientModdedRowOffsets = null; + } + + long numModRows = 0; + for (int ii = 0; ii < modColumnData.length; ++ii) { + final ModColumnGenerator mcd = modColumnData[ii]; + + if (keyspaceViewport != null) { + try (final WritableRowSet intersect = keyspaceViewport.intersect(mcd.rowsModified.original)) { + if (isFullSubscription) { + clientModdedRows[ii] = intersect.copy(); + } else { + // some rows may be marked both as included and modified; viewport clients must be sent + // the full row data for these rows, so we do not also need to send them as modified + intersect.remove(rowsIncluded.original); + clientModdedRows[ii] = keyspaceViewport.invert(intersect); + } + clientModdedRowOffsets[ii] = mcd.rowsModified.original.invert(intersect); + numModRows = Math.max(numModRows, clientModdedRows[ii].size()); + } + } else { + numModRows = Math.max(numModRows, mcd.rowsModified.original.size()); + } + } + this.numClientModRows = numModRows; } @Override diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java index fb29c9e36bb..001af543759 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageRedirectedTable.java @@ -265,9 +265,6 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC return coalescer; } - if (!isFullSubscription) { - totalMods.remove(update.rowsIncluded); - } final TableUpdate downstream = new TableUpdateImpl( update.rowsAdded, update.rowsRemoved, totalMods, updateShiftData, modifiedColumnSet); From d568eb7cfc575d5c3d98d921f4bef0051e2973c0 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Thu, 14 Nov 2024 13:58:47 -0700 Subject: [PATCH 20/35] Minor cleanup from personal review --- .../barrage/BarrageStreamGeneratorImpl.java | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 1c59a38c054..bac680947b5 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -322,39 +322,40 @@ public SubView(final BarrageSubscriptionOptions options, clientRemovedRows = keyspaceViewportPrev.invert(noLongerExistingRows); } } - this.numClientIncludedRows = clientIncludedRowOffsets.size(); + numClientIncludedRows = clientIncludedRowOffsets.size(); // precompute the modified column indexes, and calculate total rows needed if (keyspaceViewport != null) { - this.clientModdedRows = new WritableRowSet[modColumnData.length]; - this.clientModdedRowOffsets = new WritableRowSet[modColumnData.length]; + clientModdedRows = new WritableRowSet[modColumnData.length]; + clientModdedRowOffsets = new WritableRowSet[modColumnData.length]; } else { - this.clientModdedRows = null; - this.clientModdedRowOffsets = null; + clientModdedRows = null; + clientModdedRowOffsets = null; } long numModRows = 0; for (int ii = 0; ii < modColumnData.length; ++ii) { final ModColumnGenerator mcd = modColumnData[ii]; - if (keyspaceViewport != null) { - try (final WritableRowSet intersect = keyspaceViewport.intersect(mcd.rowsModified.original)) { - if (isFullSubscription) { - clientModdedRows[ii] = intersect.copy(); - } else { - // some rows may be marked both as included and modified; viewport clients must be sent - // the full row data for these rows, so we do not also need to send them as modified - intersect.remove(rowsIncluded.original); - clientModdedRows[ii] = keyspaceViewport.invert(intersect); - } - clientModdedRowOffsets[ii] = mcd.rowsModified.original.invert(intersect); - numModRows = Math.max(numModRows, clientModdedRows[ii].size()); - } - } else { + if (keyspaceViewport == null) { numModRows = Math.max(numModRows, mcd.rowsModified.original.size()); + continue; + } + + try (final WritableRowSet intersect = keyspaceViewport.intersect(mcd.rowsModified.original)) { + if (isFullSubscription) { + clientModdedRows[ii] = intersect.copy(); + } else { + // some rows may be marked both as included and modified; viewport clients must be sent + // the full row data for these rows, so we do not also need to send them as modified + intersect.remove(rowsIncluded.original); + clientModdedRows[ii] = keyspaceViewport.invert(intersect); + } + clientModdedRowOffsets[ii] = mcd.rowsModified.original.invert(intersect); + numModRows = Math.max(numModRows, clientModdedRows[ii].size()); } } - this.numClientModRows = numModRows; + numClientModRows = numModRows; } @Override From 6653ca680f2832980416c1d3921479071f3581f6 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Thu, 14 Nov 2024 14:45:16 -0700 Subject: [PATCH 21/35] Ryan's feedback latest round. --- .../extensions/barrage/BarrageStreamGeneratorImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index bac680947b5..d7cef7ad5a8 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -343,16 +343,16 @@ public SubView(final BarrageSubscriptionOptions options, } try (final WritableRowSet intersect = keyspaceViewport.intersect(mcd.rowsModified.original)) { + // some rows may be marked both as included and modified; viewport clients must be sent + // the full row data for these rows, so we do not also need to send them as modified + intersect.remove(rowsIncluded.original); if (isFullSubscription) { clientModdedRows[ii] = intersect.copy(); } else { - // some rows may be marked both as included and modified; viewport clients must be sent - // the full row data for these rows, so we do not also need to send them as modified - intersect.remove(rowsIncluded.original); clientModdedRows[ii] = keyspaceViewport.invert(intersect); } clientModdedRowOffsets[ii] = mcd.rowsModified.original.invert(intersect); - numModRows = Math.max(numModRows, clientModdedRows[ii].size()); + numModRows = Math.max(numModRows, intersect.size()); } } numClientModRows = numModRows; From 44cdf939639fbe6d322db979976866a8c1e2e6c1 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 07:46:48 -0700 Subject: [PATCH 22/35] jsAPI mostly complete; looking for tree table issue --- .../table/impl/remote/ConstructSnapshot.java | 1 + .../table/impl/util/BarrageMessage.java | 1 + .../barrage/BarrageSnapshotOptions.java | 4 +- .../barrage/BarrageStreamGeneratorImpl.java | 1 + .../barrage/BarrageSubscriptionOptions.java | 4 +- .../barrage/util/BarrageStreamReader.java | 1 + .../barrage/util/StreamReaderOptions.java | 2 +- gradle/libs.versions.toml | 2 +- .../client/api/barrage/WebBarrageMessage.java | 1 + .../api/barrage/WebBarrageStreamReader.java | 5 +- .../barrage/data/WebBarrageSubscription.java | 104 +++++++++++++++++- .../api/barrage/data/WebByteColumnData.java | 67 +++++++++++ .../api/barrage/data/WebCharColumnData.java | 67 +++++++++++ .../api/barrage/data/WebColumnData.java | 15 ++- .../api/barrage/data/WebDoubleColumnData.java | 67 +++++++++++ .../api/barrage/data/WebFloatColumnData.java | 67 +++++++++++ .../api/barrage/data/WebIntColumnData.java | 67 +++++++++++ .../api/barrage/data/WebLongColumnData.java | 67 +++++++++++ .../api/barrage/data/WebObjectColumnData.java | 68 ++++++++++++ .../api/barrage/data/WebShortColumnData.java | 67 +++++++++++ .../AbstractTableSubscription.java | 26 +++-- .../api/subscription/SubscriptionType.java | 13 +++ .../api/subscription/TableSubscription.java | 2 +- .../TableViewportSubscription.java | 11 +- .../web/client/api/tree/JsTreeTable.java | 3 +- .../deephaven/web/shared/data/RangeSet.java | 25 +++++ 26 files changed, 729 insertions(+), 29 deletions(-) create mode 100644 web/client-api/src/main/java/io/deephaven/web/client/api/subscription/SubscriptionType.java diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java index e690cd052dc..d2f8468c089 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java @@ -1314,6 +1314,7 @@ private static boolean snapshotAllTable( @Nullable final RowSet keysToSnapshot) { snapshot.rowsAdded = (usePrev ? table.getRowSet().prev() : table.getRowSet()).copy(); + snapshot.tableSize = snapshot.rowsAdded.size(); snapshot.rowsRemoved = RowSetFactory.empty(); snapshot.addColumnData = new BarrageMessage.AddColumnData[table.getColumnSources().size()]; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java index a61ed5b7f03..6b60a57310e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java @@ -46,6 +46,7 @@ public static class AddColumnData { public long firstSeq = -1; public long lastSeq = -1; public long step = -1; + public long tableSize = -1; public boolean isSnapshot; public RowSet snapshotRowSet; diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java index 005601ba355..7843fe4eed9 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSnapshotOptions.java @@ -54,7 +54,7 @@ public int maxMessageSize() { @Override @Default - public int previewListLengthLimit() { + public long previewListLengthLimit() { return 0; } @@ -111,7 +111,7 @@ default Builder columnConversionMode(ColumnConversionMode columnConversionMode) * @param previewListLengthLimit the magnitude of the number of elements to include in a preview list * @return this builder */ - Builder previewListLengthLimit(int previewListLengthLimit); + Builder previewListLengthLimit(long previewListLengthLimit); /** * @return a new BarrageSnapshotOptions instance diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index d7cef7ad5a8..827ed68eed0 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -520,6 +520,7 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { BarrageUpdateMetadata.addAddedRowsIncluded(metadata, addedRowsIncludedOffset); BarrageUpdateMetadata.addModColumnNodes(metadata, nodesOffset); BarrageUpdateMetadata.addEffectiveReverseViewport(metadata, reverseViewport); + BarrageUpdateMetadata.addTableSize(metadata, message.tableSize); metadata.finish(BarrageUpdateMetadata.endBarrageUpdateMetadata(metadata)); final FlatBufferBuilder header = new FlatBufferBuilder(); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java index 8c45da1e10b..f26803283ef 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageSubscriptionOptions.java @@ -86,7 +86,7 @@ public int maxMessageSize() { @Override @Default - public int previewListLengthLimit() { + public long previewListLengthLimit() { return 0; } @@ -162,7 +162,7 @@ default Builder columnConversionMode(ColumnConversionMode columnConversionMode) * @param previewListLengthLimit the magnitude of the number of elements to include in a preview list * @return this builder */ - Builder previewListLengthLimit(int previewListLengthLimit); + Builder previewListLengthLimit(long previewListLengthLimit); /** * @return a new BarrageSubscriptionOptions instance diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index f78ce861495..0abde8fd91e 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -133,6 +133,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.firstSeq = metadata.firstSeq(); msg.lastSeq = metadata.lastSeq(); + msg.tableSize = metadata.tableSize(); msg.rowsAdded = extractIndex(metadata.addedRowsAsByteBuffer()); msg.rowsRemoved = extractIndex(metadata.removedRowsAsByteBuffer()); final ByteBuffer shiftData = metadata.shiftDataAsByteBuffer(); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java index 45f6bf49ed3..3ea7291a0e1 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/StreamReaderOptions.java @@ -60,7 +60,7 @@ default boolean columnsAsList() { * @return the maximum length of any list / array to encode; zero means no limit; negative values indicate to treat * the limit as a tail instead of a head */ - default int previewListLengthLimit() { + default long previewListLengthLimit() { return 0; } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6c91cb8fc32..1c78b148213 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,7 +17,7 @@ commons-text = "1.12.0" confluent = "7.6.0" confluent-kafka-clients = "7.6.0-ccs" dagger = "2.52" -deephaven-barrage = "0.7.0" +deephaven-barrage = "0.7.2" deephaven-csv = "0.15.0" deephaven-hash = "0.1.0" deephaven-suan-shu = "0.1.1" diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageMessage.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageMessage.java index 1b26f2ccadb..d0e7b87ccda 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageMessage.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageMessage.java @@ -30,6 +30,7 @@ public static class AddColumnData { public long firstSeq = -1; public long lastSeq = -1; public long step = -1; + public long tableSize = -1; public boolean isSnapshot; public RangeSet snapshotRowSet; diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageStreamReader.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageStreamReader.java index 6bf5196034e..21730296f13 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageStreamReader.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/WebBarrageStreamReader.java @@ -11,6 +11,7 @@ import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.RowSetShiftData; import io.deephaven.extensions.barrage.chunk.ChunkInputStreamGenerator; import io.deephaven.extensions.barrage.chunk.ChunkReader; import io.deephaven.extensions.barrage.util.FlatBufferIteratorAdapter; @@ -103,9 +104,11 @@ public WebBarrageMessage parseFrom( msg.firstSeq = metadata.firstSeq(); msg.lastSeq = metadata.lastSeq(); + msg.tableSize = metadata.tableSize(); msg.rowsAdded = extractIndex(metadata.addedRowsAsByteBuffer()); msg.rowsRemoved = extractIndex(metadata.removedRowsAsByteBuffer()); - msg.shifted = extractIndexShiftData(metadata.shiftDataAsByteBuffer()); + final ByteBuffer shiftData = metadata.shiftDataAsByteBuffer(); + msg.shifted = shiftData != null ? extractIndexShiftData(shiftData) : new ShiftedRange[0]; final ByteBuffer rowsIncluded = metadata.addedRowsIncludedAsByteBuffer(); msg.rowsIncluded = rowsIncluded != null ? extractIndex(rowsIncluded) : msg.rowsAdded; diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java index b12f14ac2cb..97ba53ed071 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java @@ -8,6 +8,7 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.web.client.api.barrage.WebBarrageMessage; import io.deephaven.web.client.api.barrage.def.InitialTableDefinition; +import io.deephaven.web.client.api.subscription.SubscriptionType; import io.deephaven.web.client.state.ClientTableState; import io.deephaven.web.shared.data.Range; import io.deephaven.web.shared.data.RangeSet; @@ -37,8 +38,11 @@ public abstract class WebBarrageSubscription { public static final int MAX_MESSAGE_SIZE = 10_000_000; public static final int BATCH_SIZE = 100_000; - public static WebBarrageSubscription subscribe(ClientTableState cts, ViewportChangedHandler viewportChangedHandler, - DataChangedHandler dataChangedHandler) { + public static WebBarrageSubscription subscribe( + final SubscriptionType subscriptionType, + final ClientTableState cts, + final ViewportChangedHandler viewportChangedHandler, + final DataChangedHandler dataChangedHandler) { WebColumnData[] dataSinks = new WebColumnData[cts.columnTypes().length]; ChunkType[] chunkTypes = cts.chunkTypes(); @@ -75,8 +79,11 @@ public static WebBarrageSubscription subscribe(ClientTableState cts, ViewportCha if (cts.getTableDef().getAttributes().isBlinkTable()) { return new BlinkImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); + } else if (subscriptionType == SubscriptionType.FULL_SUBSCRIPTION) { + return new RedirectedImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); + } else { + return new ViewportImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); } - return new RedirectedImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); } public interface ViewportChangedHandler { @@ -109,6 +116,13 @@ protected WebBarrageSubscription(ClientTableState state, ViewportChangedHandler public abstract void applyUpdates(WebBarrageMessage message); + /** + * @return the current size of the table + */ + public long getCurrentSize() { + return currentRowSet.size(); + } + protected void updateServerViewport(RangeSet viewport, BitSet columns, boolean reverseViewport) { serverViewport = viewport; serverColumns = columns == null || columns.cardinality() == numColumns() ? null : columns; @@ -447,6 +461,90 @@ private void freeRows(RangeSet removed) { } } + public static class ViewportImpl extends WebBarrageSubscription { + private long lastTableSize = -1; + + public ViewportImpl(ClientTableState state, ViewportChangedHandler viewportChangedHandler, + DataChangedHandler dataChangedHandler, WebColumnData[] dataSinks) { + super(state, viewportChangedHandler, dataChangedHandler, dataSinks); + } + + @Override + public long getCurrentSize() { + return lastTableSize; + } + + @Override + public RangeSet getCurrentRowSet() { + return RangeSet.ofRange(0, lastTableSize - 1); + } + + @Override + public void applyUpdates(WebBarrageMessage message) { + lastTableSize = message.tableSize; + + if (message.isSnapshot) { + updateServerViewport(message.snapshotRowSet, message.snapshotColumns, message.snapshotRowSetIsReversed); + viewportChangedHandler.onServerViewportChanged(serverViewport, serverColumns, serverReverseViewport); + } + + // Update the currentRowSet; we're guaranteed to be flat + final long prevSize = currentRowSet.size(); + final long newSize = prevSize - message.rowsRemoved.size() + message.rowsAdded.size(); + if (prevSize < newSize) { + currentRowSet.addRange(new Range(prevSize, newSize - 1)); + } else if (prevSize > newSize) { + currentRowSet.removeRange(new Range(newSize, prevSize - 1)); + } + + if (!message.rowsAdded.isEmpty() || !message.rowsRemoved.isEmpty()) { + for (int ii = 0; ii < message.addColumnData.length; ii++) { + if (!isSubscribedColumn(ii)) { + continue; + } + + final WebBarrageMessage.AddColumnData column = message.addColumnData[ii]; + for (int j = 0; j < column.data.size(); j++) { + destSources[ii].applyUpdate(column.data, message.rowsAdded, message.rowsRemoved); + } + } + } + + final BitSet modifiedColumnSet = new BitSet(numColumns()); + for (int ii = 0; ii < message.modColumnData.length; ii++) { + WebBarrageMessage.ModColumnData column = message.modColumnData[ii]; + if (column.rowsModified.isEmpty()) { + continue; + } + + modifiedColumnSet.set(ii); + + for (int j = 0; j < column.data.size(); j++) { + Chunk chunk = column.data.get(j); + destSources[ii].fillChunk(chunk, column.rowsModified.indexIterator()); + } + } + + state.setSize(message.tableSize); + dataChangedHandler.onDataChanged( + RangeSet.ofRange(0, currentRowSet.size()), + RangeSet.ofRange(0, prevSize), + RangeSet.empty(), new ShiftedRange[0], modifiedColumnSet); + } + + @Override + public Any getData(long key, int col) { + if (!isSubscribedColumn(col)) { + throw new NoSuchElementException("No column at index " + col); + } + long pos = serverViewport.find(key); + if (pos < 0) { + return null; + } + return this.destSources[col].get(pos); + } + } + /** * Helper to avoid appending many times when modifying indexes. The append() method should be called for each key * in order to ensure that addRange/removeRange isn't called excessively. When no more items will be added, diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebByteColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebByteColumnData.java index 4f0593d05a9..5a9dd8aaa94 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebByteColumnData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebByteColumnData.java @@ -7,14 +7,23 @@ // @formatter:off package io.deephaven.web.client.api.barrage.data; +import elemental2.core.JsArray; import io.deephaven.chunk.ByteChunk; import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.attributes.Values; import io.deephaven.util.QueryConstants; +import io.deephaven.web.shared.data.Range; +import io.deephaven.web.shared.data.RangeSet; +import jsinterop.base.Any; import jsinterop.base.Js; +import java.util.Iterator; +import java.util.List; import java.util.PrimitiveIterator; public class WebByteColumnData extends WebColumnData { + private JsArray tmpStorage; + @Override public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { ByteChunk byteChunk = data.asByteChunk(); @@ -24,4 +33,62 @@ public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_BYTE ? null : Js.asAny(value)); } } + + @Override + public void applyUpdate( + final List> data, + final RangeSet added, + final RangeSet removed) { + // ensure tmpStorage exists + if (tmpStorage == null) { + tmpStorage = new JsArray<>(); + } + final int newLength = (int) (length - removed.size() + added.size()); + + int destOffset = 0; + int retainSourceOffset = 0; + int chunkSourceOffset = 0; + final Iterator addIter = added.rangeIterator(); + final Iterator removeIter = removed.rangeIterator(); + final Iterator> dataIter = data.iterator(); + + Range nextAdd = addIter.hasNext() ? addIter.next() : null; + Range nextRemove = removeIter.hasNext() ? removeIter.next() : null; + ByteChunk byteChunk = dataIter.hasNext() ? dataIter.next().asByteChunk() : null; + while (destOffset < newLength) { + if (nextRemove != null && nextRemove.getFirst() == retainSourceOffset) { + // skip the range from the source chunk + retainSourceOffset += (int) nextRemove.size(); + nextRemove = removeIter.hasNext() ? removeIter.next() : null; + } else if (nextAdd != null && nextAdd.getFirst() == destOffset) { + // copy the range from the source chunk + long size = nextAdd.size(); + for (long ii = 0; ii < size; ++ii) { + while (byteChunk != null && chunkSourceOffset == byteChunk.size()) { + byteChunk = dataIter.hasNext() ? dataIter.next().asByteChunk() : null; + chunkSourceOffset = 0; + } + assert byteChunk != null; + byte value = byteChunk.get(chunkSourceOffset++); + tmpStorage.setAt(destOffset++, value == QueryConstants.NULL_BYTE ? null : Js.asAny(value)); + } + nextAdd = addIter.hasNext() ? addIter.next() : null; + } else { + // copy the range from the source chunk + long size = (nextRemove == null ? length : nextRemove.getFirst()) - retainSourceOffset; + if (nextAdd != null) { + size = Math.min(size, nextAdd.getFirst() - destOffset); + } + for (long ii = 0; ii < size; ++ii) { + tmpStorage.setAt(destOffset++, arr.getAt(retainSourceOffset++)); + } + } + } + + // swap arrays to avoid copying and garbage collection + JsArray tmp = arr; + arr = tmpStorage; + tmpStorage = tmp; + length = newLength; + } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebCharColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebCharColumnData.java index dfe561d90df..5228b45dee9 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebCharColumnData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebCharColumnData.java @@ -3,14 +3,23 @@ // package io.deephaven.web.client.api.barrage.data; +import elemental2.core.JsArray; import io.deephaven.chunk.CharChunk; import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.attributes.Values; import io.deephaven.util.QueryConstants; +import io.deephaven.web.shared.data.Range; +import io.deephaven.web.shared.data.RangeSet; +import jsinterop.base.Any; import jsinterop.base.Js; +import java.util.Iterator; +import java.util.List; import java.util.PrimitiveIterator; public class WebCharColumnData extends WebColumnData { + private JsArray tmpStorage; + @Override public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { CharChunk charChunk = data.asCharChunk(); @@ -20,4 +29,62 @@ public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_CHAR ? null : Js.asAny(value)); } } + + @Override + public void applyUpdate( + final List> data, + final RangeSet added, + final RangeSet removed) { + // ensure tmpStorage exists + if (tmpStorage == null) { + tmpStorage = new JsArray<>(); + } + final int newLength = (int) (length - removed.size() + added.size()); + + int destOffset = 0; + int retainSourceOffset = 0; + int chunkSourceOffset = 0; + final Iterator addIter = added.rangeIterator(); + final Iterator removeIter = removed.rangeIterator(); + final Iterator> dataIter = data.iterator(); + + Range nextAdd = addIter.hasNext() ? addIter.next() : null; + Range nextRemove = removeIter.hasNext() ? removeIter.next() : null; + CharChunk charChunk = dataIter.hasNext() ? dataIter.next().asCharChunk() : null; + while (destOffset < newLength) { + if (nextRemove != null && nextRemove.getFirst() == retainSourceOffset) { + // skip the range from the source chunk + retainSourceOffset += (int) nextRemove.size(); + nextRemove = removeIter.hasNext() ? removeIter.next() : null; + } else if (nextAdd != null && nextAdd.getFirst() == destOffset) { + // copy the range from the source chunk + long size = nextAdd.size(); + for (long ii = 0; ii < size; ++ii) { + while (charChunk != null && chunkSourceOffset == charChunk.size()) { + charChunk = dataIter.hasNext() ? dataIter.next().asCharChunk() : null; + chunkSourceOffset = 0; + } + assert charChunk != null; + char value = charChunk.get(chunkSourceOffset++); + tmpStorage.setAt(destOffset++, value == QueryConstants.NULL_CHAR ? null : Js.asAny(value)); + } + nextAdd = addIter.hasNext() ? addIter.next() : null; + } else { + // copy the range from the source chunk + long size = (nextRemove == null ? length : nextRemove.getFirst()) - retainSourceOffset; + if (nextAdd != null) { + size = Math.min(size, nextAdd.getFirst() - destOffset); + } + for (long ii = 0; ii < size; ++ii) { + tmpStorage.setAt(destOffset++, arr.getAt(retainSourceOffset++)); + } + } + } + + // swap arrays to avoid copying and garbage collection + JsArray tmp = arr; + arr = tmpStorage; + tmpStorage = tmp; + length = newLength; + } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebColumnData.java index 3c8fee323b4..3a0a0190c7f 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebColumnData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebColumnData.java @@ -5,18 +5,31 @@ import elemental2.core.JsArray; import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.web.shared.data.RangeSet; import jsinterop.base.Any; +import java.util.List; import java.util.PrimitiveIterator; /** * Holds data from or intended for web clients, normalizing over nulls, with helpers to handle typed chunks. */ public abstract class WebColumnData { - protected final JsArray arr = new JsArray<>(); + protected int length = 0; + protected JsArray arr = new JsArray<>(); public abstract void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator); + /** + * Apply a viewport update directly to this column data. + * + * @param data the data source for added rows + * @param added rows that are new in a post-update position-space + * @param removed rows that no longer exist in a pre-update position-space + */ + public abstract void applyUpdate(List> data, RangeSet added, RangeSet removed); + public void ensureCapacity(long size) { // Current impl does nothing, js arrays don't behave better when told the size up front } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebDoubleColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebDoubleColumnData.java index d2c8e764ce7..01239a70d4c 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebDoubleColumnData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebDoubleColumnData.java @@ -7,14 +7,23 @@ // @formatter:off package io.deephaven.web.client.api.barrage.data; +import elemental2.core.JsArray; import io.deephaven.chunk.DoubleChunk; import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.attributes.Values; import io.deephaven.util.QueryConstants; +import io.deephaven.web.shared.data.Range; +import io.deephaven.web.shared.data.RangeSet; +import jsinterop.base.Any; import jsinterop.base.Js; +import java.util.Iterator; +import java.util.List; import java.util.PrimitiveIterator; public class WebDoubleColumnData extends WebColumnData { + private JsArray tmpStorage; + @Override public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { DoubleChunk doubleChunk = data.asDoubleChunk(); @@ -24,4 +33,62 @@ public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_DOUBLE ? null : Js.asAny(value)); } } + + @Override + public void applyUpdate( + final List> data, + final RangeSet added, + final RangeSet removed) { + // ensure tmpStorage exists + if (tmpStorage == null) { + tmpStorage = new JsArray<>(); + } + final int newLength = (int) (length - removed.size() + added.size()); + + int destOffset = 0; + int retainSourceOffset = 0; + int chunkSourceOffset = 0; + final Iterator addIter = added.rangeIterator(); + final Iterator removeIter = removed.rangeIterator(); + final Iterator> dataIter = data.iterator(); + + Range nextAdd = addIter.hasNext() ? addIter.next() : null; + Range nextRemove = removeIter.hasNext() ? removeIter.next() : null; + DoubleChunk doubleChunk = dataIter.hasNext() ? dataIter.next().asDoubleChunk() : null; + while (destOffset < newLength) { + if (nextRemove != null && nextRemove.getFirst() == retainSourceOffset) { + // skip the range from the source chunk + retainSourceOffset += (int) nextRemove.size(); + nextRemove = removeIter.hasNext() ? removeIter.next() : null; + } else if (nextAdd != null && nextAdd.getFirst() == destOffset) { + // copy the range from the source chunk + long size = nextAdd.size(); + for (long ii = 0; ii < size; ++ii) { + while (doubleChunk != null && chunkSourceOffset == doubleChunk.size()) { + doubleChunk = dataIter.hasNext() ? dataIter.next().asDoubleChunk() : null; + chunkSourceOffset = 0; + } + assert doubleChunk != null; + double value = doubleChunk.get(chunkSourceOffset++); + tmpStorage.setAt(destOffset++, value == QueryConstants.NULL_DOUBLE ? null : Js.asAny(value)); + } + nextAdd = addIter.hasNext() ? addIter.next() : null; + } else { + // copy the range from the source chunk + long size = (nextRemove == null ? length : nextRemove.getFirst()) - retainSourceOffset; + if (nextAdd != null) { + size = Math.min(size, nextAdd.getFirst() - destOffset); + } + for (long ii = 0; ii < size; ++ii) { + tmpStorage.setAt(destOffset++, arr.getAt(retainSourceOffset++)); + } + } + } + + // swap arrays to avoid copying and garbage collection + JsArray tmp = arr; + arr = tmpStorage; + tmpStorage = tmp; + length = newLength; + } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebFloatColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebFloatColumnData.java index a624affbda6..708e39f3f26 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebFloatColumnData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebFloatColumnData.java @@ -7,14 +7,23 @@ // @formatter:off package io.deephaven.web.client.api.barrage.data; +import elemental2.core.JsArray; import io.deephaven.chunk.FloatChunk; import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.attributes.Values; import io.deephaven.util.QueryConstants; +import io.deephaven.web.shared.data.Range; +import io.deephaven.web.shared.data.RangeSet; +import jsinterop.base.Any; import jsinterop.base.Js; +import java.util.Iterator; +import java.util.List; import java.util.PrimitiveIterator; public class WebFloatColumnData extends WebColumnData { + private JsArray tmpStorage; + @Override public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { FloatChunk floatChunk = data.asFloatChunk(); @@ -24,4 +33,62 @@ public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_FLOAT ? null : Js.asAny(value)); } } + + @Override + public void applyUpdate( + final List> data, + final RangeSet added, + final RangeSet removed) { + // ensure tmpStorage exists + if (tmpStorage == null) { + tmpStorage = new JsArray<>(); + } + final int newLength = (int) (length - removed.size() + added.size()); + + int destOffset = 0; + int retainSourceOffset = 0; + int chunkSourceOffset = 0; + final Iterator addIter = added.rangeIterator(); + final Iterator removeIter = removed.rangeIterator(); + final Iterator> dataIter = data.iterator(); + + Range nextAdd = addIter.hasNext() ? addIter.next() : null; + Range nextRemove = removeIter.hasNext() ? removeIter.next() : null; + FloatChunk floatChunk = dataIter.hasNext() ? dataIter.next().asFloatChunk() : null; + while (destOffset < newLength) { + if (nextRemove != null && nextRemove.getFirst() == retainSourceOffset) { + // skip the range from the source chunk + retainSourceOffset += (int) nextRemove.size(); + nextRemove = removeIter.hasNext() ? removeIter.next() : null; + } else if (nextAdd != null && nextAdd.getFirst() == destOffset) { + // copy the range from the source chunk + long size = nextAdd.size(); + for (long ii = 0; ii < size; ++ii) { + while (floatChunk != null && chunkSourceOffset == floatChunk.size()) { + floatChunk = dataIter.hasNext() ? dataIter.next().asFloatChunk() : null; + chunkSourceOffset = 0; + } + assert floatChunk != null; + float value = floatChunk.get(chunkSourceOffset++); + tmpStorage.setAt(destOffset++, value == QueryConstants.NULL_FLOAT ? null : Js.asAny(value)); + } + nextAdd = addIter.hasNext() ? addIter.next() : null; + } else { + // copy the range from the source chunk + long size = (nextRemove == null ? length : nextRemove.getFirst()) - retainSourceOffset; + if (nextAdd != null) { + size = Math.min(size, nextAdd.getFirst() - destOffset); + } + for (long ii = 0; ii < size; ++ii) { + tmpStorage.setAt(destOffset++, arr.getAt(retainSourceOffset++)); + } + } + } + + // swap arrays to avoid copying and garbage collection + JsArray tmp = arr; + arr = tmpStorage; + tmpStorage = tmp; + length = newLength; + } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebIntColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebIntColumnData.java index 996cf43c6a8..2606b6f25de 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebIntColumnData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebIntColumnData.java @@ -7,14 +7,23 @@ // @formatter:off package io.deephaven.web.client.api.barrage.data; +import elemental2.core.JsArray; import io.deephaven.chunk.IntChunk; import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.attributes.Values; import io.deephaven.util.QueryConstants; +import io.deephaven.web.shared.data.Range; +import io.deephaven.web.shared.data.RangeSet; +import jsinterop.base.Any; import jsinterop.base.Js; +import java.util.Iterator; +import java.util.List; import java.util.PrimitiveIterator; public class WebIntColumnData extends WebColumnData { + private JsArray tmpStorage; + @Override public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { IntChunk intChunk = data.asIntChunk(); @@ -24,4 +33,62 @@ public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_INT ? null : Js.asAny(value)); } } + + @Override + public void applyUpdate( + final List> data, + final RangeSet added, + final RangeSet removed) { + // ensure tmpStorage exists + if (tmpStorage == null) { + tmpStorage = new JsArray<>(); + } + final int newLength = (int) (length - removed.size() + added.size()); + + int destOffset = 0; + int retainSourceOffset = 0; + int chunkSourceOffset = 0; + final Iterator addIter = added.rangeIterator(); + final Iterator removeIter = removed.rangeIterator(); + final Iterator> dataIter = data.iterator(); + + Range nextAdd = addIter.hasNext() ? addIter.next() : null; + Range nextRemove = removeIter.hasNext() ? removeIter.next() : null; + IntChunk intChunk = dataIter.hasNext() ? dataIter.next().asIntChunk() : null; + while (destOffset < newLength) { + if (nextRemove != null && nextRemove.getFirst() == retainSourceOffset) { + // skip the range from the source chunk + retainSourceOffset += (int) nextRemove.size(); + nextRemove = removeIter.hasNext() ? removeIter.next() : null; + } else if (nextAdd != null && nextAdd.getFirst() == destOffset) { + // copy the range from the source chunk + long size = nextAdd.size(); + for (long ii = 0; ii < size; ++ii) { + while (intChunk != null && chunkSourceOffset == intChunk.size()) { + intChunk = dataIter.hasNext() ? dataIter.next().asIntChunk() : null; + chunkSourceOffset = 0; + } + assert intChunk != null; + int value = intChunk.get(chunkSourceOffset++); + tmpStorage.setAt(destOffset++, value == QueryConstants.NULL_INT ? null : Js.asAny(value)); + } + nextAdd = addIter.hasNext() ? addIter.next() : null; + } else { + // copy the range from the source chunk + long size = (nextRemove == null ? length : nextRemove.getFirst()) - retainSourceOffset; + if (nextAdd != null) { + size = Math.min(size, nextAdd.getFirst() - destOffset); + } + for (long ii = 0; ii < size; ++ii) { + tmpStorage.setAt(destOffset++, arr.getAt(retainSourceOffset++)); + } + } + } + + // swap arrays to avoid copying and garbage collection + JsArray tmp = arr; + arr = tmpStorage; + tmpStorage = tmp; + length = newLength; + } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebLongColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebLongColumnData.java index 080c05e6034..779fe7d208b 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebLongColumnData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebLongColumnData.java @@ -7,14 +7,23 @@ // @formatter:off package io.deephaven.web.client.api.barrage.data; +import elemental2.core.JsArray; import io.deephaven.chunk.LongChunk; import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.attributes.Values; import io.deephaven.util.QueryConstants; +import io.deephaven.web.shared.data.Range; +import io.deephaven.web.shared.data.RangeSet; +import jsinterop.base.Any; import jsinterop.base.Js; +import java.util.Iterator; +import java.util.List; import java.util.PrimitiveIterator; public class WebLongColumnData extends WebColumnData { + private JsArray tmpStorage; + @Override public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { LongChunk longChunk = data.asLongChunk(); @@ -24,4 +33,62 @@ public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_LONG ? null : Js.asAny(value)); } } + + @Override + public void applyUpdate( + final List> data, + final RangeSet added, + final RangeSet removed) { + // ensure tmpStorage exists + if (tmpStorage == null) { + tmpStorage = new JsArray<>(); + } + final int newLength = (int) (length - removed.size() + added.size()); + + int destOffset = 0; + int retainSourceOffset = 0; + int chunkSourceOffset = 0; + final Iterator addIter = added.rangeIterator(); + final Iterator removeIter = removed.rangeIterator(); + final Iterator> dataIter = data.iterator(); + + Range nextAdd = addIter.hasNext() ? addIter.next() : null; + Range nextRemove = removeIter.hasNext() ? removeIter.next() : null; + LongChunk longChunk = dataIter.hasNext() ? dataIter.next().asLongChunk() : null; + while (destOffset < newLength) { + if (nextRemove != null && nextRemove.getFirst() == retainSourceOffset) { + // skip the range from the source chunk + retainSourceOffset += (int) nextRemove.size(); + nextRemove = removeIter.hasNext() ? removeIter.next() : null; + } else if (nextAdd != null && nextAdd.getFirst() == destOffset) { + // copy the range from the source chunk + long size = nextAdd.size(); + for (long ii = 0; ii < size; ++ii) { + while (longChunk != null && chunkSourceOffset == longChunk.size()) { + longChunk = dataIter.hasNext() ? dataIter.next().asLongChunk() : null; + chunkSourceOffset = 0; + } + assert longChunk != null; + long value = longChunk.get(chunkSourceOffset++); + tmpStorage.setAt(destOffset++, value == QueryConstants.NULL_LONG ? null : Js.asAny(value)); + } + nextAdd = addIter.hasNext() ? addIter.next() : null; + } else { + // copy the range from the source chunk + long size = (nextRemove == null ? length : nextRemove.getFirst()) - retainSourceOffset; + if (nextAdd != null) { + size = Math.min(size, nextAdd.getFirst() - destOffset); + } + for (long ii = 0; ii < size; ++ii) { + tmpStorage.setAt(destOffset++, arr.getAt(retainSourceOffset++)); + } + } + } + + // swap arrays to avoid copying and garbage collection + JsArray tmp = arr; + arr = tmpStorage; + tmpStorage = tmp; + length = newLength; + } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebObjectColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebObjectColumnData.java index 251bca22e67..33c4817a857 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebObjectColumnData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebObjectColumnData.java @@ -3,13 +3,22 @@ // package io.deephaven.web.client.api.barrage.data; +import elemental2.core.JsArray; import io.deephaven.chunk.Chunk; import io.deephaven.chunk.ObjectChunk; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.web.shared.data.Range; +import io.deephaven.web.shared.data.RangeSet; +import jsinterop.base.Any; import jsinterop.base.Js; +import java.util.Iterator; +import java.util.List; import java.util.PrimitiveIterator; public class WebObjectColumnData extends WebColumnData { + private JsArray tmpStorage; + @Override public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { ObjectChunk objectChunk = data.asObjectChunk(); @@ -19,4 +28,63 @@ public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { arr.setAt((int) destIterator.nextLong(), Js.asAny(value)); } } + + + @Override + public void applyUpdate( + final List> data, + final RangeSet added, + final RangeSet removed) { + // ensure tmpStorage exists + if (tmpStorage == null) { + tmpStorage = new JsArray<>(); + } + final int newLength = (int) (length - removed.size() + added.size()); + + int destOffset = 0; + int retainSourceOffset = 0; + int chunkSourceOffset = 0; + final Iterator addIter = added.rangeIterator(); + final Iterator removeIter = removed.rangeIterator(); + final Iterator> dataIter = data.iterator(); + + Range nextAdd = addIter.hasNext() ? addIter.next() : null; + Range nextRemove = removeIter.hasNext() ? removeIter.next() : null; + ObjectChunk objectChunk = dataIter.hasNext() ? dataIter.next().asObjectChunk() : null; + while (destOffset < newLength) { + if (nextRemove != null && nextRemove.getFirst() == retainSourceOffset) { + // skip the range from the source chunk + retainSourceOffset += (int) nextRemove.size(); + nextRemove = removeIter.hasNext() ? removeIter.next() : null; + } else if (nextAdd != null && nextAdd.getFirst() == destOffset) { + // copy the range from the source chunk + long size = nextAdd.size(); + for (long ii = 0; ii < size; ++ii) { + while (objectChunk != null && chunkSourceOffset == objectChunk.size()) { + objectChunk = dataIter.hasNext() ? dataIter.next().asObjectChunk() : null; + chunkSourceOffset = 0; + } + assert objectChunk != null; + Object value = objectChunk.get(chunkSourceOffset++); + tmpStorage.setAt(destOffset++, Js.asAny(value)); + } + nextAdd = addIter.hasNext() ? addIter.next() : null; + } else { + // copy the range from the source chunk + long size = (nextRemove == null ? length : nextRemove.getFirst()) - retainSourceOffset; + if (nextAdd != null) { + size = Math.min(size, nextAdd.getFirst() - destOffset); + } + for (long ii = 0; ii < size; ++ii) { + tmpStorage.setAt(destOffset++, arr.getAt(retainSourceOffset++)); + } + } + } + + // swap arrays to avoid copying and garbage collection + JsArray tmp = arr; + arr = tmpStorage; + tmpStorage = tmp; + length = newLength; + } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebShortColumnData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebShortColumnData.java index 328a0f654a4..31e6fda181c 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebShortColumnData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebShortColumnData.java @@ -7,14 +7,23 @@ // @formatter:off package io.deephaven.web.client.api.barrage.data; +import elemental2.core.JsArray; import io.deephaven.chunk.ShortChunk; import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.attributes.Values; import io.deephaven.util.QueryConstants; +import io.deephaven.web.shared.data.Range; +import io.deephaven.web.shared.data.RangeSet; +import jsinterop.base.Any; import jsinterop.base.Js; +import java.util.Iterator; +import java.util.List; import java.util.PrimitiveIterator; public class WebShortColumnData extends WebColumnData { + private JsArray tmpStorage; + @Override public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { ShortChunk shortChunk = data.asShortChunk(); @@ -24,4 +33,62 @@ public void fillChunk(Chunk data, PrimitiveIterator.OfLong destIterator) { arr.setAt((int) destIterator.nextLong(), value == QueryConstants.NULL_SHORT ? null : Js.asAny(value)); } } + + @Override + public void applyUpdate( + final List> data, + final RangeSet added, + final RangeSet removed) { + // ensure tmpStorage exists + if (tmpStorage == null) { + tmpStorage = new JsArray<>(); + } + final int newLength = (int) (length - removed.size() + added.size()); + + int destOffset = 0; + int retainSourceOffset = 0; + int chunkSourceOffset = 0; + final Iterator addIter = added.rangeIterator(); + final Iterator removeIter = removed.rangeIterator(); + final Iterator> dataIter = data.iterator(); + + Range nextAdd = addIter.hasNext() ? addIter.next() : null; + Range nextRemove = removeIter.hasNext() ? removeIter.next() : null; + ShortChunk shortChunk = dataIter.hasNext() ? dataIter.next().asShortChunk() : null; + while (destOffset < newLength) { + if (nextRemove != null && nextRemove.getFirst() == retainSourceOffset) { + // skip the range from the source chunk + retainSourceOffset += (int) nextRemove.size(); + nextRemove = removeIter.hasNext() ? removeIter.next() : null; + } else if (nextAdd != null && nextAdd.getFirst() == destOffset) { + // copy the range from the source chunk + long size = nextAdd.size(); + for (long ii = 0; ii < size; ++ii) { + while (shortChunk != null && chunkSourceOffset == shortChunk.size()) { + shortChunk = dataIter.hasNext() ? dataIter.next().asShortChunk() : null; + chunkSourceOffset = 0; + } + assert shortChunk != null; + short value = shortChunk.get(chunkSourceOffset++); + tmpStorage.setAt(destOffset++, value == QueryConstants.NULL_SHORT ? null : Js.asAny(value)); + } + nextAdd = addIter.hasNext() ? addIter.next() : null; + } else { + // copy the range from the source chunk + long size = (nextRemove == null ? length : nextRemove.getFirst()) - retainSourceOffset; + if (nextAdd != null) { + size = Math.min(size, nextAdd.getFirst() - destOffset); + } + for (long ii = 0; ii < size; ++ii) { + tmpStorage.setAt(destOffset++, arr.getAt(retainSourceOffset++)); + } + } + } + + // swap arrays to avoid copying and garbage collection + JsArray tmp = arr; + arr = tmpStorage; + tmpStorage = tmp; + length = newLength; + } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java index e91ec5e4776..5165bf5fcc2 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java @@ -70,6 +70,7 @@ public enum Status { DONE; } + private final SubscriptionType subscriptionType; private final ClientTableState state; private final WorkerConnection connection; protected final int rowStyleColumn; @@ -86,8 +87,12 @@ public enum Status { private String failMsg; - public AbstractTableSubscription(ClientTableState state, WorkerConnection connection) { + protected AbstractTableSubscription( + SubscriptionType subscriptionType, + ClientTableState state, + WorkerConnection connection) { state.retain(this); + this.subscriptionType = subscriptionType; this.state = state; this.connection = connection; rowStyleColumn = state.getRowFormatColumn() == null ? TableData.NO_ROW_FORMAT_COLUMN @@ -111,16 +116,15 @@ protected void revive() { WebBarrageSubscription.DataChangedHandler dataChangedHandler = this::onDataChanged; status = Status.ACTIVE; - this.barrageSubscription = - WebBarrageSubscription.subscribe(state, viewportChangedHandler, dataChangedHandler); + this.barrageSubscription = WebBarrageSubscription.subscribe( + subscriptionType, state, viewportChangedHandler, dataChangedHandler); - doExchange = - connection.streamFactory().create( - headers -> connection.flightServiceClient().doExchange(headers), - (first, headers) -> connection.browserFlightServiceClient().openDoExchange(first, headers), - (next, headers, c) -> connection.browserFlightServiceClient().nextDoExchange(next, headers, - c::apply), - new FlightData()); + doExchange = connection.streamFactory().create( + headers -> connection.flightServiceClient().doExchange(headers), + (first, headers) -> connection.browserFlightServiceClient().openDoExchange(first, headers), + (next, headers, c) -> connection.browserFlightServiceClient().nextDoExchange(next, headers, + c::apply), + new FlightData()); doExchange.onData(this::onFlightData); doExchange.onEnd(this::onStreamEnd); @@ -214,7 +218,7 @@ protected boolean isSubscriptionReady() { public double size() { if (status == Status.ACTIVE) { - return barrageSubscription.getCurrentRowSet().size(); + return barrageSubscription.getCurrentSize(); } if (status == Status.DONE) { throw new IllegalStateException("Can't read size when already closed"); diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/SubscriptionType.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/SubscriptionType.java new file mode 100644 index 00000000000..64390243711 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/SubscriptionType.java @@ -0,0 +1,13 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.subscription; + +public enum SubscriptionType { + /** a subscription that will receive all updates to the table */ + FULL_SUBSCRIPTION, + /** a subscription that will receive updates only for the current viewport */ + VIEWPORT_SUBSCRIPTION, + /** a non-refreshing subscription that receives data once and then stops */ + SNAPSHOT +} diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableSubscription.java index e8399666d9c..1decd7c5713 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableSubscription.java @@ -31,7 +31,7 @@ public final class TableSubscription extends AbstractTableSubscription { @JsIgnore public TableSubscription(JsArray columns, JsTable existingTable, Double updateIntervalMs) { - super(existingTable.state(), existingTable.getConnection()); + super(SubscriptionType.FULL_SUBSCRIPTION, existingTable.state(), existingTable.getConnection()); this.columns = columns; this.updateIntervalMs = updateIntervalMs; } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java index f9bcdd76b30..c84d187acc0 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java @@ -11,7 +11,6 @@ import io.deephaven.barrage.flatbuf.BarrageMessageType; import io.deephaven.barrage.flatbuf.BarrageSnapshotRequest; import io.deephaven.extensions.barrage.BarrageSnapshotOptions; -import io.deephaven.extensions.barrage.ColumnConversionMode; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightData; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.config_pb.ConfigValue; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.table_pb.FlattenRequest; @@ -110,7 +109,7 @@ public static TableViewportSubscription make(double firstRow, double lastRow, Co } public TableViewportSubscription(ClientTableState state, WorkerConnection connection, JsTable existingTable) { - super(state, connection); + super(SubscriptionType.VIEWPORT_SUBSCRIPTION, state, connection); this.original = existingTable; initialState = existingTable.state(); @@ -343,9 +342,11 @@ public Promise snapshot(JsRangeSet rows, Column[] columns) { .previewListLengthLimit(0) .build(); - WebBarrageSubscription snapshot = - WebBarrageSubscription.subscribe(state(), (serverViewport1, serverColumns, serverReverseViewport) -> { - }, (rowsAdded, rowsRemoved, totalMods, shifted, modifiedColumnSet) -> { + WebBarrageSubscription snapshot = WebBarrageSubscription.subscribe( + SubscriptionType.SNAPSHOT, state(), + (serverViewport1, serverColumns, serverReverseViewport) -> { + }, + (rowsAdded, rowsRemoved, totalMods, shifted, modifiedColumnSet) -> { }); WebBarrageStreamReader reader = new WebBarrageStreamReader(); diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/tree/JsTreeTable.java b/web/client-api/src/main/java/io/deephaven/web/client/api/tree/JsTreeTable.java index 483d9465fc4..cc9ab1221ef 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/tree/JsTreeTable.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/tree/JsTreeTable.java @@ -34,6 +34,7 @@ import io.deephaven.web.client.api.impl.TicketAndPromise; import io.deephaven.web.client.api.lifecycle.HasLifecycle; import io.deephaven.web.client.api.subscription.AbstractTableSubscription; +import io.deephaven.web.client.api.subscription.SubscriptionType; import io.deephaven.web.client.api.widget.JsWidget; import io.deephaven.web.client.fu.JsItr; import io.deephaven.web.client.fu.JsLog; @@ -535,7 +536,7 @@ public Format getFormat(Column column) { private RangeSet serverViewport; public TreeSubscription(ClientTableState state, WorkerConnection connection) { - super(state, connection); + super(SubscriptionType.VIEWPORT_SUBSCRIPTION, state, connection); } @Override diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java index 4260069c524..8f56cfddc2f 100644 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java +++ b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java @@ -545,6 +545,31 @@ public boolean includesAnyOf(Range range) { return range.getFirst() <= target.getLast() && range.getLast() >= target.getFirst(); } + public long find(long key) { + long cnt = 0; + Iterator seenIterator = rangeIterator(); + + while (seenIterator.hasNext()) { + Range current = seenIterator.next(); + + if (key < current.getFirst()) { + // can't match at all, starts too early + return -cnt - 1; + } + + if (key > current.getLast()) { + // doesn't start until after the current range, so keep moving forward + cnt += current.size(); + continue; + } + if (key <= current.getLast()) { + // this is a match + return cnt + key - current.getFirst(); + } + } + return -cnt - 1; + } + @Override public String toString() { return "RangeSet{" + From d549d794df20affca14645ce47547da2383d1a79 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 13:11:50 -0700 Subject: [PATCH 23/35] Fixes for jsapi and HierarchicalTable --- .../table/impl/remote/ConstructSnapshot.java | 2 +- .../table/impl/util/BarrageMessage.java | 2 +- .../barrage/BarrageMessageProducer.java | 3 +- .../HierarchicalTableViewSubscription.java | 44 ++++++++++--------- .../barrage/data/WebBarrageSubscription.java | 35 ++++++++++----- 5 files changed, 52 insertions(+), 34 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java index d2f8468c089..88d05fdbf92 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java @@ -618,7 +618,7 @@ public static BarrageMessage constructBackplaneSnapshotInPositionSpace( final long clockStep = callDataSnapshotFunction(System.identityHashCode(logIdentityObject), control, doSnapshot); final BarrageMessage snapshot = snapshotMsg.getValue(); - snapshot.step = snapshot.firstSeq = snapshot.lastSeq = clockStep; + snapshot.firstSeq = snapshot.lastSeq = clockStep; return snapshot; } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java index 6b60a57310e..6041925aa6a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java @@ -45,7 +45,7 @@ public static class AddColumnData { public long firstSeq = -1; public long lastSeq = -1; - public long step = -1; + /** The size of the table after this update. -1 if unknown. */ public long tableSize = -1; public boolean isSnapshot; diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index ecf5d73e289..fd86e5b7860 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -1413,7 +1413,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { } // prepare updates to propagate - final long maxStep = snapshot != null ? snapshot.step : Long.MAX_VALUE; + final long maxStep = snapshot != null ? snapshot.firstSeq : Long.MAX_VALUE; int deltaSplitIdx = pendingDeltas.size(); for (; deltaSplitIdx > 0; --deltaSplitIdx) { @@ -2037,6 +2037,7 @@ final class ColumnInfo { propagationRowSet.remove(downstream.rowsRemoved); downstream.shifted.apply(propagationRowSet); propagationRowSet.insert(downstream.rowsAdded); + downstream.tableSize = propagationRowSet.size(); return downstream; } diff --git a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java index fb631c8fbbc..9bcfbc9e142 100644 --- a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java +++ b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java @@ -14,6 +14,7 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.RowSetShiftData; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableUpdate; import io.deephaven.engine.table.TableUpdateListener; @@ -92,7 +93,7 @@ HierarchicalTableViewSubscription create( // region Guarded by snapshot lock private BitSet columns; private RowSet rows; - private long lastExpandedSize; + private final WritableRowSet prevKeyspaceViewportRows = RowSetFactory.empty(); // endregion Guarded by snapshot lock private enum State { @@ -282,8 +283,8 @@ private void process() { return; } try { - lastExpandedSize = buildAndSendSnapshot(streamGeneratorFactory, listener, subscriptionOptions, view, - this::recordSnapshotNanos, this::recordWriteMetrics, columns, rows, lastExpandedSize); + buildAndSendSnapshot(streamGeneratorFactory, listener, subscriptionOptions, view, + this::recordSnapshotNanos, this::recordWriteMetrics, columns, rows, prevKeyspaceViewportRows); } catch (Exception e) { GrpcUtil.safelyError(listener, errorTransformer.transform(e)); state = State.Done; @@ -291,7 +292,7 @@ private void process() { } } - private static long buildAndSendSnapshot( + private static void buildAndSendSnapshot( @NotNull final BarrageStreamGenerator.Factory streamGeneratorFactory, @NotNull final StreamObserver listener, @NotNull final BarrageSubscriptionOptions subscriptionOptions, @@ -300,7 +301,7 @@ private static long buildAndSendSnapshot( @NotNull final BarragePerformanceLog.WriteMetricsConsumer writeMetricsConsumer, @NotNull final BitSet columns, @NotNull final RowSet rows, - final long lastExpandedSize) { + @NotNull final WritableRowSet prevKeyspaceViewportRows) { // 1. Grab some schema and snapshot information final List> columnDefinitions = view.getHierarchicalTable().getAvailableColumnDefinitions(); @@ -322,18 +323,20 @@ private static long buildAndSendSnapshot( columns, rows, destinations); snapshotNanosConsumer.accept(System.nanoTime() - snapshotStartNanos); + // note that keyspace is identical to position space for HierarchicalTableView snapshots + final RowSet snapshotRows = RowSetFactory.flat(expandedSize); + final RowSet keyspaceViewportRows = rows.intersect(snapshotRows); + // 4. Make and populate a BarrageMessage final BarrageMessage barrageMessage = new BarrageMessage(); + // We don't populate firstSeq, or lastSeq debugging information; they are not relevant to this use case. + barrageMessage.isSnapshot = true; - // We don't populate length, snapshotRowSet, snapshotRowSetIsReversed, or snapshotColumns; they are only set by - // the client. - // We don't populate step, firstSeq, or lastSeq debugging information; they are not relevant to this use case. - - barrageMessage.rowsAdded = RowSetFactory.flat(expandedSize); - barrageMessage.rowsIncluded = RowSetFactory.fromRange(rows.firstRowKey(), - Math.min(barrageMessage.rowsAdded.lastRowKey(), rows.lastRowKey())); - barrageMessage.rowsRemoved = RowSetFactory.flat(lastExpandedSize); + barrageMessage.rowsAdded = snapshotRows; + barrageMessage.rowsIncluded = keyspaceViewportRows; + barrageMessage.rowsRemoved = RowSetFactory.empty(); barrageMessage.shifted = RowSetShiftData.EMPTY; + barrageMessage.tableSize = expandedSize; barrageMessage.addColumnData = new BarrageMessage.AddColumnData[numAvailableColumns]; for (int ci = 0, di = 0; ci < numAvailableColumns; ++ci) { @@ -357,15 +360,16 @@ private static long buildAndSendSnapshot( // 5. Send the BarrageMessage try (final BarrageStreamGenerator streamGenerator = streamGeneratorFactory.newGenerator(barrageMessage, writeMetricsConsumer)) { - // Note that we're always specifying "isInitialSnapshot=true". This is to provoke the subscription view to - // send the added rows on every snapshot, since (1) our added rows are flat, and thus cheap to send, and - // (2) we're relying on added rows to signal the full expanded size to the client. - GrpcUtil.safelyOnNext(listener, - streamGenerator.getSubView(subscriptionOptions, true, true, rows, false, rows, rows, columns)); + // initialSnapshot flag is ignored for non-growing viewports + final boolean initialSnapshot = false; + final boolean isFullSubscription = false; + GrpcUtil.safelyOnNext(listener, streamGenerator.getSubView( + subscriptionOptions, initialSnapshot, isFullSubscription, rows, false, + prevKeyspaceViewportRows, keyspaceViewportRows, columns)); } - // 6. Let the caller know what the expanded size was - return expandedSize; + prevKeyspaceViewportRows.clear(); + prevKeyspaceViewportRows.insert(keyspaceViewportRows); } public void setViewport( diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java index 97ba53ed071..c7609e25f53 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java @@ -467,6 +467,7 @@ public static class ViewportImpl extends WebBarrageSubscription { public ViewportImpl(ClientTableState state, ViewportChangedHandler viewportChangedHandler, DataChangedHandler dataChangedHandler, WebColumnData[] dataSinks) { super(state, viewportChangedHandler, dataChangedHandler, dataSinks); + serverViewport = RangeSet.empty(); } @Override @@ -481,8 +482,10 @@ public RangeSet getCurrentRowSet() { @Override public void applyUpdates(WebBarrageMessage message) { + final BitSet prevServerColumns = serverColumns == null ? null : (BitSet) serverColumns.clone(); lastTableSize = message.tableSize; + final RangeSet prevServerViewport = serverViewport.copy(); if (message.isSnapshot) { updateServerViewport(message.snapshotRowSet, message.snapshotColumns, message.snapshotRowSetIsReversed); viewportChangedHandler.onServerViewportChanged(serverViewport, serverColumns, serverReverseViewport); @@ -497,15 +500,22 @@ public void applyUpdates(WebBarrageMessage message) { currentRowSet.removeRange(new Range(newSize, prevSize - 1)); } - if (!message.rowsAdded.isEmpty() || !message.rowsRemoved.isEmpty()) { - for (int ii = 0; ii < message.addColumnData.length; ii++) { - if (!isSubscribedColumn(ii)) { - continue; - } + for (int ii = 0; ii < message.addColumnData.length; ii++) { + final WebBarrageMessage.AddColumnData column = message.addColumnData[ii]; + final boolean prevSubscribed = prevServerColumns == null || prevServerColumns.get(ii); + final boolean currSubscribed = serverColumns == null || serverColumns.get(ii); - final WebBarrageMessage.AddColumnData column = message.addColumnData[ii]; - for (int j = 0; j < column.data.size(); j++) { + if (!currSubscribed && prevSubscribed && prevSize > 0) { + destSources[ii].applyUpdate(column.data, RangeSet.empty(), RangeSet.ofRange(0, prevSize - 1)); + continue; + } + + if (!message.rowsAdded.isEmpty() || !message.rowsRemoved.isEmpty()) { + if (prevSubscribed && currSubscribed) { destSources[ii].applyUpdate(column.data, message.rowsAdded, message.rowsRemoved); + } else if (currSubscribed) { + // this column is a new subscription + destSources[ii].applyUpdate(column.data, message.rowsAdded, RangeSet.empty()); } } } @@ -513,7 +523,7 @@ public void applyUpdates(WebBarrageMessage message) { final BitSet modifiedColumnSet = new BitSet(numColumns()); for (int ii = 0; ii < message.modColumnData.length; ii++) { WebBarrageMessage.ModColumnData column = message.modColumnData[ii]; - if (column.rowsModified.isEmpty()) { + if (!isSubscribedColumn(ii) || column.rowsModified.isEmpty()) { continue; } @@ -525,11 +535,14 @@ public void applyUpdates(WebBarrageMessage message) { } } + assert message.tableSize >= 0; state.setSize(message.tableSize); dataChangedHandler.onDataChanged( - RangeSet.ofRange(0, currentRowSet.size()), - RangeSet.ofRange(0, prevSize), - RangeSet.empty(), new ShiftedRange[0], modifiedColumnSet); + serverViewport.copy(), + prevServerViewport.copy(), + RangeSet.empty(), + new ShiftedRange[0], + serverColumns == null ? null : (BitSet) serverColumns.clone()); } @Override From b4d5b690627162735f6732cd639c19300461fd84 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 13:36:42 -0700 Subject: [PATCH 24/35] Lazily compute rowset encoding --- .../barrage/BarrageStreamGeneratorImpl.java | 71 ++++++++++++------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 827ed68eed0..6a9a73b4c03 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -149,7 +149,7 @@ public void close() { private final RowSetGenerator rowsAdded; private final RowSetGenerator rowsIncluded; private final RowSetGenerator rowsRemoved; - private final RowSetShiftDataGenerator shifted; + private final RowSetShiftDataGenerator original; private final ChunkListInputStreamGenerator[] addColumnData; private final ModColumnGenerator[] modColumnData; @@ -172,7 +172,7 @@ public BarrageStreamGeneratorImpl(final BarrageMessage message, rowsAdded = new RowSetGenerator(message.rowsAdded); rowsIncluded = new RowSetGenerator(message.rowsIncluded); rowsRemoved = new RowSetGenerator(message.rowsRemoved); - shifted = new RowSetShiftDataGenerator(message.shifted); + original = new RowSetShiftDataGenerator(message.shifted); addColumnData = new ChunkListInputStreamGenerator[message.addColumnData.length]; for (int i = 0; i < message.addColumnData.length; ++i) { @@ -304,7 +304,7 @@ public SubView(final BarrageSubscriptionOptions options, // note that snapshot rowsAdded contain all rows; we "repaint" only rows shared between prev and // new viewports. keyspacePrevClientRepaintedRows.remove(rowsAdded.original); - shifted.original.unapply(keyspacePrevClientRepaintedRows); + original.original.unapply(keyspacePrevClientRepaintedRows); } keyspacePrevClientRepaintedRows.retain(keyspaceViewportPrev); @@ -315,7 +315,7 @@ public SubView(final BarrageSubscriptionOptions options, rowsToRetain = toClose.add(keyspaceViewport.copy()); } else { rowsToRetain = toClose.add(keyspaceViewport.minus(rowsAdded.original)); - shifted.original.unapply(rowsToRetain); + original.original.unapply(rowsToRetain); } final WritableRowSet noLongerExistingRows = toClose.add(keyspaceViewportPrev.minus(rowsToRetain)); noLongerExistingRows.insert(keyspacePrevClientRepaintedRows); @@ -476,7 +476,7 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { // we only send shifts to full table subscriptions shiftDataOffset = 0; } else { - shiftDataOffset = shifted.addToFlatBuffer(metadata); + shiftDataOffset = original.addToFlatBuffer(metadata); } // Added Chunk Data: @@ -650,7 +650,7 @@ private ByteBuffer getSnapshotMetadata() throws IOException { final int rowsAddedOffset = rowsAdded.addToFlatBuffer(metadata); // no shifts in a snapshot, but need to provide a valid structure - final int shiftDataOffset = shifted.addToFlatBuffer(metadata); + final int shiftDataOffset = original.addToFlatBuffer(metadata); // Added Chunk Data: int addedRowsIncludedOffset = 0; @@ -1121,7 +1121,10 @@ public static abstract class ByteArrayGenerator { protected int len; protected byte[] raw; - protected int addToFlatBuffer(final FlatBufferBuilder builder) { + protected abstract void ensureComputed() throws IOException; + + protected int addToFlatBuffer(final FlatBufferBuilder builder) throws IOException { + ensureComputed(); return builder.createByteVector(raw, 0, len); } } @@ -1131,13 +1134,6 @@ public static class RowSetGenerator extends ByteArrayGenerator implements SafeCl public RowSetGenerator(final RowSet rowSet) throws IOException { this.original = rowSet.copy(); - try (final ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream(); - final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos)) { - ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, rowSet); - oos.flush(); - raw = baos.peekBuffer(); - len = baos.size(); - } } @Override @@ -1145,8 +1141,18 @@ public void close() { original.close(); } - public DrainableByteArrayInputStream getInputStream() { - return new DrainableByteArrayInputStream(raw, 0, len); + protected synchronized void ensureComputed() throws IOException { + if (raw != null) { + return; + } + + try (final ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream(); + final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos)) { + ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, original); + oos.flush(); + raw = baos.peekBuffer(); + len = baos.size(); + } } /** @@ -1158,6 +1164,7 @@ public DrainableByteArrayInputStream getInputStream() { */ protected int addToFlatBuffer(final RowSet viewport, final FlatBufferBuilder builder) throws IOException { if (original.subsetOf(viewport)) { + ensureComputed(); return addToFlatBuffer(builder); } @@ -1177,11 +1184,21 @@ protected int addToFlatBuffer(final RowSet viewport, final FlatBufferBuilder bui } public static class BitSetGenerator extends ByteArrayGenerator { + private final BitSet original; + public BitSetGenerator(final BitSet bitset) { - BitSet original = bitset == null ? new BitSet() : bitset; - this.raw = original.toByteArray(); + original = bitset == null ? new BitSet() : bitset; + } + + @Override + protected synchronized void ensureComputed() { + if (raw != null) { + return; + } + + raw = original.toByteArray(); final int nBits = original.previousSetBit(Integer.MAX_VALUE - 1) + 1; - this.len = (int) ((long) nBits + 7) / 8; + len = (int) ((long) nBits + 7) / 8; } } @@ -1190,22 +1207,28 @@ public static class RowSetShiftDataGenerator extends ByteArrayGenerator { public RowSetShiftDataGenerator(final RowSetShiftData shifted) throws IOException { original = shifted; + } + + protected synchronized void ensureComputed() throws IOException { + if (raw != null) { + return; + } final RowSetBuilderSequential sRangeBuilder = RowSetFactory.builderSequential(); final RowSetBuilderSequential eRangeBuilder = RowSetFactory.builderSequential(); final RowSetBuilderSequential destBuilder = RowSetFactory.builderSequential(); - if (shifted != null) { - for (int i = 0; i < shifted.size(); ++i) { - long s = shifted.getBeginRange(i); - final long dt = shifted.getShiftDelta(i); + if (original != null) { + for (int i = 0; i < original.size(); ++i) { + long s = original.getBeginRange(i); + final long dt = original.getShiftDelta(i); if (dt < 0 && s < -dt) { s = -dt; } sRangeBuilder.appendKey(s); - eRangeBuilder.appendKey(shifted.getEndRange(i)); + eRangeBuilder.appendKey(original.getEndRange(i)); destBuilder.appendKey(s + dt); } } From 6c123148d5373d051661b3c5753d6e5eebd82909 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 14:12:04 -0700 Subject: [PATCH 25/35] Fixup jsapi tests --- .../api/barrage/data/WebBarrageSubscription.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java index c7609e25f53..9dc40b5e1b6 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java @@ -79,7 +79,8 @@ public static WebBarrageSubscription subscribe( if (cts.getTableDef().getAttributes().isBlinkTable()) { return new BlinkImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); - } else if (subscriptionType == SubscriptionType.FULL_SUBSCRIPTION) { + } else if (subscriptionType == SubscriptionType.FULL_SUBSCRIPTION + || subscriptionType == SubscriptionType.SNAPSHOT) { return new RedirectedImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); } else { return new ViewportImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); @@ -462,7 +463,7 @@ private void freeRows(RangeSet removed) { } public static class ViewportImpl extends WebBarrageSubscription { - private long lastTableSize = -1; + private long lastTableSize = 0; public ViewportImpl(ClientTableState state, ViewportChangedHandler viewportChangedHandler, DataChangedHandler dataChangedHandler, WebColumnData[] dataSinks) { @@ -477,12 +478,16 @@ public long getCurrentSize() { @Override public RangeSet getCurrentRowSet() { + if (lastTableSize <= 0) { + return RangeSet.empty(); + } return RangeSet.ofRange(0, lastTableSize - 1); } @Override public void applyUpdates(WebBarrageMessage message) { final BitSet prevServerColumns = serverColumns == null ? null : (BitSet) serverColumns.clone(); + assert message.tableSize >= 0; lastTableSize = message.tableSize; final RangeSet prevServerViewport = serverViewport.copy(); @@ -535,11 +540,10 @@ public void applyUpdates(WebBarrageMessage message) { } } - assert message.tableSize >= 0; state.setSize(message.tableSize); dataChangedHandler.onDataChanged( - serverViewport.copy(), - prevServerViewport.copy(), + serverViewport == null ? RangeSet.empty() : serverViewport.copy(), + prevServerViewport == null ? RangeSet.empty() : prevServerViewport.copy(), RangeSet.empty(), new ShiftedRange[0], serverColumns == null ? null : (BitSet) serverColumns.clone()); From f9be6e502479bcfe62803dac88c3a6a6a7635b8f Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 14:57:14 -0700 Subject: [PATCH 26/35] Quick round feedback --- .../barrage/BarrageStreamGeneratorImpl.java | 84 +++++++++++-------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 6a9a73b4c03..61ed5fb6a40 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -149,7 +149,7 @@ public void close() { private final RowSetGenerator rowsAdded; private final RowSetGenerator rowsIncluded; private final RowSetGenerator rowsRemoved; - private final RowSetShiftDataGenerator original; + private final RowSetShiftDataGenerator shifted; private final ChunkListInputStreamGenerator[] addColumnData; private final ModColumnGenerator[] modColumnData; @@ -172,7 +172,7 @@ public BarrageStreamGeneratorImpl(final BarrageMessage message, rowsAdded = new RowSetGenerator(message.rowsAdded); rowsIncluded = new RowSetGenerator(message.rowsIncluded); rowsRemoved = new RowSetGenerator(message.rowsRemoved); - original = new RowSetShiftDataGenerator(message.shifted); + shifted = new RowSetShiftDataGenerator(message.shifted); addColumnData = new ChunkListInputStreamGenerator[message.addColumnData.length]; for (int i = 0; i < message.addColumnData.length; ++i) { @@ -304,7 +304,7 @@ public SubView(final BarrageSubscriptionOptions options, // note that snapshot rowsAdded contain all rows; we "repaint" only rows shared between prev and // new viewports. keyspacePrevClientRepaintedRows.remove(rowsAdded.original); - original.original.unapply(keyspacePrevClientRepaintedRows); + shifted.original.unapply(keyspacePrevClientRepaintedRows); } keyspacePrevClientRepaintedRows.retain(keyspaceViewportPrev); @@ -315,7 +315,7 @@ public SubView(final BarrageSubscriptionOptions options, rowsToRetain = toClose.add(keyspaceViewport.copy()); } else { rowsToRetain = toClose.add(keyspaceViewport.minus(rowsAdded.original)); - original.original.unapply(rowsToRetain); + shifted.original.unapply(rowsToRetain); } final WritableRowSet noLongerExistingRows = toClose.add(keyspaceViewportPrev.minus(rowsToRetain)); noLongerExistingRows.insert(keyspacePrevClientRepaintedRows); @@ -476,7 +476,7 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { // we only send shifts to full table subscriptions shiftDataOffset = 0; } else { - shiftDataOffset = original.addToFlatBuffer(metadata); + shiftDataOffset = shifted.addToFlatBuffer(metadata); } // Added Chunk Data: @@ -650,7 +650,7 @@ private ByteBuffer getSnapshotMetadata() throws IOException { final int rowsAddedOffset = rowsAdded.addToFlatBuffer(metadata); // no shifts in a snapshot, but need to provide a valid structure - final int shiftDataOffset = original.addToFlatBuffer(metadata); + final int shiftDataOffset = shifted.addToFlatBuffer(metadata); // Added Chunk Data: int addedRowsIncludedOffset = 0; @@ -1187,18 +1187,23 @@ public static class BitSetGenerator extends ByteArrayGenerator { private final BitSet original; public BitSetGenerator(final BitSet bitset) { - original = bitset == null ? new BitSet() : bitset; + original = bitset == null ? new BitSet() : (BitSet) bitset.clone(); } @Override - protected synchronized void ensureComputed() { + protected void ensureComputed() { if (raw != null) { return; } + synchronized (this) { + if (raw != null) { + return; + } - raw = original.toByteArray(); - final int nBits = original.previousSetBit(Integer.MAX_VALUE - 1) + 1; - len = (int) ((long) nBits + 7) / 8; + raw = original.toByteArray(); + final int nBits = original.previousSetBit(Integer.MAX_VALUE - 1) + 1; + len = (int) ((long) nBits + 7) / 8; + } } } @@ -1209,41 +1214,46 @@ public RowSetShiftDataGenerator(final RowSetShiftData shifted) throws IOExceptio original = shifted; } - protected synchronized void ensureComputed() throws IOException { + protected void ensureComputed() throws IOException { if (raw != null) { return; } + synchronized (this) { + if (raw != null) { + return; + } - final RowSetBuilderSequential sRangeBuilder = RowSetFactory.builderSequential(); - final RowSetBuilderSequential eRangeBuilder = RowSetFactory.builderSequential(); - final RowSetBuilderSequential destBuilder = RowSetFactory.builderSequential(); + final RowSetBuilderSequential sRangeBuilder = RowSetFactory.builderSequential(); + final RowSetBuilderSequential eRangeBuilder = RowSetFactory.builderSequential(); + final RowSetBuilderSequential destBuilder = RowSetFactory.builderSequential(); - if (original != null) { - for (int i = 0; i < original.size(); ++i) { - long s = original.getBeginRange(i); - final long dt = original.getShiftDelta(i); + if (original != null) { + for (int i = 0; i < original.size(); ++i) { + long s = original.getBeginRange(i); + final long dt = original.getShiftDelta(i); - if (dt < 0 && s < -dt) { - s = -dt; - } + if (dt < 0 && s < -dt) { + s = -dt; + } - sRangeBuilder.appendKey(s); - eRangeBuilder.appendKey(original.getEndRange(i)); - destBuilder.appendKey(s + dt); + sRangeBuilder.appendKey(s); + eRangeBuilder.appendKey(original.getEndRange(i)); + destBuilder.appendKey(s + dt); + } } - } - try (final RowSet sRange = sRangeBuilder.build(); - final RowSet eRange = eRangeBuilder.build(); - final RowSet dest = destBuilder.build(); - final ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream(); - final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos)) { - ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, sRange); - ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, eRange); - ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, dest); - oos.flush(); - raw = baos.peekBuffer(); - len = baos.size(); + try (final RowSet sRange = sRangeBuilder.build(); + final RowSet eRange = eRangeBuilder.build(); + final RowSet dest = destBuilder.build(); + final ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream(); + final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos)) { + ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, sRange); + ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, eRange); + ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, dest); + oos.flush(); + raw = baos.peekBuffer(); + len = baos.size(); + } } } } From 425262282e76046fb90febe3385681b44708bf0c Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 14:57:38 -0700 Subject: [PATCH 27/35] spotless --- .../extensions/barrage/BarrageStreamGeneratorImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 61ed5fb6a40..c5e025f3b92 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -1243,10 +1243,10 @@ protected void ensureComputed() throws IOException { } try (final RowSet sRange = sRangeBuilder.build(); - final RowSet eRange = eRangeBuilder.build(); - final RowSet dest = destBuilder.build(); - final ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream(); - final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos)) { + final RowSet eRange = eRangeBuilder.build(); + final RowSet dest = destBuilder.build(); + final ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream(); + final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos)) { ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, sRange); ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, eRange); ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, dest); From 2767defb15d405b9149c9b121bf31b8d26274159 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 15:00:08 -0700 Subject: [PATCH 28/35] Double checked locking fixes --- .../barrage/BarrageStreamGeneratorImpl.java | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index c5e025f3b92..dd83444e01e 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -1119,7 +1119,7 @@ private int appendModColumns(final RecordBatchMessageView view, final long start public static abstract class ByteArrayGenerator { protected int len; - protected byte[] raw; + protected volatile byte[] raw; protected abstract void ensureComputed() throws IOException; @@ -1141,17 +1141,23 @@ public void close() { original.close(); } - protected synchronized void ensureComputed() throws IOException { + protected void ensureComputed() throws IOException { if (raw != null) { return; } - try (final ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream(); - final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos)) { - ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, original); - oos.flush(); - raw = baos.peekBuffer(); - len = baos.size(); + synchronized (this) { + if (raw != null) { + return; + } + + try (final ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream(); + final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos)) { + ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, original); + oos.flush(); + len = baos.size(); + raw = baos.peekBuffer(); + } } } @@ -1195,14 +1201,15 @@ protected void ensureComputed() { if (raw != null) { return; } + synchronized (this) { if (raw != null) { return; } - raw = original.toByteArray(); final int nBits = original.previousSetBit(Integer.MAX_VALUE - 1) + 1; len = (int) ((long) nBits + 7) / 8; + raw = original.toByteArray(); } } } @@ -1218,6 +1225,7 @@ protected void ensureComputed() throws IOException { if (raw != null) { return; } + synchronized (this) { if (raw != null) { return; @@ -1251,8 +1259,8 @@ protected void ensureComputed() throws IOException { ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, eRange); ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, dest); oos.flush(); - raw = baos.peekBuffer(); len = baos.size(); + raw = baos.peekBuffer(); } } } From 78c4cb761da06352c60f22dae2737888308cf03e Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 16:01:34 -0700 Subject: [PATCH 29/35] Ryan's final review --- .../engine/table/impl/util/BarrageMessage.java | 5 ++++- .../barrage/BarrageStreamGeneratorImpl.java | 10 +++++----- .../server/barrage/BarrageMessageProducer.java | 1 + .../HierarchicalTableViewSubscription.java | 18 +++++++----------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java index 6041925aa6a..b1f0db57cb7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java @@ -48,11 +48,14 @@ public static class AddColumnData { /** The size of the table after this update. -1 if unknown. */ public long tableSize = -1; - public boolean isSnapshot; + /** The RowSet the server is now respecting for this client; only set when parsing on the client. */ public RowSet snapshotRowSet; + /** Whether the server-respecting viewport is a tail; only set when parsing on the client. */ public boolean snapshotRowSetIsReversed; + /** The BitSet of columns the server is now respecting for this client; only set when parsing on the client. */ public BitSet snapshotColumns; + public boolean isSnapshot; public RowSet rowsAdded; public RowSet rowsIncluded; public RowSet rowsRemoved; diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index dd83444e01e..3178b9bbfe3 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -242,7 +242,7 @@ private final class SubView implements RecordBatchMessageView { private final boolean isFullSubscription; private final RowSet viewport; private final boolean reverseViewport; - private final RowSet keyspaceViewport; + private final boolean hasViewport; private final BitSet subscribedColumns; private final long numClientIncludedRows; @@ -266,7 +266,7 @@ public SubView(final BarrageSubscriptionOptions options, this.isFullSubscription = isFullSubscription; this.viewport = viewport; this.reverseViewport = reverseViewport; - this.keyspaceViewport = keyspaceViewport; + this.hasViewport = keyspaceViewport != null; this.subscribedColumns = subscribedColumns; // precompute the included rows / offsets and viewport removed rows @@ -491,7 +491,7 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { TIntArrayList modOffsets = new TIntArrayList(modColumnData.length); for (int ii = 0; ii < modColumnData.length; ++ii) { final int myModRowOffset; - if (keyspaceViewport != null) { + if (hasViewport) { try (final RowSetGenerator modRowsGen = new RowSetGenerator(clientModdedRows[ii])) { myModRowOffset = modRowsGen.addToFlatBuffer(metadata); } @@ -1152,7 +1152,7 @@ protected void ensureComputed() throws IOException { } try (final ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream(); - final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos)) { + final LittleEndianDataOutputStream oos = new LittleEndianDataOutputStream(baos)) { ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, original); oos.flush(); len = baos.size(); @@ -1181,8 +1181,8 @@ protected int addToFlatBuffer(final RowSet viewport, final FlatBufferBuilder bui final RowSet viewOfOriginal = original.intersect(viewport)) { ExternalizableRowSetUtils.writeExternalCompressedDeltas(oos, viewOfOriginal); oos.flush(); - nraw = baos.peekBuffer(); nlen = baos.size(); + nraw = baos.peekBuffer(); } return builder.createByteVector(nraw, 0, nlen); diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index fd86e5b7860..a1efd97b3e7 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -1379,6 +1379,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { if (!snapshot.rowsAdded.isEmpty()) { snapshot.rowsAdded.close(); snapshot.rowsAdded = RowSetFactory.empty(); + snapshot.tableSize = 0; } } long elapsed = System.nanoTime() - start; diff --git a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java index 9bcfbc9e142..73164acd941 100644 --- a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java +++ b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java @@ -323,17 +323,14 @@ private static void buildAndSendSnapshot( columns, rows, destinations); snapshotNanosConsumer.accept(System.nanoTime() - snapshotStartNanos); - // note that keyspace is identical to position space for HierarchicalTableView snapshots - final RowSet snapshotRows = RowSetFactory.flat(expandedSize); - final RowSet keyspaceViewportRows = rows.intersect(snapshotRows); - // 4. Make and populate a BarrageMessage final BarrageMessage barrageMessage = new BarrageMessage(); - // We don't populate firstSeq, or lastSeq debugging information; they are not relevant to this use case. + // We don't populate firstSeq or lastSeq debugging information; they are not relevant to this use case. barrageMessage.isSnapshot = true; - barrageMessage.rowsAdded = snapshotRows; - barrageMessage.rowsIncluded = keyspaceViewportRows; + barrageMessage.rowsAdded = RowSetFactory.flat(expandedSize); + barrageMessage.rowsIncluded = RowSetFactory.fromRange( + rows.firstRowKey(), Math.min(expandedSize - 1, rows.lastRowKey())); barrageMessage.rowsRemoved = RowSetFactory.empty(); barrageMessage.shifted = RowSetShiftData.EMPTY; barrageMessage.tableSize = expandedSize; @@ -365,11 +362,10 @@ private static void buildAndSendSnapshot( final boolean isFullSubscription = false; GrpcUtil.safelyOnNext(listener, streamGenerator.getSubView( subscriptionOptions, initialSnapshot, isFullSubscription, rows, false, - prevKeyspaceViewportRows, keyspaceViewportRows, columns)); - } + prevKeyspaceViewportRows, barrageMessage.rowsIncluded, columns)); - prevKeyspaceViewportRows.clear(); - prevKeyspaceViewportRows.insert(keyspaceViewportRows); + prevKeyspaceViewportRows.resetTo(barrageMessage.rowsIncluded); + } } public void setViewport( From ea6f8982207a3296e88ffde5211ed2068f155aac Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 16:22:51 -0700 Subject: [PATCH 30/35] Clarify strategy on who owns RowSets passed into getSubView --- .../barrage/BarrageStreamGenerator.java | 9 ++- .../barrage/BarrageStreamGeneratorImpl.java | 59 +++++++++---------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java index a571c7ba183..b8dce7527aa 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGenerator.java @@ -66,6 +66,8 @@ BarrageStreamGenerator newGenerator( /** * Obtain a View of this StreamGenerator that can be sent to a single subscriber. + *

+ * Note that all passed in arguments are owned by the caller and may be modified external to this method. * * @param options serialization options for this specific view * @param isInitialSnapshot indicates whether this is the first snapshot for the listener @@ -97,6 +99,8 @@ MessageView getSubView( /** * Obtain a View of this StreamGenerator that can be sent to a single requestor. + *

+ * Note that all passed in arguments are owned by the caller and may be modified external to this method. * * @param options serialization options for this specific view * @param viewport is the position-space viewport @@ -104,7 +108,10 @@ MessageView getSubView( * @param snapshotColumns are the columns included for this view * @return a MessageView filtered by the snapshot properties that can be sent to that requestor */ - MessageView getSnapshotView(BarrageSnapshotOptions options, @Nullable RowSet viewport, boolean reverseViewport, + MessageView getSnapshotView( + BarrageSnapshotOptions options, + @Nullable RowSet viewport, + boolean reverseViewport, @Nullable RowSet keyspaceViewport, BitSet snapshotColumns); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 3178b9bbfe3..55c2afe7ad8 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -75,8 +75,6 @@ public class BarrageStreamGeneratorImpl implements BarrageStreamGenerator { 100 * 1024 * 1024); public interface RecordBatchMessageView extends MessageView { - boolean isViewport(); - StreamReaderOptions options(); RowSet addRowOffsets(); @@ -240,13 +238,13 @@ private final class SubView implements RecordBatchMessageView { private final BarrageSubscriptionOptions options; private final boolean isInitialSnapshot; private final boolean isFullSubscription; - private final RowSet viewport; private final boolean reverseViewport; private final boolean hasViewport; private final BitSet subscribedColumns; private final long numClientIncludedRows; private final long numClientModRows; + private final WritableRowSet clientViewport; private final WritableRowSet clientIncludedRows; private final WritableRowSet clientIncludedRowOffsets; private final WritableRowSet[] clientModdedRows; @@ -264,7 +262,7 @@ public SubView(final BarrageSubscriptionOptions options, this.options = options; this.isInitialSnapshot = isInitialSnapshot; this.isFullSubscription = isFullSubscription; - this.viewport = viewport; + this.clientViewport = viewport.copy(); this.reverseViewport = reverseViewport; this.hasViewport = keyspaceViewport != null; this.subscribedColumns = subscribedColumns; @@ -389,6 +387,7 @@ public void forEachStream(Consumer visitor) throws IOExcepti numClientIncludedRows > 0 ? null : metadata, BarrageStreamGeneratorImpl.this::appendModColumns, bytesWritten); } finally { + clientViewport.close(); clientIncludedRows.close(); clientIncludedRowOffsets.close(); if (clientModdedRowOffsets != null) { @@ -410,11 +409,6 @@ private int batchSize() { return batchSize; } - @Override - public boolean isViewport() { - return viewport != null; - } - @Override public StreamReaderOptions options() { return options; @@ -437,8 +431,8 @@ private ByteBuffer getSubscriptionMetadata() throws IOException { final FlatBufferBuilder metadata = new FlatBufferBuilder(); int effectiveViewportOffset = 0; - if (isSnapshot && isViewport()) { - try (final RowSetGenerator viewportGen = new RowSetGenerator(viewport)) { + if (isSnapshot && clientViewport != null) { + try (final RowSetGenerator viewportGen = new RowSetGenerator(clientViewport)) { effectiveViewportOffset = viewportGen.addToFlatBuffer(metadata); } } @@ -551,12 +545,13 @@ public MessageView getSnapshotView(BarrageSnapshotOptions options) { private final class SnapshotView implements RecordBatchMessageView { private final BarrageSnapshotOptions options; - private final RowSet viewport; private final boolean reverseViewport; private final BitSet subscribedColumns; private final long numClientAddRows; - private final RowSet clientAddedRows; - private final RowSet clientAddedRowOffsets; + + private final WritableRowSet clientViewport; + private final WritableRowSet clientAddedRows; + private final WritableRowSet clientAddedRowOffsets; public SnapshotView(final BarrageSnapshotOptions options, @Nullable final RowSet viewport, @@ -564,7 +559,7 @@ public SnapshotView(final BarrageSnapshotOptions options, @Nullable final RowSet keyspaceViewport, @Nullable final BitSet subscribedColumns) { this.options = options; - this.viewport = viewport; + this.clientViewport = viewport.copy(); this.reverseViewport = reverseViewport; this.subscribedColumns = subscribedColumns; @@ -590,17 +585,22 @@ public void forEachStream(Consumer visitor) throws IOExcepti // batch size is maximum, will write fewer rows when needed int maxBatchSize = batchSize(); final MutableInt actualBatchSize = new MutableInt(); - if (numClientAddRows == 0) { - // we still need to send a message containing metadata when there are no rows - visitor.accept(getInputStream(this, 0, 0, actualBatchSize, metadata, - BarrageStreamGeneratorImpl.this::appendAddColumns)); - } else { - // send the add batches - processBatches(visitor, this, numClientAddRows, maxBatchSize, metadata, - BarrageStreamGeneratorImpl.this::appendAddColumns, bytesWritten); + try { + if (numClientAddRows == 0) { + // we still need to send a message containing metadata when there are no rows + visitor.accept(getInputStream(this, 0, 0, actualBatchSize, metadata, + BarrageStreamGeneratorImpl.this::appendAddColumns)); + } else { + // send the add batches + processBatches(visitor, this, numClientAddRows, maxBatchSize, metadata, + BarrageStreamGeneratorImpl.this::appendAddColumns, bytesWritten); + } + } finally { + clientViewport.close(); + clientAddedRowOffsets.close(); + clientAddedRows.close(); } - clientAddedRowOffsets.close(); - clientAddedRows.close(); + writeConsumer.onWrite(bytesWritten.get(), System.nanoTime() - startTm); } @@ -612,11 +612,6 @@ private int batchSize() { return batchSize; } - @Override - public boolean isViewport() { - return viewport != null; - } - @Override public StreamReaderOptions options() { return options; @@ -636,8 +631,8 @@ private ByteBuffer getSnapshotMetadata() throws IOException { final FlatBufferBuilder metadata = new FlatBufferBuilder(); int effectiveViewportOffset = 0; - if (isViewport()) { - try (final RowSetGenerator viewportGen = new RowSetGenerator(viewport)) { + if (clientViewport != null) { + try (final RowSetGenerator viewportGen = new RowSetGenerator(clientViewport)) { effectiveViewportOffset = viewportGen.addToFlatBuffer(metadata); } } From 3eeb6286c8fcf968f19cf3cd6c336b7a1df97e14 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 16:25:25 -0700 Subject: [PATCH 31/35] npe fix --- .../barrage/BarrageStreamGeneratorImpl.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java index 55c2afe7ad8..03ccc941508 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/BarrageStreamGeneratorImpl.java @@ -262,7 +262,7 @@ public SubView(final BarrageSubscriptionOptions options, this.options = options; this.isInitialSnapshot = isInitialSnapshot; this.isFullSubscription = isFullSubscription; - this.clientViewport = viewport.copy(); + this.clientViewport = viewport == null ? null : viewport.copy(); this.reverseViewport = reverseViewport; this.hasViewport = keyspaceViewport != null; this.subscribedColumns = subscribedColumns; @@ -387,16 +387,11 @@ public void forEachStream(Consumer visitor) throws IOExcepti numClientIncludedRows > 0 ? null : metadata, BarrageStreamGeneratorImpl.this::appendModColumns, bytesWritten); } finally { - clientViewport.close(); - clientIncludedRows.close(); - clientIncludedRowOffsets.close(); + SafeCloseable.closeAll(clientViewport, clientIncludedRows, clientIncludedRowOffsets, clientRemovedRows); if (clientModdedRowOffsets != null) { SafeCloseable.closeAll(clientModdedRows); SafeCloseable.closeAll(clientModdedRowOffsets); } - if (clientRemovedRows != null) { - clientRemovedRows.close(); - } } writeConsumer.onWrite(bytesWritten.get(), System.nanoTime() - startTm); } @@ -559,7 +554,7 @@ public SnapshotView(final BarrageSnapshotOptions options, @Nullable final RowSet keyspaceViewport, @Nullable final BitSet subscribedColumns) { this.options = options; - this.clientViewport = viewport.copy(); + this.clientViewport = viewport == null ? null : viewport.copy(); this.reverseViewport = reverseViewport; this.subscribedColumns = subscribedColumns; @@ -596,9 +591,7 @@ public void forEachStream(Consumer visitor) throws IOExcepti BarrageStreamGeneratorImpl.this::appendAddColumns, bytesWritten); } } finally { - clientViewport.close(); - clientAddedRowOffsets.close(); - clientAddedRows.close(); + SafeCloseable.closeAll(clientViewport, clientAddedRows, clientAddedRowOffsets); } writeConsumer.onWrite(bytesWritten.get(), System.nanoTime() - startTm); From 84a6100c792b3c12e43d955f364ff22ab3b065b6 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 17:54:48 -0700 Subject: [PATCH 32/35] Bugfix if HT is empty or viewport past end of table --- .../HierarchicalTableViewSubscription.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java index 73164acd941..9cca856f5e0 100644 --- a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java +++ b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java @@ -329,8 +329,12 @@ private static void buildAndSendSnapshot( barrageMessage.isSnapshot = true; barrageMessage.rowsAdded = RowSetFactory.flat(expandedSize); - barrageMessage.rowsIncluded = RowSetFactory.fromRange( - rows.firstRowKey(), Math.min(expandedSize - 1, rows.lastRowKey())); + if (rows.isEmpty() || expandedSize <= rows.firstRowKey()) { + barrageMessage.rowsIncluded = RowSetFactory.empty(); + } else { + barrageMessage.rowsIncluded = RowSetFactory.fromRange( + rows.firstRowKey(), Math.min(expandedSize - 1, rows.lastRowKey())); + } barrageMessage.rowsRemoved = RowSetFactory.empty(); barrageMessage.shifted = RowSetShiftData.EMPTY; barrageMessage.tableSize = expandedSize; From 476ae6502b49d718849d112e7f7614a59913ae60 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 15 Nov 2024 18:05:17 -0700 Subject: [PATCH 33/35] Colin's feedback --- .../extensions/barrage/chunk/DefaultChunkReadingFactory.java | 1 - .../main/java/io/deephaven/client/examples/DoExchange.java | 4 +--- .../hierarchicaltable/HierarchicalTableViewSubscription.java | 2 -- .../deephaven/server/barrage/BarrageMessageRoundTripTest.java | 4 ++-- .../io/deephaven/server/test/FlightMessageRoundTripTest.java | 4 +--- .../web/client/api/barrage/data/WebBarrageSubscription.java | 2 ++ .../client/api/subscription/TableViewportSubscription.java | 1 - .../src/main/java/io/deephaven/web/shared/data/RangeSet.java | 4 ++++ 8 files changed, 10 insertions(+), 12 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java index ddecdb09ca8..2156c95628c 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/DefaultChunkReadingFactory.java @@ -6,7 +6,6 @@ import com.google.common.base.Charsets; import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.Values; -import io.deephaven.extensions.barrage.ColumnConversionMode; import io.deephaven.extensions.barrage.util.ArrowIpcUtil; import io.deephaven.extensions.barrage.util.StreamReaderOptions; import io.deephaven.time.DateTimeUtils; diff --git a/java-client/flight-examples/src/main/java/io/deephaven/client/examples/DoExchange.java b/java-client/flight-examples/src/main/java/io/deephaven/client/examples/DoExchange.java index a3527cbdbcc..6a957222000 100644 --- a/java-client/flight-examples/src/main/java/io/deephaven/client/examples/DoExchange.java +++ b/java-client/flight-examples/src/main/java/io/deephaven/client/examples/DoExchange.java @@ -4,7 +4,6 @@ package io.deephaven.client.examples; import com.google.flatbuffers.FlatBufferBuilder; -import com.google.protobuf.ByteString; import io.deephaven.client.impl.FlightSession; import io.deephaven.proto.util.ScopeTicketHelper; import org.apache.arrow.flight.FlightClient; @@ -44,8 +43,7 @@ protected void execute(FlightSession flight) throws Exception { final FlatBufferBuilder metadata = new FlatBufferBuilder(); // you can use 0 for batch size and max message size to use server-side defaults - int optOffset = - BarrageSnapshotOptions.createBarrageSnapshotOptions(metadata, false, 0, 0, 0); + int optOffset = BarrageSnapshotOptions.createBarrageSnapshotOptions(metadata, false, 0, 0, 0); final int ticOffset = BarrageSnapshotRequest.createTicketVector(metadata, diff --git a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java index 9cca856f5e0..1e6706639b0 100644 --- a/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java +++ b/server/src/main/java/io/deephaven/server/hierarchicaltable/HierarchicalTableViewSubscription.java @@ -60,8 +60,6 @@ HierarchicalTableViewSubscription create( long intervalMillis); } - private static final Logger log = LoggerFactory.getLogger(HierarchicalTableViewSubscription.class); - private final Scheduler scheduler; private final SessionService.ErrorTransformer errorTransformer; private final BarrageStreamGenerator.Factory streamGeneratorFactory; diff --git a/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java b/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java index 954d7a519c2..d5698c55221 100644 --- a/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java +++ b/server/src/test/java/io/deephaven/server/barrage/BarrageMessageRoundTripTest.java @@ -247,8 +247,8 @@ public void validate(final String msg, QueryTable expected) { if (viewport == null) { // Since key-space needs to be kept the same, the RowSets should also be identical between producer and // consumer (not RowSets between expected and consumer; as the consumer maintains the entire RowSet). - Assert.equals(barrageMessageProducer.getRowSet(), "barrageMessageProducer.build()", - barrageTable.getRowSet(), ".build()"); + Assert.equals(barrageMessageProducer.getRowSet(), "barrageMessageProducer.getRowSet()", + barrageTable.getRowSet(), "barrageTable.getRowSet()"); } else { // otherwise, the RowSet should represent a flattened view of the viewport Assert.eqTrue(barrageTable.getRowSet().isFlat(), "barrageTable.getRowSet().isFlat()"); diff --git a/server/test-utils/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java b/server/test-utils/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java index 9a9b934a3b0..a4ad7bfd443 100644 --- a/server/test-utils/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java +++ b/server/test-utils/src/main/java/io/deephaven/server/test/FlightMessageRoundTripTest.java @@ -766,9 +766,7 @@ public void testDoExchangeSnapshot() throws Exception { final FlatBufferBuilder metadata = new FlatBufferBuilder(); // use 0 for batch size and max message size to use server-side defaults - int optOffset = - BarrageSnapshotOptions.createBarrageSnapshotOptions(metadata, - false, 0, 0, 0); + int optOffset = BarrageSnapshotOptions.createBarrageSnapshotOptions(metadata, false, 0, 0, 0); final int ticOffset = BarrageSnapshotRequest.createTicketVector(metadata, diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java index 9dc40b5e1b6..a2e40026fd9 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java @@ -497,6 +497,7 @@ public void applyUpdates(WebBarrageMessage message) { } // Update the currentRowSet; we're guaranteed to be flat + assert currentRowSet.isFlat(); final long prevSize = currentRowSet.size(); final long newSize = prevSize - message.rowsRemoved.size() + message.rowsAdded.size(); if (prevSize < newSize) { @@ -504,6 +505,7 @@ public void applyUpdates(WebBarrageMessage message) { } else if (prevSize > newSize) { currentRowSet.removeRange(new Range(newSize, prevSize - 1)); } + assert currentRowSet.isFlat(); for (int ii = 0; ii < message.addColumnData.length; ii++) { final WebBarrageMessage.AddColumnData column = message.addColumnData[ii]; diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java index c84d187acc0..c199be4d497 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java @@ -339,7 +339,6 @@ public Promise snapshot(JsRangeSet rows, Column[] columns) { .batchSize(WebBarrageSubscription.BATCH_SIZE) .maxMessageSize(WebBarrageSubscription.MAX_MESSAGE_SIZE) .useDeephavenNulls(true) - .previewListLengthLimit(0) .build(); WebBarrageSubscription snapshot = WebBarrageSubscription.subscribe( diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java index 8f56cfddc2f..f1a79e21602 100644 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java +++ b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java @@ -451,6 +451,10 @@ public int rangeCount() { return sortedRanges.size(); } + public boolean isFlat() { + return sortedRanges.isEmpty() || sortedRanges.size() == 1 && getFirstRow() == 0; + } + /** * The total count of items contained in this collection. In some cases this can be expensive to compute, and * generally should not be needed except for debugging purposes, or preallocating space (i.e., do not call this From 738cb115353acac18a0c472cd6dc1eca8943b289 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Sat, 16 Nov 2024 01:38:37 -0700 Subject: [PATCH 34/35] Limit jsapi data change event to prev and curr table sizes --- .../barrage/data/WebBarrageSubscription.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java index a2e40026fd9..b0cbc415824 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java @@ -463,7 +463,7 @@ private void freeRows(RangeSet removed) { } public static class ViewportImpl extends WebBarrageSubscription { - private long lastTableSize = 0; + private long tableSize = 0; public ViewportImpl(ClientTableState state, ViewportChangedHandler viewportChangedHandler, DataChangedHandler dataChangedHandler, WebColumnData[] dataSinks) { @@ -473,22 +473,23 @@ public ViewportImpl(ClientTableState state, ViewportChangedHandler viewportChang @Override public long getCurrentSize() { - return lastTableSize; + return tableSize; } @Override public RangeSet getCurrentRowSet() { - if (lastTableSize <= 0) { + if (tableSize <= 0) { return RangeSet.empty(); } - return RangeSet.ofRange(0, lastTableSize - 1); + return RangeSet.ofRange(0, tableSize - 1); } @Override public void applyUpdates(WebBarrageMessage message) { final BitSet prevServerColumns = serverColumns == null ? null : (BitSet) serverColumns.clone(); assert message.tableSize >= 0; - lastTableSize = message.tableSize; + final long prevTableSize = tableSize; + tableSize = message.tableSize; final RangeSet prevServerViewport = serverViewport.copy(); if (message.isSnapshot) { @@ -543,11 +544,16 @@ public void applyUpdates(WebBarrageMessage message) { } state.setSize(message.tableSize); + final RangeSet rowsAdded = serverViewport == null ? RangeSet.empty() : serverViewport.copy(); + if (!rowsAdded.isEmpty() && rowsAdded.getLastRow() >= tableSize) { + rowsAdded.removeRange(new Range(tableSize, rowsAdded.getLastRow())); + } + final RangeSet rowsRemoved = prevServerViewport == null ? RangeSet.empty() : prevServerViewport.copy(); + if (!rowsRemoved.isEmpty() && rowsRemoved.getLastRow() >= prevTableSize) { + rowsRemoved.removeRange(new Range(prevTableSize, rowsRemoved.getLastRow())); + } dataChangedHandler.onDataChanged( - serverViewport == null ? RangeSet.empty() : serverViewport.copy(), - prevServerViewport == null ? RangeSet.empty() : prevServerViewport.copy(), - RangeSet.empty(), - new ShiftedRange[0], + rowsAdded, rowsRemoved, RangeSet.empty(), new ShiftedRange[0], serverColumns == null ? null : (BitSet) serverColumns.clone()); } From 44fadff32d453a7b279249e1f6c0d7b05af0ed2a Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 18 Nov 2024 08:55:39 -0700 Subject: [PATCH 35/35] Colin's Final Feedback --- .../client/api/barrage/data/WebBarrageSubscription.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java index b0cbc415824..03e20646d68 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/barrage/data/WebBarrageSubscription.java @@ -79,11 +79,10 @@ public static WebBarrageSubscription subscribe( if (cts.getTableDef().getAttributes().isBlinkTable()) { return new BlinkImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); - } else if (subscriptionType == SubscriptionType.FULL_SUBSCRIPTION - || subscriptionType == SubscriptionType.SNAPSHOT) { - return new RedirectedImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); - } else { + } else if (subscriptionType == SubscriptionType.VIEWPORT_SUBSCRIPTION) { return new ViewportImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); + } else { + return new RedirectedImpl(cts, viewportChangedHandler, dataChangedHandler, dataSinks); } }