chore: collect server statistics only on server side

Server side statistics are not sent to the browser anymore
but directly registered when dev-tools websocket connection
is opened and closed.

Fixes vaadin/platform#6724

flow-server/src/main/java/com/vaadin/flow/internal/UsageStatisticsExporter.java

@@ -17,21 +17,20 @@
 package com.vaadin.flow.internal;
 
 import java.io.Serializable;
-import java.util.stream.Collectors;
 
 import org.jsoup.nodes.Document;
 
-import elemental.json.Json;
-import elemental.json.JsonObject;
-
 /**
  * A class for exporting {@link UsageStatistics} entries.
  * <p>
  * For internal use only. May be renamed or removed in a future release.
  *
  * @author Vaadin Ltd
  * @since 3.0
+ * @deprecated server side statistics should not be sent to the client. Will be
+ *             removed without replacement.
  */
+@Deprecated(since = "24.5", forRemoval = true)
 public class UsageStatisticsExporter implements Serializable {
 
     /**
@@ -40,29 +39,14 @@ public class UsageStatisticsExporter implements Serializable {
      *
      * @param document
      *            the document where the statistic entries to be exported to.
+     * @deprecated server side statistics should not be sent to the client. The
+     *             method throws an exception if called. Will be removed without
+     *             replacement.
      */
+    @Deprecated
     public static void exportUsageStatisticsToDocument(Document document) {
-        String entries = UsageStatistics.getEntries()
-                .map(UsageStatisticsExporter::createUsageStatisticsJson)
-                .collect(Collectors.joining(","));
-
-        if (!entries.isEmpty()) {
-            // Registers the entries in a way that is picked up as a Vaadin
-            // WebComponent by the usage stats gatherer
-            String builder = "window.Vaadin = window.Vaadin || {};\n"
-                    + "window.Vaadin.registrations = window.Vaadin.registrations || [];\n"
-                    + "window.Vaadin.registrations.push(" + entries + ");";
-            document.body().appendElement("script").text(builder);
-        }
+        throw new UnsupportedOperationException(
+                "Server side usage statistics must not be exported to the client");
     }
 
-    private static String createUsageStatisticsJson(
-            UsageStatistics.UsageEntry entry) {
-        JsonObject json = Json.createObject();
-
-        json.put("is", entry.getName());
-        json.put("version", entry.getVersion());
-
-        return json.toJson();
-    }
 }

flow-server/src/main/java/com/vaadin/flow/server/BootstrapHandler.java

@@ -749,8 +749,6 @@ public Document getBootstrapPage(BootstrapContext context) {
                             document.body(), targets));
 
             if (!config.isProductionMode()) {
-                UsageStatisticsExporter
-                        .exportUsageStatisticsToDocument(document);
                 IndexHtmlRequestHandler.addLicenseChecker(document);
             }
 

flow-server/src/main/java/com/vaadin/flow/server/communication/IndexHtmlRequestHandler.java

@@ -145,11 +145,6 @@ public boolean synchronizedHandleRequest(VaadinSession session,
         VaadinContext context = session.getService().getContext();
         AppShellRegistry registry = AppShellRegistry.getInstance(context);
 
-        if (!config.isProductionMode()) {
-            UsageStatisticsExporter
-                    .exportUsageStatisticsToDocument(indexDocument);
-        }
-
         // modify the page based on the @PWA annotation
         setupPwa(indexDocument, session.getService());
 

flow-server/src/main/java/com/vaadin/flow/server/communication/JavaScriptBootstrapHandler.java

@@ -217,22 +217,6 @@ protected static String getServiceUrl(VaadinRequest vaadinRequest) {
         return BootstrapHandlerHelper.getServiceUrl(vaadinRequest);
     }
 
-    private JsonObject getStats() {
-        JsonObject stats = Json.createObject();
-        UsageStatistics.getEntries().forEach(entry -> {
-            String name = entry.getName();
-            String version = entry.getVersion();
-
-            JsonObject json = Json.createObject();
-            json.put("is", name);
-            json.put("version", version);
-
-            String escapedName = Json.create(name).toJson();
-            stats.put(escapedName, json);
-        });
-        return stats;
-    }
-
     private JsonValue getErrors(VaadinService service) {
         JsonObject errors = Json.createObject();
         Optional<DevModeHandler> devModeHandler = DevModeHandlerManager
@@ -287,9 +271,6 @@ protected JsonObject getInitialJson(VaadinRequest request,
         if (context.getPushMode().isEnabled()) {
             initial.put("pushScript", getPushScript(context));
         }
-        if (!session.getConfiguration().isProductionMode()) {
-            initial.put("stats", getStats());
-        }
         initial.put("errors", getErrors(request.getService()));
 
         return initial;

flow-server/src/test/java/com/vaadin/flow/server/communication/IndexHtmlRequestHandlerTest.java

@@ -681,60 +681,12 @@ public void should_add_elements_when_appShellWithConfigurator()
         Elements bodyInlineElements = document.body()
                 .getElementsByTag("script");
         // <script>window.Vaadin = window.Vaadin || {};window.Vaadin
-        // .registrations = window.Vaadin.registrations ||
-        // [];window.Vaadin.registrations.push({"is":"java","version":"17.0.2"});
+        // .registrations = window.Vaadin.registrations || [];
         // </script>"
         // <script type="text/javascript">window.messages = window.messages
         // || [];window.messages.push("inline.js");
         // </script>"
-        assertEquals(2, bodyInlineElements.size());
-    }
-
-    @Test
-    public void should_export_usage_statistics_in_development_mode()
-            throws IOException {
-        File projectRootFolder = temporaryFolder.newFolder();
-        TestUtil.createIndexHtmlStub(projectRootFolder);
-        TestUtil.createStatsJsonStub(projectRootFolder);
-        deploymentConfiguration.setProductionMode(false);
-        deploymentConfiguration.setProjectFolder(projectRootFolder);
-        VaadinServletRequest request = createVaadinRequest("/");
-        Mockito.when(request.getHttpServletRequest().getRemoteAddr())
-                .thenReturn("127.0.0.1");
-        indexHtmlRequestHandler.synchronizedHandleRequest(session, request,
-                response);
-
-        String indexHtml = responseOutput.toString(StandardCharsets.UTF_8);
-        Document document = Jsoup.parse(indexHtml);
-
-        Elements bodyInlineElements = document.body()
-                .getElementsByTag("script");
-        // <script>window.Vaadin = window.Vaadin || {};
-        // window.Vaadin.registrations = window.Vaadin.registrations || [];
-        // window.Vaadin.registrations.push({"is":"java","version":"17.0.2"});
-        // </script>
         assertEquals(1, bodyInlineElements.size());
-
-        String entries = UsageStatistics.getEntries().map(entry -> {
-            JsonObject json = Json.createObject();
-
-            json.put("is", entry.getName());
-            json.put("version", entry.getVersion());
-
-            return json.toString();
-        }).collect(Collectors.joining(","));
-
-        String expected = StringUtil
-                .normaliseWhitespace("window.Vaadin = window.Vaadin || {}; "
-                        + "window.Vaadin.registrations = window.Vaadin.registrations || [];\n"
-                        + "window.Vaadin.registrations.push(" + entries + ");");
-
-        assertTrue(isTokenPresent(indexHtml));
-
-        String htmlContent = bodyInlineElements.get(0).childNode(0).outerHtml();
-        htmlContent = htmlContent.replace("\r", "");
-        htmlContent = htmlContent.replace("\n", " ");
-        assertEquals(StringUtil.normaliseWhitespace(expected), htmlContent);
     }
 
     // Regular expression to match a UUID in the format 8-4-4-4-12

flow-server/src/test/java/com/vaadin/flow/server/communication/JavaScriptBootstrapHandlerTest.java

@@ -102,7 +102,6 @@ public void should_produceValidJsonResponse() throws Exception {
 
         JsonObject json = Json.parse(response.getPayload());
 
-        Assert.assertTrue(json.hasKey("stats"));
         Assert.assertTrue(json.hasKey("errors"));
         Assert.assertTrue(json.hasKey("appConfig"));
         Assert.assertTrue(json.getObject("appConfig").hasKey("appId"));

vaadin-dev-server/src/main/java/com/vaadin/base/devserver/DebugWindowConnection.java

@@ -186,6 +186,7 @@ public void onConnect(AtmosphereResource resource) {
         if (DevToolsToken.getToken()
                 .equals(resource.getRequest().getParameter("token"))) {
             handleConnect(resource);
+            DevModeUsageStatistics.handleServerData();
         } else {
             getLogger().debug(
                     "Connection denied because of a missing or invalid token. Either the host is not on the 'vaadin.devmode.hosts-allowed' list or it is using an outdated token");
@@ -243,6 +244,7 @@ public void onDisconnect(AtmosphereResource resource) {
                     "Push connection {} is not a live-reload connection or already closed",
                     uuid);
         }
+        DevModeUsageStatistics.handleServerData();
     }
 
     @Override

vaadin-dev-server/src/main/java/com/vaadin/base/devserver/stats/DevModeUsageStatistics.java

@@ -17,12 +17,15 @@
 package com.vaadin.base.devserver.stats;
 
 import java.io.File;
+import java.util.List;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.vaadin.base.devserver.ServerInfo;
+import com.vaadin.flow.internal.UsageStatistics;
 import com.vaadin.flow.server.Version;
 import com.vaadin.pro.licensechecker.MachineId;
 
@@ -156,7 +159,21 @@ public static void handleBrowserData(JsonObject data) {
                         .readTree(json);
                 if (clientData != null && clientData.isObject()) {
                     clientData.fields().forEachRemaining(
-                            e -> project.setValue(e.getKey(), e.getValue()));
+                            e -> project.computeValue(e.getKey(), jsonNode -> {
+                                ObjectNode container;
+                                if (jsonNode == null || !jsonNode.isObject()) {
+                                    container = JsonHelpers.getJsonMapper()
+                                            .createObjectNode();
+                                } else {
+                                    container = (ObjectNode) jsonNode;
+                                }
+                                // Replace exising entries with data coming from
+                                // the browser, preserving server side entries
+                                if (e.getValue() instanceof ObjectNode newData) {
+                                    container.setAll(newData);
+                                }
+                                return container;
+                            }));
                 }
             } catch (Exception e) {
                 getLogger().debug("Failed to update client telemetry data", e);
@@ -165,6 +182,50 @@ public static void handleBrowserData(JsonObject data) {
 
     }
 
+    /**
+     * Stores telemetry data collected on the server.
+     */
+    public static void handleServerData() {
+        if (!isStatisticsEnabled()) {
+            return;
+        }
+        getLogger().debug("Persisting server side usage statistics");
+        List<UsageStatistics.UsageEntry> entries = UsageStatistics.getEntries()
+                .toList();
+        get().storage.update((global, project) -> {
+            try {
+                project.computeValue("elements", jsonNode -> {
+                    ObjectNode elements;
+                    if (jsonNode == null || !jsonNode.isObject()) {
+                        elements = JsonHelpers.getJsonMapper()
+                                .createObjectNode();
+                    } else {
+                        elements = (ObjectNode) jsonNode;
+                    }
+                    for (UsageStatistics.UsageEntry entry : entries) {
+                        if (elements.get(entry
+                                .getName()) instanceof ObjectNode jsonEntry) {
+                            jsonEntry.put("lastUsed",
+                                    System.currentTimeMillis());
+                        } else {
+                            ObjectNode jsonEntry = JsonHelpers.getJsonMapper()
+                                    .createObjectNode();
+                            jsonEntry.put("version", entry.getVersion());
+                            jsonEntry.put("firstUsed",
+                                    System.currentTimeMillis());
+                            elements.set(entry.getName(), jsonEntry);
+                        }
+                    }
+                    return elements;
+                });
+            } catch (Exception e) {
+                getLogger().debug(
+                        "Failed to update server side usage statistics data",
+                        e);
+            }
+        });
+    }
+
     /**
      * Checks if usage statistic collection is currently enabled.
      *

vaadin-dev-server/src/main/java/com/vaadin/base/devserver/stats/StatisticsContainer.java

@@ -15,6 +15,8 @@
  */
 package com.vaadin.base.devserver.stats;
 
+import java.util.function.UnaryOperator;
+
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
@@ -166,4 +168,21 @@ public double getValueAsDouble(String name) {
         return 0.0;
     }
 
+    /**
+     * Stores or updates a JSON object using the given field name.
+     * <p>
+     * </p>
+     * The mapping function is given the existing JSON object, or
+     * {@literal null} if field is not yet present in the storage.
+     *
+     * @param name
+     *            name of the field to set or update
+     * @param mappingFunction
+     *            the mapping function to compute a value
+     */
+    public void computeValue(String name,
+            UnaryOperator<JsonNode> mappingFunction) {
+        json.set(name, mappingFunction.apply(json.get(name)));
+    }
+
 }

vaadin-dev-server/src/main/java/com/vaadin/base/devserver/stats/StatisticsStorage.java

@@ -18,6 +18,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.FileSystems;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;