From 5d6192c48cccbd0db6a52dc51028ad48e0abd487 Mon Sep 17 00:00:00 2001 From: Nathan Xu Date: Fri, 18 Jul 2025 11:26:08 -0400 Subject: [PATCH 1/4] sunset Astrolabe --- .../com/mongodb/client/unified/Entities.java | 216 +----------------- .../mongodb/client/unified/UnifiedTest.java | 68 ------ driver-workload-executor/build.gradle.kts | 57 ----- .../mongodb/workload/WorkloadExecutor.java | 181 --------------- .../src/resources/logback.xml | 12 - settings.gradle.kts | 1 - 6 files changed, 2 insertions(+), 533 deletions(-) delete mode 100644 driver-workload-executor/build.gradle.kts delete mode 100644 driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java delete mode 100644 driver-workload-executor/src/resources/logback.xml diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java b/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java index f1429431690..5a948c054a1 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java @@ -37,25 +37,6 @@ import com.mongodb.client.vault.ClientEncryption; import com.mongodb.connection.ClusterConnectionMode; import com.mongodb.connection.ClusterDescription; -import com.mongodb.connection.ConnectionId; -import com.mongodb.connection.ServerId; -import com.mongodb.event.CommandEvent; -import com.mongodb.event.CommandFailedEvent; -import com.mongodb.event.CommandListener; -import com.mongodb.event.CommandStartedEvent; -import com.mongodb.event.CommandSucceededEvent; -import com.mongodb.event.ConnectionCheckOutFailedEvent; -import com.mongodb.event.ConnectionCheckOutStartedEvent; -import com.mongodb.event.ConnectionCheckedInEvent; -import com.mongodb.event.ConnectionCheckedOutEvent; -import com.mongodb.event.ConnectionClosedEvent; -import com.mongodb.event.ConnectionCreatedEvent; -import com.mongodb.event.ConnectionPoolClearedEvent; -import com.mongodb.event.ConnectionPoolClosedEvent; -import com.mongodb.event.ConnectionPoolCreatedEvent; -import com.mongodb.event.ConnectionPoolListener; -import com.mongodb.event.ConnectionPoolReadyEvent; -import com.mongodb.event.ConnectionReadyEvent; import com.mongodb.event.TestServerMonitorListener; import com.mongodb.internal.connection.ServerMonitoringModeUtil; import com.mongodb.internal.connection.TestClusterListener; @@ -63,15 +44,12 @@ import com.mongodb.internal.connection.TestConnectionPoolListener; import com.mongodb.internal.connection.TestServerListener; import com.mongodb.internal.logging.LogMessage; -import com.mongodb.lang.NonNull; import com.mongodb.lang.Nullable; import com.mongodb.logging.TestLoggingInterceptor; import org.bson.BsonArray; import org.bson.BsonBoolean; import org.bson.BsonDocument; -import org.bson.BsonDouble; import org.bson.BsonInt32; -import org.bson.BsonInt64; import org.bson.BsonString; import org.bson.BsonValue; @@ -97,7 +75,6 @@ import static com.mongodb.assertions.Assertions.notNull; import static com.mongodb.client.Fixture.getMongoClientSettingsBuilder; import static com.mongodb.client.Fixture.getMultiMongosMongoClientSettingsBuilder; -import static com.mongodb.client.unified.EventMatcher.getReasonString; import static com.mongodb.client.unified.UnifiedClientEncryptionHelper.createKmsProvidersMap; import static com.mongodb.client.unified.UnifiedCrudHelper.asReadConcern; import static com.mongodb.client.unified.UnifiedCrudHelper.asReadPreference; @@ -105,14 +82,13 @@ import static com.mongodb.internal.connection.AbstractConnectionPoolTest.waitForPoolAsyncWorkManagerStart; import static java.lang.System.getenv; import static java.util.Arrays.asList; -import static java.util.Collections.synchronizedList; import static org.junit.Assume.assumeTrue; public final class Entities { private static final Set SUPPORTED_CLIENT_ENTITY_OPTIONS = new HashSet<>( asList( - "id", "uriOptions", "serverApi", "useMultipleMongoses", "storeEventsAsEntities", - "observeEvents", "observeLogMessages", "observeSensitiveCommands", "ignoreCommandMonitoringEvents")); + "id", "uriOptions", "serverApi", "useMultipleMongoses", "observeEvents", + "observeLogMessages", "observeSensitiveCommands", "ignoreCommandMonitoringEvents")); private final Set entityNames = new HashSet<>(); private final Map threads = new HashMap<>(); private final Map>> tasks = new HashMap<>(); @@ -136,7 +112,6 @@ public final class Entities { private final Map iterationCounts = new HashMap<>(); private final Map errorDocumentsMap = new HashMap<>(); private final Map failureDocumentsMap = new HashMap<>(); - private final Map> eventsMap = new HashMap<>(); public boolean hasSuccessCount(final String id) { return successCounts.containsKey(id); @@ -186,14 +161,6 @@ public BsonArray getFailureDocuments(final String id) { return getEntity(id, failureDocumentsMap, "failureDocuments"); } - public boolean hasEvents(final String id) { - return eventsMap.containsKey(id); - } - - public List getEvents(final String id) { - return getEntity(id, eventsMap, "events"); - } - public void addResult(final String id, final BsonValue result) { putEntity(id, result, results); } @@ -433,35 +400,6 @@ private void initClient(final BsonDocument entity, final String id, putEntity(id + "-connection-pool-listener", testConnectionPoolListener, clientConnectionPoolListeners); } - if (entity.containsKey("storeEventsAsEntities")) { - BsonArray storeEventsAsEntitiesArray = entity.getArray("storeEventsAsEntities"); - for (BsonValue eventValue : storeEventsAsEntitiesArray) { - BsonDocument eventDocument = eventValue.asDocument(); - String key = eventDocument.getString("id").getValue(); - BsonArray eventList = eventDocument.getArray("events"); - List eventDocumentList = synchronizedList(new ArrayList<>()); - putEntity(key, eventDocumentList, eventsMap); - - if (eventList.stream() - .map(value -> value.asString().getValue()) - .anyMatch(value -> value.startsWith("Command"))) { - clientSettingsBuilder.addCommandListener(new EntityCommandListener(eventList.stream() - .map(value -> value.asString().getValue()) - .collect(Collectors.toSet()), - eventDocumentList)); - } - if (eventList.stream() - .map(value -> value.asString().getValue()) - .anyMatch(value -> value.startsWith("Pool") || value.startsWith("Connection"))) { - clientSettingsBuilder. - applyToConnectionPoolSettings(builder -> - builder.addConnectionPoolListener(new EntityConnectionPoolListener(eventList.stream() - .map(value -> value.asString().getValue()) - .collect(Collectors.toSet()), - eventDocumentList))); - } - } - } clientSettingsBuilder.applyToServerSettings(builder -> { builder.heartbeatFrequency(50, TimeUnit.MILLISECONDS); builder.minHeartbeatFrequency(50, TimeUnit.MILLISECONDS); @@ -795,154 +733,4 @@ public void close() { clientLoggingInterceptors.values().forEach(TestLoggingInterceptor::close); threads.values().forEach(ExecutorService::shutdownNow); } - - private static class EntityCommandListener implements CommandListener { - private final List eventDocumentList; - private final Set enabledEvents; - - EntityCommandListener(final Set enabledEvents, final List eventDocumentList) { - this.eventDocumentList = eventDocumentList; - this.enabledEvents = enabledEvents; - } - - @Override - public void commandStarted(final CommandStartedEvent event) { - if (enabledEvents.contains("CommandStartedEvent")) { - eventDocumentList.add(createEventDocument(event, "CommandStartedEvent") - .append("databaseName", new BsonString(event.getDatabaseName()))); - } - } - - @Override - public void commandSucceeded(final CommandSucceededEvent event) { - if (enabledEvents.contains("CommandSucceededEvent")) { - eventDocumentList.add(createEventDocument(event, "CommandSucceededEvent") - .append("duration", new BsonInt64(event.getElapsedTime(TimeUnit.MILLISECONDS)))); - } - } - - @Override - public void commandFailed(final CommandFailedEvent event) { - if (enabledEvents.contains("CommandFailedEvent")) { - eventDocumentList.add(createEventDocument(event, "CommandFailedEvent") - .append("duration", - new BsonDouble(event.getElapsedTime(TimeUnit.NANOSECONDS) / 1_000_000_000.0)) - .append("failure", new BsonString(event.getThrowable().toString()))); - } - } - - private BsonDocument createEventDocument(final CommandEvent event, final String name) { - return new BsonDocument() - .append("name", new BsonString(name)) - .append("observedAt", new BsonDouble(System.currentTimeMillis() / 1000.0)) - .append("commandName", new BsonString(event.getCommandName())) - .append("requestId", new BsonInt32(event.getRequestId())); - } - } - - private static class EntityConnectionPoolListener implements ConnectionPoolListener { - private final List eventDocumentList; - private final Set enabledEvents; - - EntityConnectionPoolListener(final Set enabledEvents, final List eventDocumentList) { - this.eventDocumentList = eventDocumentList; - this.enabledEvents = enabledEvents; - } - - @Override - public void connectionPoolCreated(final ConnectionPoolCreatedEvent event) { - if (enabledEvents.contains("PoolCreatedEvent")) { - eventDocumentList.add(createEventDocument("PoolCreatedEvent", event.getServerId())); - } - } - - @Override - public void connectionPoolCleared(final ConnectionPoolClearedEvent event) { - if (enabledEvents.contains("PoolClearedEvent")) { - eventDocumentList.add(createEventDocument("PoolClearedEvent", event.getServerId())); - } - } - - @Override - public void connectionPoolReady(final ConnectionPoolReadyEvent event) { - if (enabledEvents.contains("PoolReadyEvent")) { - eventDocumentList.add(createEventDocument("PoolReadyEvent", event.getServerId())); - } - } - - @Override - public void connectionPoolClosed(final ConnectionPoolClosedEvent event) { - if (enabledEvents.contains("PoolClosedEvent")) { - eventDocumentList.add(createEventDocument("PoolClosedEvent", event.getServerId())); - } - } - - @Override - public void connectionCheckOutStarted(final ConnectionCheckOutStartedEvent event) { - if (enabledEvents.contains("ConnectionCheckOutStartedEvent")) { - eventDocumentList.add(createEventDocument("ConnectionCheckOutStartedEvent", event.getServerId())); - } - } - - @Override - public void connectionCheckedOut(final ConnectionCheckedOutEvent event) { - if (enabledEvents.contains("ConnectionCheckedOutEvent")) { - eventDocumentList.add(createEventDocument("ConnectionCheckedOutEvent", event.getConnectionId())); - } - } - - @Override - public void connectionCheckOutFailed(final ConnectionCheckOutFailedEvent event) { - if (enabledEvents.contains("ConnectionCheckOutFailedEvent")) { - eventDocumentList.add(createEventDocument("ConnectionCheckOutFailedEvent", event.getServerId()) - .append("reason", new BsonString(getReasonString(event.getReason())))); - } - } - - @Override - public void connectionCheckedIn(final ConnectionCheckedInEvent event) { - if (enabledEvents.contains("ConnectionCheckedInEvent")) { - eventDocumentList.add(createEventDocument("ConnectionCheckedInEvent", event.getConnectionId())); - } - } - - @Override - public void connectionCreated(final ConnectionCreatedEvent event) { - if (enabledEvents.contains("ConnectionCreatedEvent")) { - eventDocumentList.add(createEventDocument("ConnectionCreatedEvent", event.getConnectionId())); - } - } - - @Override - public void connectionReady(final ConnectionReadyEvent event) { - if (enabledEvents.contains("ConnectionReadyEvent")) { - eventDocumentList.add(createEventDocument("ConnectionReadyEvent", event.getConnectionId())); - } - } - - @Override - public void connectionClosed(final ConnectionClosedEvent event) { - if (enabledEvents.contains("ConnectionClosedEvent")) { - eventDocumentList.add(createEventDocument("ConnectionClosedEvent", event.getConnectionId()) - .append("reason", new BsonString(getReasonString(event.getReason())))); - } - } - - private BsonDocument createEventDocument(final String name, final ConnectionId connectionId) { - return createEventDocument(name, connectionId.getServerId()) - .append("connectionId", new BsonString(Long.toString(connectionId.getLocalValue()))); - } - - private BsonDocument createEventDocument(final String name, final ServerId serverId) { - return new BsonDocument() - .append("name", new BsonString(name)) - .append("observedAt", new BsonDouble(System.currentTimeMillis() / 1000.0)) - .append("address", new BsonString(getAddressAsString(serverId))); - } - - @NonNull - private String getAddressAsString(final ServerId serverId) { - return serverId.getAddress().getHost() + ":" + serverId.getAddress().getPort(); - } - } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index e067e36d993..693643336ef 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -47,7 +47,6 @@ import org.bson.BsonArray; import org.bson.BsonBoolean; import org.bson.BsonDocument; -import org.bson.BsonDouble; import org.bson.BsonInt32; import org.bson.BsonString; import org.bson.BsonValue; @@ -699,8 +698,6 @@ private OperationResult executeOperation(final UnifiedTestContext context, final return gridFSHelper.executeUpload(operation); case "runCommand": return crudHelper.executeRunCommand(operation); - case "loop": - return loop(context, operation); case "createDataKey": return clientEncryptionHelper.executeCreateDataKey(operation); case "addKeyAltName": @@ -731,71 +728,6 @@ private OperationResult executeOperation(final UnifiedTestContext context, final } } - private OperationResult loop(final UnifiedTestContext context, final BsonDocument operation) { - BsonDocument arguments = operation.getDocument("arguments"); - - int numIterations = 0; - int numSuccessfulOperations = 0; - boolean storeFailures = arguments.containsKey("storeFailuresAsEntity"); - boolean storeErrors = arguments.containsKey("storeErrorsAsEntity"); - BsonArray failureDescriptionDocuments = new BsonArray(); - BsonArray errorDescriptionDocuments = new BsonArray(); - - while (!terminateLoop()) { - BsonArray array = arguments.getArray("operations"); - for (int i = 0; i < array.size(); i++) { - BsonValue cur = array.get(i); - try { - assertOperation(context, cur.asDocument().clone(), i); - numSuccessfulOperations++; - } catch (AssertionError e) { - if (storeFailures) { - failureDescriptionDocuments.add(createDocumentFromException(e)); - } else if (storeErrors) { - errorDescriptionDocuments.add(createDocumentFromException(e)); - } else { - throw e; - } - break; - } catch (Exception e) { - if (storeErrors) { - errorDescriptionDocuments.add(createDocumentFromException(e)); - } else if (storeFailures) { - failureDescriptionDocuments.add(createDocumentFromException(e)); - } else { - throw e; - } - break; - } - } - numIterations++; - } - - if (arguments.containsKey("storeSuccessesAsEntity")) { - entities.addSuccessCount(arguments.getString("storeSuccessesAsEntity").getValue(), numSuccessfulOperations); - } - if (arguments.containsKey("storeIterationsAsEntity")) { - entities.addIterationCount(arguments.getString("storeIterationsAsEntity").getValue(), numIterations); - } - if (storeFailures) { - entities.addFailureDocuments(arguments.getString("storeFailuresAsEntity").getValue(), failureDescriptionDocuments); - } - if (storeErrors) { - entities.addErrorDocuments(arguments.getString("storeErrorsAsEntity").getValue(), errorDescriptionDocuments); - } - - return OperationResult.NONE; - } - - private BsonDocument createDocumentFromException(final Throwable throwable) { - return new BsonDocument("error", new BsonString(throwable.toString())) - .append("time", new BsonDouble(System.currentTimeMillis() / 1000.0)); - } - - protected boolean terminateLoop() { - return true; - } - private OperationResult executeCreateEntities(final BsonDocument operation) { entities.init(operation.getDocument("arguments").getArray("entities"), startingClusterTime, diff --git a/driver-workload-executor/build.gradle.kts b/driver-workload-executor/build.gradle.kts deleted file mode 100644 index 2cb5f2e1073..00000000000 --- a/driver-workload-executor/build.gradle.kts +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2008-present MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { - id("application") - id("java-library") - id("project.base") - id("conventions.test-artifacts") -} - -application { - mainClass = "com.mongodb.workload.WorkloadExecutor" -} - -sourceSets { - main { - java { setSrcDirs(listOf("src/main")) } - resources { setSrcDirs(listOf("src/resources")) } - } -} - -dependencies { - implementation(project(":driver-sync")) - implementation(project(path = ":driver-core", configuration = "testArtifacts")) - implementation(project(path = ":driver-sync", configuration = "testArtifacts")) - implementation(platform(libs.junit.bom)) - implementation(libs.bundles.junit.vintage) -} - -tasks.withType().configureEach { - enabled = false -} - -java { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 -} - - -tasks.withType { - manifest { - attributes["Main-Class"] = "com.mongodb.workload.WorkloadExecutor" - } -} diff --git a/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java b/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java deleted file mode 100644 index 7aba736aebc..00000000000 --- a/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright 2008-present MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.mongodb.workload; - -import com.mongodb.client.unified.Entities; -import com.mongodb.client.unified.UnifiedSyncTest; -import com.mongodb.client.unified.UnifiedTest; -import com.mongodb.internal.diagnostics.logging.Logger; -import com.mongodb.internal.diagnostics.logging.Loggers; -import org.bson.BsonArray; -import org.bson.BsonDocument; -import org.bson.BsonInt64; -import org.bson.codecs.BsonDocumentCodec; -import org.bson.codecs.DecoderContext; -import org.bson.json.JsonReader; -import org.bson.json.JsonWriterSettings; - -import java.io.FileReader; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -public class WorkloadExecutor { - private static final Logger LOGGER = Loggers.getLogger("workload-executor"); - private static final CountDownLatch terminationLatch = new CountDownLatch(1); - private static volatile boolean terminateLoop; - - public static void main(String[] args) throws IOException { - if (args.length != 2) { - System.out.println("Usage: AstrolabeTestRunner "); - System.exit(1); - } - - String pathToWorkloadFile = args[0]; - String pathToResultsDirectory = args[1]; - - LOGGER.info("Max memory (GB): " + (Runtime.getRuntime().maxMemory() / 1_073_741_824.0)); - LOGGER.info("Path to workload file: '" + pathToWorkloadFile + "'"); - LOGGER.info("Path to results directory: '" + pathToResultsDirectory + "'"); - - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - LOGGER.info("Running shutdown hook"); - terminateLoop = true; - try { - if (!terminationLatch.await(1, TimeUnit.MINUTES)) { - LOGGER.warn("Terminating after waiting for 1 minute for results to be written"); - } else { - LOGGER.info("Terminating."); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - })); - - - BsonDocument fileDocument; - - try (FileReader reader = new FileReader(pathToWorkloadFile)) { - fileDocument = new BsonDocumentCodec().decode(new JsonReader(reader), DecoderContext.builder().build()); - } - - LOGGER.info("Executing workload: " + fileDocument.toJson(JsonWriterSettings.builder().indent(true).build())); - - BsonArray testArray = fileDocument.getArray("tests"); - if (testArray.size() != 1) { - throw new IllegalArgumentException("Expected exactly one test"); - } - BsonDocument testDocument = testArray.get(0).asDocument(); - - UnifiedTest unifiedTest = new UnifiedSyncTest() { - @Override - protected boolean terminateLoop() { - return terminateLoop; - } - }; - - try { - String schemaVersion = fileDocument.getString("schemaVersion").getValue(); - BsonArray runOnRequirements = fileDocument.getArray("runOnRequirements", null); - BsonArray createEntities = fileDocument.getArray("createEntities", new BsonArray()); - BsonArray initialData = fileDocument.getArray("initialData", new BsonArray()); - unifiedTest.setUp( - "", - null, - null, - null, - 1, - 1, - schemaVersion, - runOnRequirements, - createEntities, - initialData, - testDocument); - unifiedTest.shouldPassAllOutcomes( - "", - null, - null, - null, - 1, - 1, - schemaVersion, - runOnRequirements, - createEntities, - initialData, - testDocument); - Entities entities = unifiedTest.getEntities(); - - long iterationCount = -1; - if (entities.hasIterationCount("iterations")) { - iterationCount = entities.getIterationCount("iterations"); - } - - long successCount = -1; - if (entities.hasSuccessCount("successes")) { - successCount = entities.getSuccessCount("successes"); - } - - BsonArray errorDocuments = null; - long errorCount = 0; - if (entities.hasErrorDocuments("errors")) { - errorDocuments = entities.getErrorDocuments("errors"); - errorCount = errorDocuments.size(); - } - - BsonArray failureDocuments = null; - long failureCount = 0; - if (entities.hasFailureDocuments("failures")) { - failureDocuments = entities.getFailureDocuments("failures"); - failureCount = failureDocuments.size(); - } - - BsonArray eventDocuments = new BsonArray(); - if (entities.hasEvents("events")) { - eventDocuments = new BsonArray(entities.getEvents("events")); - } - - BsonDocument eventsDocument = new BsonDocument() - .append("errors", errorDocuments == null ? new BsonArray() : errorDocuments) - .append("failures", failureDocuments == null ? new BsonArray() : failureDocuments) - .append("events", eventDocuments); - - BsonDocument resultsDocument = new BsonDocument() - .append("numErrors", new BsonInt64(errorCount)) - .append("numFailures", new BsonInt64(failureCount)) - .append("numSuccesses", new BsonInt64(successCount)) - .append("numIterations", new BsonInt64(iterationCount)); - - writeFile(eventsDocument, Paths.get(pathToResultsDirectory, "events.json")); - writeFile(resultsDocument, Paths.get(pathToResultsDirectory, "results.json")); - } finally { - unifiedTest.cleanUp(); - terminationLatch.countDown(); - } - } - - private static void writeFile(final BsonDocument document, final Path path) throws IOException { - LOGGER.info("Writing file: '" + path.toFile().getAbsolutePath()); - Files.deleteIfExists(path); - String json = document.toJson(JsonWriterSettings.builder().indent(true).build()); - LOGGER.debug("File contents: " + json); - Files.write(path, (json + "\n").getBytes(StandardCharsets.UTF_8)); - } -} diff --git a/driver-workload-executor/src/resources/logback.xml b/driver-workload-executor/src/resources/logback.xml deleted file mode 100644 index 706bbca3534..00000000000 --- a/driver-workload-executor/src/resources/logback.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - diff --git a/settings.gradle.kts b/settings.gradle.kts index 019d3b8290d..29d17792ad4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -43,7 +43,6 @@ include(":driver-scala") include(":driver-benchmarks") include(":driver-lambda") -include(":driver-workload-executor") if (providers.gradleProperty("includeGraalvm").isPresent) { include(":graalvm-native-image-app") } From bd0643ea65533f8c4b74b9e187a3112a6d17f0cc Mon Sep 17 00:00:00 2001 From: Nathan Xu Date: Fri, 18 Jul 2025 15:02:33 -0400 Subject: [PATCH 2/4] experiment restoring driver-workload-executor --- .../com/mongodb/client/unified/Entities.java | 9 + .../mongodb/client/unified/UnifiedTest.java | 4 + driver-workload-executor/build.gradle.kts | 57 ++++++ .../mongodb/workload/WorkloadExecutor.java | 181 ++++++++++++++++++ .../src/resources/logback.xml | 12 ++ settings.gradle.kts | 1 + 6 files changed, 264 insertions(+) create mode 100644 driver-workload-executor/build.gradle.kts create mode 100644 driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java create mode 100644 driver-workload-executor/src/resources/logback.xml diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java b/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java index 5a948c054a1..1125c6cf3f0 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java @@ -112,6 +112,7 @@ public final class Entities { private final Map iterationCounts = new HashMap<>(); private final Map errorDocumentsMap = new HashMap<>(); private final Map failureDocumentsMap = new HashMap<>(); + private final Map> eventsMap = new HashMap<>(); public boolean hasSuccessCount(final String id) { return successCounts.containsKey(id); @@ -161,6 +162,14 @@ public BsonArray getFailureDocuments(final String id) { return getEntity(id, failureDocumentsMap, "failureDocuments"); } + public boolean hasEvents(final String id) { + return eventsMap.containsKey(id); + } + + public List getEvents(final String id) { + return getEntity(id, eventsMap, "events"); + } + public void addResult(final String id, final BsonValue result) { putEntity(id, result, results); } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index 693643336ef..3364cf4183d 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -728,6 +728,10 @@ private OperationResult executeOperation(final UnifiedTestContext context, final } } + protected boolean terminateLoop() { + return true; + } + private OperationResult executeCreateEntities(final BsonDocument operation) { entities.init(operation.getDocument("arguments").getArray("entities"), startingClusterTime, diff --git a/driver-workload-executor/build.gradle.kts b/driver-workload-executor/build.gradle.kts new file mode 100644 index 00000000000..2cb5f2e1073 --- /dev/null +++ b/driver-workload-executor/build.gradle.kts @@ -0,0 +1,57 @@ +/* + * Copyright 2008-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("application") + id("java-library") + id("project.base") + id("conventions.test-artifacts") +} + +application { + mainClass = "com.mongodb.workload.WorkloadExecutor" +} + +sourceSets { + main { + java { setSrcDirs(listOf("src/main")) } + resources { setSrcDirs(listOf("src/resources")) } + } +} + +dependencies { + implementation(project(":driver-sync")) + implementation(project(path = ":driver-core", configuration = "testArtifacts")) + implementation(project(path = ":driver-sync", configuration = "testArtifacts")) + implementation(platform(libs.junit.bom)) + implementation(libs.bundles.junit.vintage) +} + +tasks.withType().configureEach { + enabled = false +} + +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + + +tasks.withType { + manifest { + attributes["Main-Class"] = "com.mongodb.workload.WorkloadExecutor" + } +} diff --git a/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java b/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java new file mode 100644 index 00000000000..7aba736aebc --- /dev/null +++ b/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java @@ -0,0 +1,181 @@ +/* + * Copyright 2008-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.workload; + +import com.mongodb.client.unified.Entities; +import com.mongodb.client.unified.UnifiedSyncTest; +import com.mongodb.client.unified.UnifiedTest; +import com.mongodb.internal.diagnostics.logging.Logger; +import com.mongodb.internal.diagnostics.logging.Loggers; +import org.bson.BsonArray; +import org.bson.BsonDocument; +import org.bson.BsonInt64; +import org.bson.codecs.BsonDocumentCodec; +import org.bson.codecs.DecoderContext; +import org.bson.json.JsonReader; +import org.bson.json.JsonWriterSettings; + +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class WorkloadExecutor { + private static final Logger LOGGER = Loggers.getLogger("workload-executor"); + private static final CountDownLatch terminationLatch = new CountDownLatch(1); + private static volatile boolean terminateLoop; + + public static void main(String[] args) throws IOException { + if (args.length != 2) { + System.out.println("Usage: AstrolabeTestRunner "); + System.exit(1); + } + + String pathToWorkloadFile = args[0]; + String pathToResultsDirectory = args[1]; + + LOGGER.info("Max memory (GB): " + (Runtime.getRuntime().maxMemory() / 1_073_741_824.0)); + LOGGER.info("Path to workload file: '" + pathToWorkloadFile + "'"); + LOGGER.info("Path to results directory: '" + pathToResultsDirectory + "'"); + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + LOGGER.info("Running shutdown hook"); + terminateLoop = true; + try { + if (!terminationLatch.await(1, TimeUnit.MINUTES)) { + LOGGER.warn("Terminating after waiting for 1 minute for results to be written"); + } else { + LOGGER.info("Terminating."); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + })); + + + BsonDocument fileDocument; + + try (FileReader reader = new FileReader(pathToWorkloadFile)) { + fileDocument = new BsonDocumentCodec().decode(new JsonReader(reader), DecoderContext.builder().build()); + } + + LOGGER.info("Executing workload: " + fileDocument.toJson(JsonWriterSettings.builder().indent(true).build())); + + BsonArray testArray = fileDocument.getArray("tests"); + if (testArray.size() != 1) { + throw new IllegalArgumentException("Expected exactly one test"); + } + BsonDocument testDocument = testArray.get(0).asDocument(); + + UnifiedTest unifiedTest = new UnifiedSyncTest() { + @Override + protected boolean terminateLoop() { + return terminateLoop; + } + }; + + try { + String schemaVersion = fileDocument.getString("schemaVersion").getValue(); + BsonArray runOnRequirements = fileDocument.getArray("runOnRequirements", null); + BsonArray createEntities = fileDocument.getArray("createEntities", new BsonArray()); + BsonArray initialData = fileDocument.getArray("initialData", new BsonArray()); + unifiedTest.setUp( + "", + null, + null, + null, + 1, + 1, + schemaVersion, + runOnRequirements, + createEntities, + initialData, + testDocument); + unifiedTest.shouldPassAllOutcomes( + "", + null, + null, + null, + 1, + 1, + schemaVersion, + runOnRequirements, + createEntities, + initialData, + testDocument); + Entities entities = unifiedTest.getEntities(); + + long iterationCount = -1; + if (entities.hasIterationCount("iterations")) { + iterationCount = entities.getIterationCount("iterations"); + } + + long successCount = -1; + if (entities.hasSuccessCount("successes")) { + successCount = entities.getSuccessCount("successes"); + } + + BsonArray errorDocuments = null; + long errorCount = 0; + if (entities.hasErrorDocuments("errors")) { + errorDocuments = entities.getErrorDocuments("errors"); + errorCount = errorDocuments.size(); + } + + BsonArray failureDocuments = null; + long failureCount = 0; + if (entities.hasFailureDocuments("failures")) { + failureDocuments = entities.getFailureDocuments("failures"); + failureCount = failureDocuments.size(); + } + + BsonArray eventDocuments = new BsonArray(); + if (entities.hasEvents("events")) { + eventDocuments = new BsonArray(entities.getEvents("events")); + } + + BsonDocument eventsDocument = new BsonDocument() + .append("errors", errorDocuments == null ? new BsonArray() : errorDocuments) + .append("failures", failureDocuments == null ? new BsonArray() : failureDocuments) + .append("events", eventDocuments); + + BsonDocument resultsDocument = new BsonDocument() + .append("numErrors", new BsonInt64(errorCount)) + .append("numFailures", new BsonInt64(failureCount)) + .append("numSuccesses", new BsonInt64(successCount)) + .append("numIterations", new BsonInt64(iterationCount)); + + writeFile(eventsDocument, Paths.get(pathToResultsDirectory, "events.json")); + writeFile(resultsDocument, Paths.get(pathToResultsDirectory, "results.json")); + } finally { + unifiedTest.cleanUp(); + terminationLatch.countDown(); + } + } + + private static void writeFile(final BsonDocument document, final Path path) throws IOException { + LOGGER.info("Writing file: '" + path.toFile().getAbsolutePath()); + Files.deleteIfExists(path); + String json = document.toJson(JsonWriterSettings.builder().indent(true).build()); + LOGGER.debug("File contents: " + json); + Files.write(path, (json + "\n").getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/driver-workload-executor/src/resources/logback.xml b/driver-workload-executor/src/resources/logback.xml new file mode 100644 index 00000000000..706bbca3534 --- /dev/null +++ b/driver-workload-executor/src/resources/logback.xml @@ -0,0 +1,12 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + diff --git a/settings.gradle.kts b/settings.gradle.kts index 29d17792ad4..019d3b8290d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -43,6 +43,7 @@ include(":driver-scala") include(":driver-benchmarks") include(":driver-lambda") +include(":driver-workload-executor") if (providers.gradleProperty("includeGraalvm").isPresent) { include(":graalvm-native-image-app") } From 2993a68beffa6ee92f678138859d755c01aebb3e Mon Sep 17 00:00:00 2001 From: Nathan Xu Date: Fri, 1 Aug 2025 11:39:25 -0400 Subject: [PATCH 3/4] experiment deleting driver-workload-executor module --- driver-workload-executor/build.gradle.kts | 57 ------ .../mongodb/workload/WorkloadExecutor.java | 181 ------------------ .../src/resources/logback.xml | 12 -- settings.gradle.kts | 1 - 4 files changed, 251 deletions(-) delete mode 100644 driver-workload-executor/build.gradle.kts delete mode 100644 driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java delete mode 100644 driver-workload-executor/src/resources/logback.xml diff --git a/driver-workload-executor/build.gradle.kts b/driver-workload-executor/build.gradle.kts deleted file mode 100644 index 2cb5f2e1073..00000000000 --- a/driver-workload-executor/build.gradle.kts +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2008-present MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { - id("application") - id("java-library") - id("project.base") - id("conventions.test-artifacts") -} - -application { - mainClass = "com.mongodb.workload.WorkloadExecutor" -} - -sourceSets { - main { - java { setSrcDirs(listOf("src/main")) } - resources { setSrcDirs(listOf("src/resources")) } - } -} - -dependencies { - implementation(project(":driver-sync")) - implementation(project(path = ":driver-core", configuration = "testArtifacts")) - implementation(project(path = ":driver-sync", configuration = "testArtifacts")) - implementation(platform(libs.junit.bom)) - implementation(libs.bundles.junit.vintage) -} - -tasks.withType().configureEach { - enabled = false -} - -java { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 -} - - -tasks.withType { - manifest { - attributes["Main-Class"] = "com.mongodb.workload.WorkloadExecutor" - } -} diff --git a/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java b/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java deleted file mode 100644 index 7aba736aebc..00000000000 --- a/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright 2008-present MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.mongodb.workload; - -import com.mongodb.client.unified.Entities; -import com.mongodb.client.unified.UnifiedSyncTest; -import com.mongodb.client.unified.UnifiedTest; -import com.mongodb.internal.diagnostics.logging.Logger; -import com.mongodb.internal.diagnostics.logging.Loggers; -import org.bson.BsonArray; -import org.bson.BsonDocument; -import org.bson.BsonInt64; -import org.bson.codecs.BsonDocumentCodec; -import org.bson.codecs.DecoderContext; -import org.bson.json.JsonReader; -import org.bson.json.JsonWriterSettings; - -import java.io.FileReader; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -public class WorkloadExecutor { - private static final Logger LOGGER = Loggers.getLogger("workload-executor"); - private static final CountDownLatch terminationLatch = new CountDownLatch(1); - private static volatile boolean terminateLoop; - - public static void main(String[] args) throws IOException { - if (args.length != 2) { - System.out.println("Usage: AstrolabeTestRunner "); - System.exit(1); - } - - String pathToWorkloadFile = args[0]; - String pathToResultsDirectory = args[1]; - - LOGGER.info("Max memory (GB): " + (Runtime.getRuntime().maxMemory() / 1_073_741_824.0)); - LOGGER.info("Path to workload file: '" + pathToWorkloadFile + "'"); - LOGGER.info("Path to results directory: '" + pathToResultsDirectory + "'"); - - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - LOGGER.info("Running shutdown hook"); - terminateLoop = true; - try { - if (!terminationLatch.await(1, TimeUnit.MINUTES)) { - LOGGER.warn("Terminating after waiting for 1 minute for results to be written"); - } else { - LOGGER.info("Terminating."); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - })); - - - BsonDocument fileDocument; - - try (FileReader reader = new FileReader(pathToWorkloadFile)) { - fileDocument = new BsonDocumentCodec().decode(new JsonReader(reader), DecoderContext.builder().build()); - } - - LOGGER.info("Executing workload: " + fileDocument.toJson(JsonWriterSettings.builder().indent(true).build())); - - BsonArray testArray = fileDocument.getArray("tests"); - if (testArray.size() != 1) { - throw new IllegalArgumentException("Expected exactly one test"); - } - BsonDocument testDocument = testArray.get(0).asDocument(); - - UnifiedTest unifiedTest = new UnifiedSyncTest() { - @Override - protected boolean terminateLoop() { - return terminateLoop; - } - }; - - try { - String schemaVersion = fileDocument.getString("schemaVersion").getValue(); - BsonArray runOnRequirements = fileDocument.getArray("runOnRequirements", null); - BsonArray createEntities = fileDocument.getArray("createEntities", new BsonArray()); - BsonArray initialData = fileDocument.getArray("initialData", new BsonArray()); - unifiedTest.setUp( - "", - null, - null, - null, - 1, - 1, - schemaVersion, - runOnRequirements, - createEntities, - initialData, - testDocument); - unifiedTest.shouldPassAllOutcomes( - "", - null, - null, - null, - 1, - 1, - schemaVersion, - runOnRequirements, - createEntities, - initialData, - testDocument); - Entities entities = unifiedTest.getEntities(); - - long iterationCount = -1; - if (entities.hasIterationCount("iterations")) { - iterationCount = entities.getIterationCount("iterations"); - } - - long successCount = -1; - if (entities.hasSuccessCount("successes")) { - successCount = entities.getSuccessCount("successes"); - } - - BsonArray errorDocuments = null; - long errorCount = 0; - if (entities.hasErrorDocuments("errors")) { - errorDocuments = entities.getErrorDocuments("errors"); - errorCount = errorDocuments.size(); - } - - BsonArray failureDocuments = null; - long failureCount = 0; - if (entities.hasFailureDocuments("failures")) { - failureDocuments = entities.getFailureDocuments("failures"); - failureCount = failureDocuments.size(); - } - - BsonArray eventDocuments = new BsonArray(); - if (entities.hasEvents("events")) { - eventDocuments = new BsonArray(entities.getEvents("events")); - } - - BsonDocument eventsDocument = new BsonDocument() - .append("errors", errorDocuments == null ? new BsonArray() : errorDocuments) - .append("failures", failureDocuments == null ? new BsonArray() : failureDocuments) - .append("events", eventDocuments); - - BsonDocument resultsDocument = new BsonDocument() - .append("numErrors", new BsonInt64(errorCount)) - .append("numFailures", new BsonInt64(failureCount)) - .append("numSuccesses", new BsonInt64(successCount)) - .append("numIterations", new BsonInt64(iterationCount)); - - writeFile(eventsDocument, Paths.get(pathToResultsDirectory, "events.json")); - writeFile(resultsDocument, Paths.get(pathToResultsDirectory, "results.json")); - } finally { - unifiedTest.cleanUp(); - terminationLatch.countDown(); - } - } - - private static void writeFile(final BsonDocument document, final Path path) throws IOException { - LOGGER.info("Writing file: '" + path.toFile().getAbsolutePath()); - Files.deleteIfExists(path); - String json = document.toJson(JsonWriterSettings.builder().indent(true).build()); - LOGGER.debug("File contents: " + json); - Files.write(path, (json + "\n").getBytes(StandardCharsets.UTF_8)); - } -} diff --git a/driver-workload-executor/src/resources/logback.xml b/driver-workload-executor/src/resources/logback.xml deleted file mode 100644 index 706bbca3534..00000000000 --- a/driver-workload-executor/src/resources/logback.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - diff --git a/settings.gradle.kts b/settings.gradle.kts index 019d3b8290d..29d17792ad4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -43,7 +43,6 @@ include(":driver-scala") include(":driver-benchmarks") include(":driver-lambda") -include(":driver-workload-executor") if (providers.gradleProperty("includeGraalvm").isPresent) { include(":graalvm-native-image-app") } From 0811b5cda967706b2b41e237606947bb82e76233 Mon Sep 17 00:00:00 2001 From: Nathan Xu Date: Fri, 1 Aug 2025 11:55:11 -0400 Subject: [PATCH 4/4] remove methods in Entities only the deleted module ever used --- .../com/mongodb/client/unified/Entities.java | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java b/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java index 1125c6cf3f0..127ea91432b 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/Entities.java @@ -108,60 +108,8 @@ public final class Entities { private final Map serverMonitorListeners = new HashMap<>(); private final Map> cursors = new HashMap<>(); private final Map topologyDescriptions = new HashMap<>(); - private final Map successCounts = new HashMap<>(); - private final Map iterationCounts = new HashMap<>(); - private final Map errorDocumentsMap = new HashMap<>(); - private final Map failureDocumentsMap = new HashMap<>(); private final Map> eventsMap = new HashMap<>(); - public boolean hasSuccessCount(final String id) { - return successCounts.containsKey(id); - } - - public void addSuccessCount(final String id, final long count) { - putEntity(id, count, successCounts); - } - - public Long getSuccessCount(final String id) { - return getEntity(id, successCounts, "successCount"); - } - - public boolean hasIterationCount(final String id) { - return iterationCounts.containsKey(id); - } - - public void addIterationCount(final String id, final long count) { - putEntity(id, count, iterationCounts); - } - - public Long getIterationCount(final String id) { - return getEntity(id, iterationCounts, "successCount"); - } - - public boolean hasErrorDocuments(final String id) { - return errorDocumentsMap.containsKey(id); - } - - public void addErrorDocuments(final String id, final BsonArray errorDocuments) { - putEntity(id, errorDocuments, errorDocumentsMap); - } - - public BsonArray getErrorDocuments(final String id) { - return getEntity(id, errorDocumentsMap, "errorDocuments"); - } - - public boolean hasFailureDocuments(final String id) { - return failureDocumentsMap.containsKey(id); - } - - public void addFailureDocuments(final String id, final BsonArray failureDocuments) { - putEntity(id, failureDocuments, failureDocumentsMap); - } - - public BsonArray getFailureDocuments(final String id) { - return getEntity(id, failureDocumentsMap, "failureDocuments"); - } - public boolean hasEvents(final String id) { return eventsMap.containsKey(id); }