From a370a7bef9d0c9c9f0c7610b5077b100c6c94f5b Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Fri, 31 May 2024 14:59:35 +0200 Subject: [PATCH 01/43] WIP reconfigure only changed namespaces --- .../runtime/engine/config/EngineConfig.java | 13 ++++- .../engine/config/EngineConfigBuilder.java | 17 ++++-- .../engine/config/EngineConfigReader.java | 52 +++++++++++++++++-- .../engine/config/NamespaceConfig.java | 1 + .../internal/registry/EngineManager.java | 30 ++++++++--- 5 files changed, 98 insertions(+), 15 deletions(-) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfig.java index 450002ff2e..34ab6785d0 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfig.java @@ -19,9 +19,12 @@ import static java.util.function.Function.identity; import java.util.List; +import java.util.Map; public class EngineConfig { + private final Map hashes; + public final List namespaces; public static EngineConfigBuilder builder() @@ -30,8 +33,16 @@ public static EngineConfigBuilder builder() } EngineConfig( - List namespaces) + List namespaces, + Map hashes) { this.namespaces = requireNonNull(namespaces); + this.hashes = requireNonNull(hashes); + } + + public String hash( + String name) + { + return hashes.get(name); } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigBuilder.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigBuilder.java index 5c4f01df83..bd7b6f7fca 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigBuilder.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigBuilder.java @@ -15,13 +15,16 @@ */ package io.aklivity.zilla.runtime.engine.config; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.function.Function; public final class EngineConfigBuilder extends ConfigBuilder> { private final Function mapper; + private final Map hashes; private List namespaces; @@ -29,6 +32,7 @@ public final class EngineConfigBuilder extends ConfigBuilder mapper) { this.mapper = mapper; + this.hashes = new HashMap<>(); } @Override @@ -50,7 +54,15 @@ public EngineConfigBuilder namespace( { namespaces = new LinkedList<>(); } - namespaces.add(namespace); + if (!hashes.containsKey(namespace.name)) + { + namespaces.add(namespace); + hashes.put(namespace.name, namespace.hash); + } + else + { + System.out.printf("Warning: duplicate namespace ignored: %s%n", namespace.name); + } return this; } @@ -68,7 +80,6 @@ public T build() namespaces = new LinkedList<>(); } - return mapper.apply(new EngineConfig( - namespaces)); + return mapper.apply(new EngineConfig(namespaces, hashes)); } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java index 5b1aa71d27..65f1b0852f 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java @@ -25,6 +25,10 @@ import java.io.StringReader; import java.io.StringWriter; import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -40,6 +44,7 @@ import jakarta.json.spi.JsonProvider; import jakarta.json.stream.JsonParser; +import org.agrona.BitUtil; import org.agrona.collections.IntArrayList; import org.leadpony.justify.api.JsonSchema; import org.leadpony.justify.api.JsonSchemaReader; @@ -59,7 +64,7 @@ public final class EngineConfigReader private final Resolver expressions; private final Collection schemaTypes; private final Consumer logger; - + private final MessageDigest md5; public EngineConfigReader( EngineConfiguration config, @@ -73,6 +78,7 @@ public EngineConfigReader( this.expressions = expressions; this.schemaTypes = schemaTypes; this.logger = logger; + this.md5 = initMessageDigest("MD5"); } public EngineConfig read( @@ -148,6 +154,16 @@ public EngineConfig read( } } + List hashes = new ArrayList<>(); + for (int i = 0; i < configsAt.size(); i++) + { + int start = configsAt.get(i); + int end = i < configsAt.size() - 1 ? configsAt.get(i + 1) : readable.length(); + byte[] bytes = readable.substring(start, end).stripTrailing().getBytes(StandardCharsets.UTF_8); + String hash = calculateHash(bytes, 0, bytes.length); + hashes.add(hash); + } + JsonbConfig config = new JsonbConfig() .withAdapters(new NamespaceAdapter(context)); Jsonb jsonb = JsonbBuilder.newBuilder() @@ -157,11 +173,13 @@ public EngineConfig read( Reader reader = new StringReader(readable); EngineConfigBuilder builder = EngineConfig.builder(); - for (int configAt : configsAt) + for (int i = 0; i < configsAt.size(); i++) { reader.reset(); - reader.skip(configAt); - builder.namespace(jsonb.fromJson(reader, NamespaceConfig.class)); + reader.skip(configsAt.get(i)); + NamespaceConfig namespace = jsonb.fromJson(reader, NamespaceConfig.class); + namespace.hash = hashes.get(i); + builder.namespace(namespace); if (!errors.isEmpty()) { @@ -262,4 +280,30 @@ private boolean validateAnnotatedSchema( return valid; } + + private String calculateHash( + byte[] input, + int offset, + int length) + { + md5.reset(); + md5.update(input, offset, length); + byte[] hash = md5.digest(); + return BitUtil.toHex(hash); + } + + private static MessageDigest initMessageDigest( + String algorithm) + { + MessageDigest messageDigest = null; + try + { + messageDigest = MessageDigest.getInstance(algorithm); + } + catch (NoSuchAlgorithmException ex) + { + rethrowUnchecked(ex); + } + return messageDigest; + } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java index a98f862482..f3489fed3f 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java @@ -25,6 +25,7 @@ public class NamespaceConfig { public transient int id; public transient Function readURL; + public transient String hash; public final String name; public final TelemetryConfig telemetry; diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index 425f9d5113..a4983774d2 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -31,6 +31,7 @@ import java.util.function.IntFunction; import java.util.function.LongFunction; import java.util.function.LongPredicate; +import java.util.function.Predicate; import java.util.function.ToIntFunction; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -129,19 +130,26 @@ public EngineConfig reconfigure( if (newConfig != null) { final EngineConfig oldConfig = current; - unregister(oldConfig); + EngineConfig newConfig0 = newConfig; + Predicate identical = name -> + { + String hash1 = oldConfig == null ? null : oldConfig.hash(name); + String hash2 = newConfig0.hash(name); + return hash1 != null && hash1.equals(hash2); + }; + unregister(oldConfig, identical); try { current = newConfig; - register(newConfig); + register(newConfig, identical); } catch (Exception ex) { context.onError(ex); current = oldConfig; - register(oldConfig); + register(oldConfig, identical); rethrowUnchecked(ex); } @@ -370,13 +378,17 @@ private void process( } private void register( - EngineConfig config) + EngineConfig config, + Predicate identical) { if (config != null) { for (NamespaceConfig namespace : config.namespaces) { - register(namespace); + if (!identical.test(namespace.name)) + { + register(namespace); + } } } @@ -384,13 +396,17 @@ private void register( } private void unregister( - EngineConfig config) + EngineConfig config, + Predicate identical) { if (config != null) { for (NamespaceConfig namespace : config.namespaces) { - unregister(namespace); + if (!identical.test(namespace.name)) + { + unregister(namespace); + } } } From 3757300832c664a848ca9ca8692b3967f637602c Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Wed, 5 Jun 2024 14:51:32 +0200 Subject: [PATCH 02/43] WIP watch resources --- .../internal/FilesystemCatalogHandler.java | 13 ++ .../aklivity/zilla/runtime/engine/Engine.java | 17 +- .../engine/catalog/CatalogHandler.java | 7 + .../internal/registry/EngineManager.java | 20 ++- .../internal/registry/EngineRegistry.java | 7 +- .../internal/registry/EngineWorker.java | 3 +- .../internal/registry/FileWatcherTask.java | 31 +++- .../internal/registry/HttpWatcherTask.java | 22 ++- .../internal/registry/NamespaceRegistry.java | 7 +- .../internal/registry/ResourceWatcher.java | 145 ++++++++++++++++++ .../engine/internal/registry/WatcherTask.java | 33 +++- .../runtime/engine/vault/VaultHandler.java | 6 + .../internal/FileSystemVaultHandler.java | 17 ++ 13 files changed, 302 insertions(+), 26 deletions(-) create mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/ResourceWatcher.java diff --git a/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java b/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java index 4716cff249..3f5542bafc 100644 --- a/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java +++ b/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java @@ -17,6 +17,7 @@ import java.io.InputStream; import java.net.URL; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -35,6 +36,7 @@ public class FilesystemCatalogHandler implements CatalogHandler private final FilesystemEventContext event; private final long catalogId; private final Function resolvePath; + private final List resources; public FilesystemCatalogHandler( FilesystemOptionsConfig config, @@ -48,6 +50,11 @@ public FilesystemCatalogHandler( this.resolvePath = context::resolvePath; this.catalogId = catalogId; registerSchema(config.subjects); + this.resources = new LinkedList<>(); + for (FilesystemSchemaConfig subject : config.subjects) + { + resources.add(subject.path); + } } @Override @@ -88,6 +95,12 @@ private void registerSchema( } } + @Override + public List resources() + { + return resources; + } + private int generateCRC32C( String schema) { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index c504dc9531..e3365a15ba 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -73,6 +73,7 @@ import io.aklivity.zilla.runtime.engine.internal.registry.EngineWorker; import io.aklivity.zilla.runtime.engine.internal.registry.FileWatcherTask; import io.aklivity.zilla.runtime.engine.internal.registry.HttpWatcherTask; +import io.aklivity.zilla.runtime.engine.internal.registry.ResourceWatcher; import io.aklivity.zilla.runtime.engine.internal.registry.WatcherTask; import io.aklivity.zilla.runtime.engine.internal.types.event.EventFW; import io.aklivity.zilla.runtime.engine.metrics.Collector; @@ -98,6 +99,7 @@ public final class Engine implements Collector, AutoCloseable private final boolean readonly; private final EngineConfiguration config; private final EngineManager manager; + private final ResourceWatcher resourceWatcher; private Future watcherTaskRef; @@ -160,13 +162,14 @@ public final class Engine implements Collector, AutoCloseable } this.tuning = tuning; + this.resourceWatcher = new ResourceWatcher(); List workers = new ArrayList<>(workerCount); for (int workerIndex = 0; workerIndex < workerCount; workerIndex++) { EngineWorker worker = new EngineWorker(config, tasks, labels, errorHandler, tuning::affinity, bindings, exporters, guards, vaults, catalogs, models, metricGroups, this, this::supplyEventReader, - eventFormatterFactory, workerIndex, readonly, this::process); + eventFormatterFactory, resourceWatcher, workerIndex, readonly, this::process); workers.add(worker); } this.workers = workers; @@ -206,18 +209,21 @@ public final class Engine implements Collector, AutoCloseable context, config, extensions, - this::readURL); + this::readURL, + resourceWatcher); this.configURL = config.configURL(); String protocol = configURL.getProtocol(); if ("file".equals(protocol) || "jar".equals(protocol)) { Function watcherReadURL = l -> readURL(configURL, l); - this.watcherTask = new FileWatcherTask(manager::reconfigure, watcherReadURL); + this.watcherTask = new FileWatcherTask(manager::reconfigure, null, watcherReadURL); + this.resourceWatcher.initialize(manager::reloadNamespacesWithChangedResources, watcherReadURL); } else if ("http".equals(protocol) || "https".equals(protocol)) { - this.watcherTask = new HttpWatcherTask(manager::reconfigure, config.configPollIntervalSeconds()); + this.watcherTask = new HttpWatcherTask(manager::reconfigure, manager::reloadNamespacesWithChangedResources, + config.configPollIntervalSeconds()); } else { @@ -259,7 +265,7 @@ public void start() throws Exception if (!readonly) { // ignore the config file in read-only mode; no config will be read so no namespaces, bindings, etc will be attached - watcherTask.watch(configURL).get(); + watcherTask.watchConfig(configURL).get(); } } @@ -273,6 +279,7 @@ public void close() throws Exception final List errors = new ArrayList<>(); + resourceWatcher.close(); watcherTask.close(); watcherTaskRef.get(); diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/catalog/CatalogHandler.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/catalog/CatalogHandler.java index d6b730e261..bfdb5a44e7 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/catalog/CatalogHandler.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/catalog/CatalogHandler.java @@ -15,6 +15,8 @@ */ package io.aklivity.zilla.runtime.engine.catalog; +import java.util.List; + import org.agrona.DirectBuffer; import io.aklivity.zilla.runtime.engine.model.function.ValueConsumer; @@ -114,4 +116,9 @@ default int decodePadding( { return 0; } + + default List resources() + { + return List.of(); + } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index a4983774d2..9bda92919b 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -84,6 +84,7 @@ public class EngineManager private final List extensions; private final BiFunction readURL; private final Resolver expressions; + private final ResourceWatcher resourceWatcher; private EngineConfig current; @@ -100,7 +101,8 @@ public EngineManager( EngineExtContext context, EngineConfiguration config, List extensions, - BiFunction readURL) + BiFunction readURL, + ResourceWatcher resourceWatcher) { this.schemaTypes = schemaTypes; this.bindingByType = bindingByType; @@ -116,6 +118,7 @@ public EngineManager( this.extensions = extensions; this.readURL = readURL; this.expressions = Resolver.instantiate(config); + this.resourceWatcher = resourceWatcher; } public EngineConfig reconfigure( @@ -171,6 +174,18 @@ public EngineConfig reconfigure( return newConfig; } + public void reloadNamespacesWithChangedResources( + Set namespaces) + { + if (namespaces != null && !namespaces.isEmpty()) + { + Set namespaces0 = new HashSet<>(namespaces); + Predicate identical = name -> !namespaces0.contains(name); + unregister(current, identical); + register(current, identical); + } + } + public void process( NamespaceConfig namespace) { @@ -387,6 +402,7 @@ private void register( { if (!identical.test(namespace.name)) { + System.out.println("register: " + namespace.name); // TODO: Ati register(namespace); } } @@ -405,7 +421,9 @@ private void unregister( { if (!identical.test(namespace.name)) { + System.out.println("unregister: " + namespace.name); // TODO: Ati unregister(namespace); + resourceWatcher.removeNamespace(namespace.name); } } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java index 452cc3b183..fd6216ea2d 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java @@ -51,6 +51,7 @@ public class EngineRegistry private final LongConsumer detachBinding; private final Collector collector; private final Consumer process; + private final ResourceWatcher resourceWatcher; public EngineRegistry( Function bindingsByType, @@ -65,7 +66,8 @@ public EngineRegistry( ObjectLongLongFunction supplyMetricRecorder, LongConsumer detachBinding, Collector collector, - Consumer process) + Consumer process, + ResourceWatcher resourceWatcher) { this.bindingsByType = bindingsByType; this.guardsByType = guardsByType; @@ -81,6 +83,7 @@ public EngineRegistry( this.detachBinding = detachBinding; this.collector = collector; this.process = process; + this.resourceWatcher = resourceWatcher; } public void process( @@ -196,7 +199,7 @@ private void attachNamespace( NamespaceRegistry registry = new NamespaceRegistry(namespace, bindingsByType, guardsByType, vaultsByType, catalogsByType, metricsByName, exportersByType, supplyLabelId, this::resolveMetric, exporterAttached, exporterDetached, - supplyMetricRecorder, detachBinding, collector); + supplyMetricRecorder, detachBinding, collector, resourceWatcher); namespacesById.put(registry.namespaceId(), registry); registry.attach(); } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java index 0d535e2d8a..d082bd4dd8 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java @@ -253,6 +253,7 @@ public EngineWorker( Collector collector, Supplier supplyEventReader, EventFormatterFactory eventFormatterFactory, + ResourceWatcher resourceWatcher, int index, boolean readonly, Consumer process) @@ -427,7 +428,7 @@ public EngineWorker( this.registry = new EngineRegistry( bindingsByType::get, guardsByType::get, vaultsByType::get, catalogsByType::get, metricsByName::get, exportersByType::get, labels::supplyLabelId, this::onExporterAttached, this::onExporterDetached, - this::supplyMetricWriter, this::detachStreams, collector, process); + this::supplyMetricWriter, this::detachStreams, collector, process, resourceWatcher); this.taskQueue = new ConcurrentLinkedDeque<>(); this.correlations = new Long2ObjectHashMap<>(); diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/FileWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/FileWatcherTask.java index fbcc466fb5..9e7a5e8382 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/FileWatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/FileWatcherTask.java @@ -25,9 +25,11 @@ import java.nio.file.WatchService; import java.util.IdentityHashMap; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Function; import io.aklivity.zilla.runtime.engine.config.EngineConfig; @@ -39,10 +41,11 @@ public class FileWatcherTask extends WatcherTask private final Function readURL; public FileWatcherTask( - BiFunction changeListener, + BiFunction configChangeListener, + Consumer> resourceChangeListener, Function readURL) { - super(changeListener); + super(configChangeListener, resourceChangeListener); this.readURL = readURL; this.watchedConfigs = new IdentityHashMap<>(); WatchService watchService = null; @@ -89,7 +92,14 @@ public Void call() if (watchedConfig.isReconfigureNeeded(newConfigHash)) { watchedConfig.setConfigHash(newConfigHash); - changeListener.apply(watchedConfig.getURL(), newConfigText); + if (configChangeListener != null) + { + configChangeListener.apply(watchedConfig.getURL(), newConfigText); + } + if (resourceChangeListener != null) + { + resourceChangeListener.accept(namespaces); + } } } } @@ -103,7 +113,7 @@ public Void call() } @Override - public CompletableFuture watch( + public CompletableFuture watchConfig( URL configURL) { WatchedConfig watchedConfig = new WatchedConfig(configURL, watchService); @@ -115,7 +125,7 @@ public CompletableFuture watch( CompletableFuture configFuture; try { - EngineConfig config = changeListener.apply(configURL, configText); + EngineConfig config = configChangeListener.apply(configURL, configText); configFuture = CompletableFuture.completedFuture(config); } catch (Exception ex) @@ -126,6 +136,17 @@ public CompletableFuture watch( return configFuture; } + @Override + public void watchResource( + URL resourceURL) + { + WatchedConfig watchedConfig = new WatchedConfig(resourceURL, watchService); + watchedConfig.register(); + watchedConfig.keys().forEach(k -> watchedConfigs.put(k, watchedConfig)); + String resource = readURL.apply(resourceURL.toString()); + watchedConfig.setConfigHash(computeHash(resource)); + } + @Override public void close() throws IOException { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/HttpWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/HttpWatcherTask.java index c4567ad7fa..2ccadb83da 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/HttpWatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/HttpWatcherTask.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -36,6 +37,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; +import java.util.function.Consumer; import io.aklivity.zilla.runtime.engine.config.EngineConfig; @@ -50,10 +52,11 @@ public class HttpWatcherTask extends WatcherTask private final int pollSeconds; public HttpWatcherTask( - BiFunction changeListener, + BiFunction configChangeListener, + Consumer> resourceChangeListener, int pollSeconds) { - super(changeListener); + super(configChangeListener, resourceChangeListener); this.etags = new ConcurrentHashMap<>(); this.configHashes = new ConcurrentHashMap<>(); this.futures = new ConcurrentHashMap<>(); @@ -84,7 +87,7 @@ public Void call() throws InterruptedException } @Override - public CompletableFuture watch( + public CompletableFuture watchConfig( URL configURL) { URI configURI = toURI(configURL); @@ -103,6 +106,13 @@ public CompletableFuture watch( return configFuture; } + @Override + public void watchResource( + URL resourceURL) + { + // TODO: Ati + } + @Override public void close() { @@ -175,7 +185,7 @@ private EngineConfig handleConfigChange( int pollIntervalSeconds = 0; if (statusCode == 404) { - config = changeListener.apply(configURI.toURL(), ""); + config = configChangeListener.apply(configURI.toURL(), ""); pollIntervalSeconds = this.pollSeconds; } else if (statusCode >= 500 && statusCode <= 599) @@ -193,7 +203,7 @@ else if (statusCode >= 500 && statusCode <= 599) if (!oldEtag.equals(etagOptional.get())) { etags.put(configURI, etagOptional.get()); - config = changeListener.apply(configURI.toURL(), configText); + config = configChangeListener.apply(configURI.toURL(), configText); } else if (response.statusCode() != 304) { @@ -207,7 +217,7 @@ else if (response.statusCode() != 304) if (!Arrays.equals(configHash, newConfigHash)) { configHashes.put(configURI, newConfigHash); - config = changeListener.apply(configURI.toURL(), configText); + config = configChangeListener.apply(configURI.toURL(), configText); } pollIntervalSeconds = this.pollSeconds; } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java index ef49c2c383..5937785181 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java @@ -74,6 +74,7 @@ public class NamespaceRegistry private final ObjectLongLongFunction supplyMetricRecorder; private final LongConsumer detachBinding; private final Collector collector; + private final ResourceWatcher resourceWatcher; public NamespaceRegistry( NamespaceConfig namespace, @@ -89,7 +90,8 @@ public NamespaceRegistry( LongConsumer exporterDetached, ObjectLongLongFunction supplyMetricRecorder, LongConsumer detachBinding, - Collector collector) + Collector collector, + ResourceWatcher resourceWatcher) { this.namespace = namespace; this.bindingsByType = bindingsByType; @@ -112,6 +114,7 @@ public NamespaceRegistry( this.metricsById = new Int2ObjectHashMap<>(); this.exportersById = new Int2ObjectHashMap<>(); this.collector = collector; + this.resourceWatcher = resourceWatcher; } public int namespaceId() @@ -265,6 +268,7 @@ private void attachVault( VaultRegistry registry = new VaultRegistry(config, context); vaultsById.put(vaultId, registry); registry.attach(); + resourceWatcher.addResources(registry.handler().resources(), config.namespace); } private void detachVault( @@ -311,6 +315,7 @@ private void attachCatalog( CatalogRegistry registry = new CatalogRegistry(config, context); catalogsById.put(catalogId, registry); registry.attach(); + resourceWatcher.addResources(registry.handler().resources(), config.namespace); } private void detachCatalog( diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/ResourceWatcher.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/ResourceWatcher.java new file mode 100644 index 0000000000..d709bd2394 --- /dev/null +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/ResourceWatcher.java @@ -0,0 +1,145 @@ +/* + * Copyright 2021-2023 Aklivity Inc. + * + * Aklivity licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.aklivity.zilla.runtime.engine.internal.registry; + +import static org.agrona.LangUtil.rethrowUnchecked; + +import java.net.URL; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; +import java.util.function.Function; + +public class ResourceWatcher +{ + private final Map> resources; + private final Map resourceTasks; + + private Consumer> resourceChangeListener; + private Function readURL; + + public ResourceWatcher() + { + this.resources = new ConcurrentHashMap<>(); + this.resourceTasks = new ConcurrentHashMap<>(); + } + + public void initialize( + Consumer> resourceChangeListener, + Function readURL) + { + this.resourceChangeListener = resourceChangeListener; + this.readURL = readURL; + } + + public void addResources( + List additionalResources, + String namespace) + { + additionalResources.forEach(resource -> + { + resources.computeIfAbsent(resource, i -> + { + startWatchingResource(resource, namespace); + return ConcurrentHashMap.newKeySet(); + } + ).add(namespace); + resourceTasks.get(resource).addNamespaces(namespace); + } + ); + } + + public void removeNamespace( + String namespace) + { + resources.entrySet().removeIf(e -> + { + String resource = e.getKey(); + Set namespaces = e.getValue(); + namespaces.remove(namespace); + boolean empty = namespaces.isEmpty(); + if (empty) + { + stopWatchingResource(resource); + } + else + { + removeNamespaceFromWatchedResource(resource, namespace); + } + return empty; + } + ); + } + + private void startWatchingResource( + String resource, + String namespace) + { + try + { + FileWatcherTask watcherTask = new FileWatcherTask(null, resourceChangeListener, readURL); + watcherTask.addNamespaces(namespace); + watcherTask.submit(); + URL resourceURL = Path.of(resource).toUri().toURL(); + watcherTask.watchResource(resourceURL); + resourceTasks.put(resource, watcherTask); + System.out.println("started watching resource: " + resource); // TODO: Ati + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } + } + + private void stopWatchingResource( + String resource) + { + try + { + resourceTasks.remove(resource).close(); + System.out.println("stopped watching resource: " + resource); // TODO: Ati + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } + } + + private void removeNamespaceFromWatchedResource( + String resource, + String namespace) + { + resourceTasks.get(resource).removeNamespace(namespace); + } + + public void close() + { + resourceTasks.forEach((resource, watcherTask) -> + { + try + { + watcherTask.close(); + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } + }); + } +} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/WatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/WatcherTask.java index 0bfc9e64e5..173f7a1d0e 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/WatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/WatcherTask.java @@ -22,12 +22,15 @@ import java.net.URL; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.function.BiFunction; +import java.util.function.Consumer; import io.aklivity.zilla.runtime.engine.config.EngineConfig; @@ -36,21 +39,41 @@ public abstract class WatcherTask implements Callable, Closeable private final MessageDigest md5; protected final ScheduledExecutorService executor; - protected final BiFunction changeListener; + protected final BiFunction configChangeListener; + protected final Consumer> resourceChangeListener; + protected final Set namespaces; protected WatcherTask( - BiFunction changeListener) + BiFunction configChangeListener, + Consumer> resourceChangeListener) { - this.changeListener = changeListener; + this.configChangeListener = configChangeListener; + this.resourceChangeListener = resourceChangeListener; this.md5 = initMessageDigest("MD5"); - this.executor = Executors.newScheduledThreadPool(2); + this.executor = Executors.newScheduledThreadPool(2); + this.namespaces = new HashSet<>(); + } + + public void addNamespaces( + String namespace) + { + namespaces.add(namespace); + } + + public void removeNamespace( + String namespace) + { + namespaces.remove(namespace); } public abstract Future submit(); - public abstract CompletableFuture watch( + public abstract CompletableFuture watchConfig( URL configURL); + public abstract void watchResource( + URL resourceURL); + protected byte[] computeHash( String configText) { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/vault/VaultHandler.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/vault/VaultHandler.java index 6c67d6c856..c9086a27e3 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/vault/VaultHandler.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/vault/VaultHandler.java @@ -16,6 +16,7 @@ package io.aklivity.zilla.runtime.engine.vault; import java.security.KeyStore; +import java.util.List; public interface VaultHandler { @@ -27,4 +28,9 @@ KeyStore.TrustedCertificateEntry certificate( KeyStore.PrivateKeyEntry[] keys( String signerRef); + + default List resources() + { + return List.of(); + } } diff --git a/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java b/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java index c465063d99..97b2ab7093 100644 --- a/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java +++ b/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java @@ -26,6 +26,7 @@ import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedList; import java.util.List; import java.util.Optional; import java.util.function.Function; @@ -47,6 +48,7 @@ public class FileSystemVaultHandler implements VaultHandler private final Function lookupTrust; private final Function lookupSigner; private final Function, KeyStore.PrivateKeyEntry[]> lookupKeys; + private final List resources; public FileSystemVaultHandler( FileSystemOptionsConfig options, @@ -56,6 +58,15 @@ public FileSystemVaultHandler( lookupTrust = supplyLookupTrustedCertificateEntry(resolvePath, options.trust); lookupSigner = supplyLookupTrustedCertificateEntry(resolvePath, options.signers); lookupKeys = supplyLookupPrivateKeyEntries(resolvePath, options.keys); + resources = new LinkedList<>(); + if (options.keys != null && options.keys.store != null) + { + resources.add(options.keys.store); + } + if (options.trust != null && options.trust.store != null) + { + resources.add(options.trust.store); + } } @Override @@ -93,6 +104,12 @@ public PrivateKeyEntry[] keys( return keys; } + @Override + public List resources() + { + return resources; + } + private static Function supplyLookupPrivateKeyEntry( Function resolvePath, FileSystemStoreConfig aliases) From c02e5c4df80d9739ceb0519986942e02422f8783 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Wed, 5 Jun 2024 17:33:11 +0200 Subject: [PATCH 03/43] WIP refactoring watch resources --- runtime/engine/pom.xml | 2 +- .../aklivity/zilla/runtime/engine/Engine.java | 38 ++--- .../internal/registry/EngineManager.java | 9 +- .../internal/registry/EngineRegistry.java | 9 +- .../internal/registry/EngineWorker.java | 5 +- .../internal/registry/NamespaceRegistry.java | 11 +- .../ConfigFileWatcherTask.java} | 61 +++----- .../ConfigHttpWatcherTask.java} | 19 +-- .../internal/watcher/ConfigWatcher.java | 30 ++++ .../watcher/ResourceFileWatcherTask.java | 140 ++++++++++++++++++ .../ResourceWatchManager.java} | 14 +- .../internal/watcher/ResourceWatcher.java | 31 ++++ .../WatchedItem.java} | 45 ++---- .../{registry => watcher}/WatcherTask.java | 48 +----- 14 files changed, 294 insertions(+), 168 deletions(-) rename runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/{registry/FileWatcherTask.java => watcher/ConfigFileWatcherTask.java} (59%) rename runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/{registry/HttpWatcherTask.java => watcher/ConfigHttpWatcherTask.java} (94%) create mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcher.java create mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceFileWatcherTask.java rename runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/{registry/ResourceWatcher.java => watcher/ResourceWatchManager.java} (90%) create mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcher.java rename runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/{registry/WatchedConfig.java => watcher/WatchedItem.java} (81%) rename runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/{registry => watcher}/WatcherTask.java (51%) diff --git a/runtime/engine/pom.xml b/runtime/engine/pom.xml index 0c55bb67f6..219c430e54 100644 --- a/runtime/engine/pom.xml +++ b/runtime/engine/pom.xml @@ -26,7 +26,7 @@ 11 11 - 0.77 + 0.76 5 diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index e3365a15ba..0de5794dc1 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -71,11 +71,11 @@ import io.aklivity.zilla.runtime.engine.internal.layouts.EventsLayout; import io.aklivity.zilla.runtime.engine.internal.registry.EngineManager; import io.aklivity.zilla.runtime.engine.internal.registry.EngineWorker; -import io.aklivity.zilla.runtime.engine.internal.registry.FileWatcherTask; -import io.aklivity.zilla.runtime.engine.internal.registry.HttpWatcherTask; -import io.aklivity.zilla.runtime.engine.internal.registry.ResourceWatcher; -import io.aklivity.zilla.runtime.engine.internal.registry.WatcherTask; import io.aklivity.zilla.runtime.engine.internal.types.event.EventFW; +import io.aklivity.zilla.runtime.engine.internal.watcher.ConfigFileWatcherTask; +import io.aklivity.zilla.runtime.engine.internal.watcher.ConfigHttpWatcherTask; +import io.aklivity.zilla.runtime.engine.internal.watcher.ConfigWatcher; +import io.aklivity.zilla.runtime.engine.internal.watcher.ResourceWatchManager; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.MetricGroup; import io.aklivity.zilla.runtime.engine.model.Model; @@ -93,15 +93,15 @@ public final class Engine implements Collector, AutoCloseable private final AtomicInteger nextTaskId; private final ThreadFactory factory; - private final WatcherTask watcherTask; + private final ConfigWatcher configWatcherTask; private final URL configURL; private final List workers; private final boolean readonly; private final EngineConfiguration config; private final EngineManager manager; - private final ResourceWatcher resourceWatcher; + private final ResourceWatchManager resourceWatchManager; - private Future watcherTaskRef; + private Future configWatcherTaskRef; Engine( EngineConfiguration config, @@ -162,14 +162,14 @@ public final class Engine implements Collector, AutoCloseable } this.tuning = tuning; - this.resourceWatcher = new ResourceWatcher(); + this.resourceWatchManager = new ResourceWatchManager(); List workers = new ArrayList<>(workerCount); for (int workerIndex = 0; workerIndex < workerCount; workerIndex++) { EngineWorker worker = new EngineWorker(config, tasks, labels, errorHandler, tuning::affinity, bindings, exporters, guards, vaults, catalogs, models, metricGroups, this, this::supplyEventReader, - eventFormatterFactory, resourceWatcher, workerIndex, readonly, this::process); + eventFormatterFactory, resourceWatchManager, workerIndex, readonly, this::process); workers.add(worker); } this.workers = workers; @@ -210,20 +210,20 @@ public final class Engine implements Collector, AutoCloseable config, extensions, this::readURL, - resourceWatcher); + resourceWatchManager); this.configURL = config.configURL(); String protocol = configURL.getProtocol(); if ("file".equals(protocol) || "jar".equals(protocol)) { Function watcherReadURL = l -> readURL(configURL, l); - this.watcherTask = new FileWatcherTask(manager::reconfigure, null, watcherReadURL); - this.resourceWatcher.initialize(manager::reloadNamespacesWithChangedResources, watcherReadURL); + this.configWatcherTask = new ConfigFileWatcherTask(manager::reconfigure, watcherReadURL); + this.resourceWatchManager.initialize(manager::reloadNamespacesWithChangedResources, watcherReadURL); } else if ("http".equals(protocol) || "https".equals(protocol)) { - this.watcherTask = new HttpWatcherTask(manager::reconfigure, manager::reloadNamespacesWithChangedResources, - config.configPollIntervalSeconds()); + this.configWatcherTask = new ConfigHttpWatcherTask(manager::reconfigure, config.configPollIntervalSeconds()); + // TODO: Ati - implement http } else { @@ -261,11 +261,11 @@ public void start() throws Exception worker.doStart(); } - watcherTaskRef = watcherTask.submit(); + configWatcherTaskRef = configWatcherTask.submit(); if (!readonly) { // ignore the config file in read-only mode; no config will be read so no namespaces, bindings, etc will be attached - watcherTask.watchConfig(configURL).get(); + configWatcherTask.watchConfig(configURL).get(); } } @@ -279,9 +279,9 @@ public void close() throws Exception final List errors = new ArrayList<>(); - resourceWatcher.close(); - watcherTask.close(); - watcherTaskRef.get(); + resourceWatchManager.close(); + configWatcherTask.close(); + configWatcherTaskRef.get(); for (EngineWorker worker : workers) { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index 9bda92919b..5459f142f4 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -63,6 +63,7 @@ import io.aklivity.zilla.runtime.engine.guard.Guard; import io.aklivity.zilla.runtime.engine.internal.Tuning; import io.aklivity.zilla.runtime.engine.internal.config.NamespaceAdapter; +import io.aklivity.zilla.runtime.engine.internal.watcher.ResourceWatchManager; import io.aklivity.zilla.runtime.engine.namespace.NamespacedId; import io.aklivity.zilla.runtime.engine.resolver.Resolver; @@ -84,7 +85,7 @@ public class EngineManager private final List extensions; private final BiFunction readURL; private final Resolver expressions; - private final ResourceWatcher resourceWatcher; + private final ResourceWatchManager resourceWatchManager; private EngineConfig current; @@ -102,7 +103,7 @@ public EngineManager( EngineConfiguration config, List extensions, BiFunction readURL, - ResourceWatcher resourceWatcher) + ResourceWatchManager resourceWatchManager) { this.schemaTypes = schemaTypes; this.bindingByType = bindingByType; @@ -118,7 +119,7 @@ public EngineManager( this.extensions = extensions; this.readURL = readURL; this.expressions = Resolver.instantiate(config); - this.resourceWatcher = resourceWatcher; + this.resourceWatchManager = resourceWatchManager; } public EngineConfig reconfigure( @@ -423,7 +424,7 @@ private void unregister( { System.out.println("unregister: " + namespace.name); // TODO: Ati unregister(namespace); - resourceWatcher.removeNamespace(namespace.name); + resourceWatchManager.removeNamespace(namespace.name); } } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java index fd6216ea2d..036dce2778 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java @@ -28,6 +28,7 @@ import io.aklivity.zilla.runtime.engine.config.NamespaceConfig; import io.aklivity.zilla.runtime.engine.exporter.ExporterContext; import io.aklivity.zilla.runtime.engine.guard.GuardContext; +import io.aklivity.zilla.runtime.engine.internal.watcher.ResourceWatchManager; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.Metric; import io.aklivity.zilla.runtime.engine.metrics.MetricContext; @@ -51,7 +52,7 @@ public class EngineRegistry private final LongConsumer detachBinding; private final Collector collector; private final Consumer process; - private final ResourceWatcher resourceWatcher; + private final ResourceWatchManager resourceWatchManager; public EngineRegistry( Function bindingsByType, @@ -67,7 +68,7 @@ public EngineRegistry( LongConsumer detachBinding, Collector collector, Consumer process, - ResourceWatcher resourceWatcher) + ResourceWatchManager resourceWatchManager) { this.bindingsByType = bindingsByType; this.guardsByType = guardsByType; @@ -83,7 +84,7 @@ public EngineRegistry( this.detachBinding = detachBinding; this.collector = collector; this.process = process; - this.resourceWatcher = resourceWatcher; + this.resourceWatchManager = resourceWatchManager; } public void process( @@ -199,7 +200,7 @@ private void attachNamespace( NamespaceRegistry registry = new NamespaceRegistry(namespace, bindingsByType, guardsByType, vaultsByType, catalogsByType, metricsByName, exportersByType, supplyLabelId, this::resolveMetric, exporterAttached, exporterDetached, - supplyMetricRecorder, detachBinding, collector, resourceWatcher); + supplyMetricRecorder, detachBinding, collector, resourceWatchManager); namespacesById.put(registry.namespaceId(), registry); registry.attach(); } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java index d082bd4dd8..f1acac9221 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java @@ -137,6 +137,7 @@ import io.aklivity.zilla.runtime.engine.internal.types.stream.ResetFW; import io.aklivity.zilla.runtime.engine.internal.types.stream.SignalFW; import io.aklivity.zilla.runtime.engine.internal.types.stream.WindowFW; +import io.aklivity.zilla.runtime.engine.internal.watcher.ResourceWatchManager; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.Metric; import io.aklivity.zilla.runtime.engine.metrics.MetricContext; @@ -253,7 +254,7 @@ public EngineWorker( Collector collector, Supplier supplyEventReader, EventFormatterFactory eventFormatterFactory, - ResourceWatcher resourceWatcher, + ResourceWatchManager resourceWatchManager, int index, boolean readonly, Consumer process) @@ -428,7 +429,7 @@ public EngineWorker( this.registry = new EngineRegistry( bindingsByType::get, guardsByType::get, vaultsByType::get, catalogsByType::get, metricsByName::get, exportersByType::get, labels::supplyLabelId, this::onExporterAttached, this::onExporterDetached, - this::supplyMetricWriter, this::detachStreams, collector, process, resourceWatcher); + this::supplyMetricWriter, this::detachStreams, collector, process, resourceWatchManager); this.taskQueue = new ConcurrentLinkedDeque<>(); this.correlations = new Long2ObjectHashMap<>(); diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java index 5937785181..2d90e82f2d 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java @@ -44,6 +44,7 @@ import io.aklivity.zilla.runtime.engine.exporter.ExporterContext; import io.aklivity.zilla.runtime.engine.exporter.ExporterHandler; import io.aklivity.zilla.runtime.engine.guard.GuardContext; +import io.aklivity.zilla.runtime.engine.internal.watcher.ResourceWatchManager; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.Metric; import io.aklivity.zilla.runtime.engine.metrics.MetricContext; @@ -74,7 +75,7 @@ public class NamespaceRegistry private final ObjectLongLongFunction supplyMetricRecorder; private final LongConsumer detachBinding; private final Collector collector; - private final ResourceWatcher resourceWatcher; + private final ResourceWatchManager resourceWatchManager; public NamespaceRegistry( NamespaceConfig namespace, @@ -91,7 +92,7 @@ public NamespaceRegistry( ObjectLongLongFunction supplyMetricRecorder, LongConsumer detachBinding, Collector collector, - ResourceWatcher resourceWatcher) + ResourceWatchManager resourceWatchManager) { this.namespace = namespace; this.bindingsByType = bindingsByType; @@ -114,7 +115,7 @@ public NamespaceRegistry( this.metricsById = new Int2ObjectHashMap<>(); this.exportersById = new Int2ObjectHashMap<>(); this.collector = collector; - this.resourceWatcher = resourceWatcher; + this.resourceWatchManager = resourceWatchManager; } public int namespaceId() @@ -268,7 +269,7 @@ private void attachVault( VaultRegistry registry = new VaultRegistry(config, context); vaultsById.put(vaultId, registry); registry.attach(); - resourceWatcher.addResources(registry.handler().resources(), config.namespace); + resourceWatchManager.addResources(registry.handler().resources(), config.namespace); } private void detachVault( @@ -315,7 +316,7 @@ private void attachCatalog( CatalogRegistry registry = new CatalogRegistry(config, context); catalogsById.put(catalogId, registry); registry.attach(); - resourceWatcher.addResources(registry.handler().resources(), config.namespace); + resourceWatchManager.addResources(registry.handler().resources(), config.namespace); } private void detachCatalog( diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/FileWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigFileWatcherTask.java similarity index 59% rename from runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/FileWatcherTask.java rename to runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigFileWatcherTask.java index 9e7a5e8382..7fda5d39da 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/FileWatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigFileWatcherTask.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations * under the License. */ -package io.aklivity.zilla.runtime.engine.internal.registry; +package io.aklivity.zilla.runtime.engine.internal.watcher; import static org.agrona.LangUtil.rethrowUnchecked; @@ -25,29 +25,27 @@ import java.nio.file.WatchService; import java.util.IdentityHashMap; import java.util.Map; -import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import java.util.function.BiFunction; -import java.util.function.Consumer; import java.util.function.Function; import io.aklivity.zilla.runtime.engine.config.EngineConfig; -public class FileWatcherTask extends WatcherTask +public class ConfigFileWatcherTask extends WatcherTask implements ConfigWatcher { - private final Map watchedConfigs; + private final Map watchedItems; private final WatchService watchService; + private final BiFunction configChangeListener; private final Function readURL; - public FileWatcherTask( + public ConfigFileWatcherTask( BiFunction configChangeListener, - Consumer> resourceChangeListener, Function readURL) { - super(configChangeListener, resourceChangeListener); + this.configChangeListener = configChangeListener; this.readURL = readURL; - this.watchedConfigs = new IdentityHashMap<>(); + this.watchedItems = new IdentityHashMap<>(); WatchService watchService = null; try @@ -78,27 +76,23 @@ public Void call() { final WatchKey key = watchService.take(); - WatchedConfig watchedConfig = watchedConfigs.get(key); + WatchedItem watchedItem = watchedItems.get(key); - if (watchedConfig != null && watchedConfig.isWatchedKey(key)) + if (watchedItem != null && watchedItem.isWatchedKey(key)) { // Even if no reconfigure needed, recalculation is necessary, since symlinks might have changed. - watchedConfig.keys().forEach(watchedConfigs::remove); - watchedConfig.unregister(); - watchedConfig.register(); - watchedConfig.keys().forEach(k -> watchedConfigs.put(k, watchedConfig)); - String newConfigText = readURL.apply(watchedConfig.getURL().toString()); - byte[] newConfigHash = computeHash(newConfigText); - if (watchedConfig.isReconfigureNeeded(newConfigHash)) + watchedItem.keys().forEach(watchedItems::remove); + watchedItem.unregister(); + watchedItem.register(); + watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); + String newText = readURL.apply(watchedItem.getURL().toString()); + byte[] newHash = computeHash(newText); + if (watchedItem.isReconfigureNeeded(newHash)) { - watchedConfig.setConfigHash(newConfigHash); + watchedItem.setHash(newHash); if (configChangeListener != null) { - configChangeListener.apply(watchedConfig.getURL(), newConfigText); - } - if (resourceChangeListener != null) - { - resourceChangeListener.accept(namespaces); + configChangeListener.apply(watchedItem.getURL(), newText); } } } @@ -116,11 +110,11 @@ public Void call() public CompletableFuture watchConfig( URL configURL) { - WatchedConfig watchedConfig = new WatchedConfig(configURL, watchService); - watchedConfig.register(); - watchedConfig.keys().forEach(k -> watchedConfigs.put(k, watchedConfig)); + WatchedItem watchedItem = new WatchedItem(configURL, watchService); + watchedItem.register(); + watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); String configText = readURL.apply(configURL.toString()); - watchedConfig.setConfigHash(computeHash(configText)); + watchedItem.setHash(computeHash(configText)); CompletableFuture configFuture; try @@ -136,17 +130,6 @@ public CompletableFuture watchConfig( return configFuture; } - @Override - public void watchResource( - URL resourceURL) - { - WatchedConfig watchedConfig = new WatchedConfig(resourceURL, watchService); - watchedConfig.register(); - watchedConfig.keys().forEach(k -> watchedConfigs.put(k, watchedConfig)); - String resource = readURL.apply(resourceURL.toString()); - watchedConfig.setConfigHash(computeHash(resource)); - } - @Override public void close() throws IOException { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/HttpWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigHttpWatcherTask.java similarity index 94% rename from runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/HttpWatcherTask.java rename to runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigHttpWatcherTask.java index 2ccadb83da..b616436981 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/HttpWatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigHttpWatcherTask.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations * under the License. */ -package io.aklivity.zilla.runtime.engine.internal.registry; +package io.aklivity.zilla.runtime.engine.internal.watcher; import static java.net.http.HttpClient.Redirect.NORMAL; import static java.net.http.HttpClient.Version.HTTP_2; @@ -29,7 +29,6 @@ import java.util.Arrays; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -37,26 +36,25 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; -import java.util.function.Consumer; import io.aklivity.zilla.runtime.engine.config.EngineConfig; -public class HttpWatcherTask extends WatcherTask +public class ConfigHttpWatcherTask extends WatcherTask implements ConfigWatcher { private static final URI CLOSE_REQUESTED = URI.create("http://localhost:12345"); + private final BiFunction configChangeListener; private final Map etags; private final Map configHashes; private final Map> futures; private final BlockingQueue configQueue; private final int pollSeconds; - public HttpWatcherTask( + public ConfigHttpWatcherTask( BiFunction configChangeListener, - Consumer> resourceChangeListener, int pollSeconds) { - super(configChangeListener, resourceChangeListener); + this.configChangeListener = configChangeListener; this.etags = new ConcurrentHashMap<>(); this.configHashes = new ConcurrentHashMap<>(); this.futures = new ConcurrentHashMap<>(); @@ -106,13 +104,6 @@ public CompletableFuture watchConfig( return configFuture; } - @Override - public void watchResource( - URL resourceURL) - { - // TODO: Ati - } - @Override public void close() { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcher.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcher.java new file mode 100644 index 0000000000..1f15cae833 --- /dev/null +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcher.java @@ -0,0 +1,30 @@ +/* + * Copyright 2021-2023 Aklivity Inc. + * + * Aklivity licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.aklivity.zilla.runtime.engine.internal.watcher; + +import java.io.Closeable; +import java.net.URL; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; + +import io.aklivity.zilla.runtime.engine.config.EngineConfig; + +public interface ConfigWatcher extends Closeable +{ + Future submit(); + + CompletableFuture watchConfig(URL configURL); +} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceFileWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceFileWatcherTask.java new file mode 100644 index 0000000000..f2c8910517 --- /dev/null +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceFileWatcherTask.java @@ -0,0 +1,140 @@ +/* + * Copyright 2021-2023 Aklivity Inc. + * + * Aklivity licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.aklivity.zilla.runtime.engine.internal.watcher; + +import static org.agrona.LangUtil.rethrowUnchecked; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.ClosedWatchServiceException; +import java.nio.file.FileSystems; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Future; +import java.util.function.Consumer; +import java.util.function.Function; + +public class ResourceFileWatcherTask extends WatcherTask implements ResourceWatcher +{ + private final Map watchedItems; + private final WatchService watchService; + private final Consumer> resourceChangeListener; + private final Function readURL; + private final Set namespaces; + + public ResourceFileWatcherTask( + Consumer> resourceChangeListener, + Function readURL) + { + this.resourceChangeListener = resourceChangeListener; + this.readURL = readURL; + this.watchedItems = new IdentityHashMap<>(); + this.namespaces = new HashSet<>(); + WatchService watchService = null; + + try + { + watchService = FileSystems.getDefault().newWatchService(); + } + catch (IOException ex) + { + rethrowUnchecked(ex); + } + + this.watchService = watchService; + + } + + @Override + public Future submit() + { + return executor.submit(this); + } + + @Override + public Void call() + { + while (true) + { + try + { + final WatchKey key = watchService.take(); + + WatchedItem watchedItem = watchedItems.get(key); + + if (watchedItem != null && watchedItem.isWatchedKey(key)) + { + // Even if no reconfigure needed, recalculation is necessary, since symlinks might have changed. + watchedItem.keys().forEach(watchedItems::remove); + watchedItem.unregister(); + watchedItem.register(); + watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); + String newText = readURL.apply(watchedItem.getURL().toString()); + byte[] newHash = computeHash(newText); + if (watchedItem.isReconfigureNeeded(newHash)) + { + watchedItem.setHash(newHash); + if (resourceChangeListener != null) + { + resourceChangeListener.accept(namespaces); + } + } + } + } + catch (InterruptedException | ClosedWatchServiceException ex) + { + break; + } + } + + return null; + } + + @Override + public void watchResource( + URL resourceURL) + { + WatchedItem watchedItem = new WatchedItem(resourceURL, watchService); + watchedItem.register(); + watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); + String resource = readURL.apply(resourceURL.toString()); + watchedItem.setHash(computeHash(resource)); + } + + @Override + public void addNamespace( + String namespace) + { + namespaces.add(namespace); + } + + @Override + public void removeNamespace( + String namespace) + { + namespaces.remove(namespace); + } + + @Override + public void close() throws IOException + { + watchService.close(); + } +} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/ResourceWatcher.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java similarity index 90% rename from runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/ResourceWatcher.java rename to runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java index d709bd2394..7b7f1b2a80 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/ResourceWatcher.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations * under the License. */ -package io.aklivity.zilla.runtime.engine.internal.registry; +package io.aklivity.zilla.runtime.engine.internal.watcher; import static org.agrona.LangUtil.rethrowUnchecked; @@ -26,15 +26,15 @@ import java.util.function.Consumer; import java.util.function.Function; -public class ResourceWatcher +public class ResourceWatchManager { private final Map> resources; - private final Map resourceTasks; + private final Map resourceTasks; private Consumer> resourceChangeListener; private Function readURL; - public ResourceWatcher() + public ResourceWatchManager() { this.resources = new ConcurrentHashMap<>(); this.resourceTasks = new ConcurrentHashMap<>(); @@ -60,7 +60,7 @@ public void addResources( return ConcurrentHashMap.newKeySet(); } ).add(namespace); - resourceTasks.get(resource).addNamespaces(namespace); + resourceTasks.get(resource).addNamespace(namespace); } ); } @@ -93,8 +93,8 @@ private void startWatchingResource( { try { - FileWatcherTask watcherTask = new FileWatcherTask(null, resourceChangeListener, readURL); - watcherTask.addNamespaces(namespace); + ResourceWatcher watcherTask = new ResourceFileWatcherTask(resourceChangeListener, readURL); + watcherTask.addNamespace(namespace); watcherTask.submit(); URL resourceURL = Path.of(resource).toUri().toURL(); watcherTask.watchResource(resourceURL); diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcher.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcher.java new file mode 100644 index 0000000000..20b0dcc9d1 --- /dev/null +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcher.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021-2023 Aklivity Inc. + * + * Aklivity licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.aklivity.zilla.runtime.engine.internal.watcher; + +import java.io.Closeable; +import java.net.URL; +import java.util.concurrent.Future; + +public interface ResourceWatcher extends Closeable +{ + Future submit(); + + void watchResource(URL configURL); + + void addNamespace(String namespace); + + void removeNamespace(String namespace); +} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/WatchedConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatchedItem.java similarity index 81% rename from runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/WatchedConfig.java rename to runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatchedItem.java index a6ee866978..5bbacf7ef3 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/WatchedConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatchedItem.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations * under the License. */ -package io.aklivity.zilla.runtime.engine.internal.registry; +package io.aklivity.zilla.runtime.engine.internal.watcher; import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; @@ -33,20 +33,20 @@ import java.util.LinkedList; import java.util.Set; -public class WatchedConfig +public class WatchedItem { private final WatchService watchService; private final Set watchKeys; - private final URL configURL; - private byte[] configHash; + private final URL watchedURL; + private byte[] hash; - public WatchedConfig( - URL configURL, + public WatchedItem( + URL watchedURL, WatchService watchService) { this.watchService = watchService; this.watchKeys = new HashSet<>(); - this.configURL = configURL; + this.watchedURL = watchedURL; } public Set keys() @@ -56,7 +56,7 @@ public Set keys() public void register() { - Path configPath = Paths.get(configURL.getPath()).toAbsolutePath(); + Path configPath = Paths.get(watchedURL.getPath()).toAbsolutePath(); try { Set watchedPaths = new HashSet<>(); @@ -123,27 +123,27 @@ public boolean isWatchedKey( public boolean isReconfigureNeeded( byte[] newConfigHash) { - return !Arrays.equals(configHash, newConfigHash); + return !Arrays.equals(hash, newConfigHash); } - public void setConfigHash( - byte[] newConfigHash) + public void setHash( + byte[] newHash) { - configHash = newConfigHash; + hash = newHash; } public URL getURL() { - return configURL; + return watchedURL; } private WatchKey registerPath( - Path configPath) + Path path) { WatchKey key = null; try { - key = configPath.register(watchService, ENTRY_MODIFY, ENTRY_CREATE, ENTRY_DELETE); + key = path.register(watchService, ENTRY_MODIFY, ENTRY_CREATE, ENTRY_DELETE); } catch (IOException ex) { @@ -151,19 +151,4 @@ private WatchKey registerPath( } return key; } - - private Path toRealPath( - Path configPath) - { - try - { - configPath = configPath.toRealPath(); - } - catch (IOException ex) - { - rethrowUnchecked(ex); - } - return configPath; - } - } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/WatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatcherTask.java similarity index 51% rename from runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/WatcherTask.java rename to runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatcherTask.java index 173f7a1d0e..77c532efd7 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/WatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatcherTask.java @@ -13,71 +13,33 @@ * License for the specific language governing permissions and limitations * under the License. */ -package io.aklivity.zilla.runtime.engine.internal.registry; +package io.aklivity.zilla.runtime.engine.internal.watcher; import static java.nio.charset.StandardCharsets.UTF_8; import static org.agrona.LangUtil.rethrowUnchecked; -import java.io.Closeable; -import java.net.URL; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.HashSet; -import java.util.Set; import java.util.concurrent.Callable; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; -import java.util.function.BiFunction; -import java.util.function.Consumer; -import io.aklivity.zilla.runtime.engine.config.EngineConfig; - -public abstract class WatcherTask implements Callable, Closeable +public abstract class WatcherTask implements Callable { private final MessageDigest md5; protected final ScheduledExecutorService executor; - protected final BiFunction configChangeListener; - protected final Consumer> resourceChangeListener; - protected final Set namespaces; - protected WatcherTask( - BiFunction configChangeListener, - Consumer> resourceChangeListener) + protected WatcherTask() { - this.configChangeListener = configChangeListener; - this.resourceChangeListener = resourceChangeListener; this.md5 = initMessageDigest("MD5"); this.executor = Executors.newScheduledThreadPool(2); - this.namespaces = new HashSet<>(); - } - - public void addNamespaces( - String namespace) - { - namespaces.add(namespace); - } - - public void removeNamespace( - String namespace) - { - namespaces.remove(namespace); } - public abstract Future submit(); - - public abstract CompletableFuture watchConfig( - URL configURL); - - public abstract void watchResource( - URL resourceURL); - protected byte[] computeHash( - String configText) + String text) { - return md5.digest(configText.getBytes(UTF_8)); + return md5.digest(text.getBytes(UTF_8)); } private MessageDigest initMessageDigest( From 8fcae70855f77ae576a50ab20c4ceddebaea9537 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Wed, 5 Jun 2024 20:46:31 +0200 Subject: [PATCH 04/43] WIP refactoring 2 --- .../aklivity/zilla/runtime/engine/Engine.java | 97 +++---- .../internal/registry/EngineManager.java | 15 +- .../watcher/ConfigHttpWatcherTask.java | 252 ------------------ .../internal/watcher/ConfigWatcher.java | 30 --- ...atcherTask.java => ConfigWatcherTask.java} | 24 +- .../watcher/ResourceWatchManager.java | 8 +- .../internal/watcher/ResourceWatcher.java | 31 --- ...cherTask.java => ResourceWatcherTask.java} | 16 +- .../engine/internal/watcher/WatcherTask.java | 5 +- 9 files changed, 68 insertions(+), 410 deletions(-) delete mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigHttpWatcherTask.java delete mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcher.java rename runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/{ConfigFileWatcherTask.java => ConfigWatcherTask.java} (87%) delete mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcher.java rename runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/{ResourceFileWatcherTask.java => ResourceWatcherTask.java} (93%) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index 0de5794dc1..c9c90bde1c 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -16,21 +16,15 @@ package io.aklivity.zilla.runtime.engine; import static io.aklivity.zilla.runtime.engine.internal.layouts.metrics.HistogramsLayout.BUCKETS; -import static java.net.http.HttpClient.Redirect.NORMAL; -import static java.net.http.HttpClient.Version.HTTP_2; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.Executors.newFixedThreadPool; import static java.util.stream.Collectors.toList; import static org.agrona.LangUtil.rethrowUnchecked; -import java.io.IOException; -import java.io.InputStream; -import java.net.URISyntaxException; +import java.net.URI; import java.net.URL; -import java.net.URLConnection; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -44,7 +38,6 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.IntFunction; import java.util.function.LongConsumer; import java.util.function.LongSupplier; @@ -72,9 +65,7 @@ import io.aklivity.zilla.runtime.engine.internal.registry.EngineManager; import io.aklivity.zilla.runtime.engine.internal.registry.EngineWorker; import io.aklivity.zilla.runtime.engine.internal.types.event.EventFW; -import io.aklivity.zilla.runtime.engine.internal.watcher.ConfigFileWatcherTask; -import io.aklivity.zilla.runtime.engine.internal.watcher.ConfigHttpWatcherTask; -import io.aklivity.zilla.runtime.engine.internal.watcher.ConfigWatcher; +import io.aklivity.zilla.runtime.engine.internal.watcher.ConfigWatcherTask; import io.aklivity.zilla.runtime.engine.internal.watcher.ResourceWatchManager; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.MetricGroup; @@ -93,7 +84,7 @@ public final class Engine implements Collector, AutoCloseable private final AtomicInteger nextTaskId; private final ThreadFactory factory; - private final ConfigWatcher configWatcherTask; + private final ConfigWatcherTask configWatcherTask; private final URL configURL; private final List workers; private final boolean readonly; @@ -213,22 +204,9 @@ public final class Engine implements Collector, AutoCloseable resourceWatchManager); this.configURL = config.configURL(); - String protocol = configURL.getProtocol(); - if ("file".equals(protocol) || "jar".equals(protocol)) - { - Function watcherReadURL = l -> readURL(configURL, l); - this.configWatcherTask = new ConfigFileWatcherTask(manager::reconfigure, watcherReadURL); - this.resourceWatchManager.initialize(manager::reloadNamespacesWithChangedResources, watcherReadURL); - } - else if ("http".equals(protocol) || "https".equals(protocol)) - { - this.configWatcherTask = new ConfigHttpWatcherTask(manager::reconfigure, config.configPollIntervalSeconds()); - // TODO: Ati - implement http - } - else - { - throw new UnsupportedOperationException(); - } + FileSystem fileSystem = getFileSystem(configURL.toString()); + this.configWatcherTask = new ConfigWatcherTask(fileSystem, manager::reconfigure, this::readURL); + this.resourceWatchManager.initialize(fileSystem, manager::reloadNamespacesWithChangedResources, this::readURL); this.bindings = bindings; this.tasks = tasks; @@ -324,46 +302,45 @@ public static EngineBuilder builder() } private String readURL( - URL configURL, String location) { - String output = null; + String result = null; try { - final URL fileURL = new URL(configURL, location); - if ("http".equals(fileURL.getProtocol()) || "https".equals(fileURL.getProtocol())) + URI uri = new URI(location); + if ("file".equals(uri.getScheme())) { - HttpClient client = HttpClient.newBuilder() - .version(HTTP_2) - .followRedirects(NORMAL) - .build(); - - HttpRequest request = HttpRequest.newBuilder() - .GET() - .uri(fileURL.toURI()) - .build(); - - HttpResponse response = client.send( - request, - HttpResponse.BodyHandlers.ofString()); - - output = response.body(); + location = uri.getSchemeSpecificPart(); } - else - { + // TODO: Ati - check this after adding hfs; this should just work fine for http + result = Files.readString(Path.of(location)); + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } + return result; + } - URLConnection connection = fileURL.openConnection(); - try (InputStream input = connection.getInputStream()) - { - output = new String(input.readAllBytes(), UTF_8); - } + private FileSystem getFileSystem( + String location) + { + FileSystem result = null; + try + { + URI uri = new URI(location); + if ("file".equals(uri.getScheme())) + { + location = uri.getSchemeSpecificPart(); } + // TODO: Ati - check this after adding hfs; this should just work fine for http + result = Path.of(location).getFileSystem(); } - catch (IOException | URISyntaxException | InterruptedException ex) + catch (Exception ex) { - output = ""; + rethrowUnchecked(ex); } - return output; + return result; } private Thread newTaskThread( diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index 5459f142f4..a10fa4f9f5 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.IntFunction; @@ -83,7 +82,7 @@ public class EngineManager private final EngineExtContext context; private final EngineConfiguration config; private final List extensions; - private final BiFunction readURL; + private final Function readURL; private final Resolver expressions; private final ResourceWatchManager resourceWatchManager; @@ -102,7 +101,7 @@ public EngineManager( EngineExtContext context, EngineConfiguration config, List extensions, - BiFunction readURL, + Function readURL, ResourceWatchManager resourceWatchManager) { this.schemaTypes = schemaTypes; @@ -123,14 +122,13 @@ public EngineManager( } public EngineConfig reconfigure( - URL configURL, String configText) { EngineConfig newConfig = null; try { - newConfig = parse(configURL, configText); + newConfig = parse(configText); if (newConfig != null) { final EngineConfig oldConfig = current; @@ -199,7 +197,6 @@ public void process( } private EngineConfig parse( - URL configURL, String configText) { EngineConfig engine = null; @@ -213,11 +210,9 @@ private EngineConfig parse( try { - final Function namespaceReadURL = l -> readURL.apply(configURL, l); - EngineConfigReader reader = new EngineConfigReader( config, - new NamespaceConfigAdapterContext(namespaceReadURL), + new NamespaceConfigAdapterContext(readURL), expressions, schemaTypes, logger); @@ -231,7 +226,7 @@ private EngineConfig parse( for (NamespaceConfig namespace : engine.namespaces) { - namespace.readURL = l -> readURL.apply(configURL, l); + namespace.readURL = readURL; process(guards, namespace); } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigHttpWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigHttpWatcherTask.java deleted file mode 100644 index b616436981..0000000000 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigHttpWatcherTask.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Copyright 2021-2023 Aklivity Inc. - * - * Aklivity licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package io.aklivity.zilla.runtime.engine.internal.watcher; - -import static java.net.http.HttpClient.Redirect.NORMAL; -import static java.net.http.HttpClient.Version.HTTP_2; -import static org.agrona.LangUtil.rethrowUnchecked; - -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.util.Arrays; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.function.BiFunction; - -import io.aklivity.zilla.runtime.engine.config.EngineConfig; - -public class ConfigHttpWatcherTask extends WatcherTask implements ConfigWatcher -{ - private static final URI CLOSE_REQUESTED = URI.create("http://localhost:12345"); - - private final BiFunction configChangeListener; - private final Map etags; - private final Map configHashes; - private final Map> futures; - private final BlockingQueue configQueue; - private final int pollSeconds; - - public ConfigHttpWatcherTask( - BiFunction configChangeListener, - int pollSeconds) - { - this.configChangeListener = configChangeListener; - this.etags = new ConcurrentHashMap<>(); - this.configHashes = new ConcurrentHashMap<>(); - this.futures = new ConcurrentHashMap<>(); - this.configQueue = new LinkedBlockingQueue<>(); - this.pollSeconds = pollSeconds; - } - - @Override - public Future submit() - { - return executor.submit(this); - } - - @Override - public Void call() throws InterruptedException - { - while (true) - { - URI configURI = configQueue.take(); - if (configURI == CLOSE_REQUESTED) - { - break; - } - String etag = etags.getOrDefault(configURI, ""); - sendAsync(configURI, etag); - } - return null; - } - - @Override - public CompletableFuture watchConfig( - URL configURL) - { - URI configURI = toURI(configURL); - - CompletableFuture configFuture; - try - { - EngineConfig config = sendSync(configURI); - configFuture = CompletableFuture.completedFuture(config); - } - catch (Exception ex) - { - configFuture = CompletableFuture.failedFuture(ex); - } - - return configFuture; - } - - @Override - public void close() - { - futures.values().forEach(future -> future.cancel(true)); - configQueue.add(CLOSE_REQUESTED); - } - - private EngineConfig sendSync( - URI configURI) - { - HttpClient client = HttpClient.newBuilder() - .version(HTTP_2) - .followRedirects(NORMAL) - .build(); - HttpRequest request = HttpRequest.newBuilder() - .GET() - .uri(configURI) - .build(); - HttpResponse response; - try - { - response = client.send(request, HttpResponse.BodyHandlers.ofString()); - } - catch (Exception ex) - { - handleException(ex, configURI); - return null; - } - return handleConfigChange(response); - } - - private void sendAsync( - URI configURI, - String etag) - { - HttpClient client = HttpClient.newBuilder() - .version(HTTP_2) - .followRedirects(NORMAL) - .build(); - HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() - .GET() - .uri(configURI); - if (etag != null && !etag.isEmpty()) - { - requestBuilder = requestBuilder.headers("If-None-Match", etag, "Prefer", "wait=86400"); - } - - CompletableFuture future = client.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) - .thenAccept(this::handleConfigChange) - .exceptionally(ex -> handleException(ex, configURI)); - futures.put(configURI, future); - } - - private Void handleException( - Throwable throwable, - URI configURI) - { - scheduleRequest(configURI, pollSeconds); - return null; - } - - private EngineConfig handleConfigChange( - HttpResponse response) - { - EngineConfig config = null; - try - { - URI configURI = response.request().uri(); - int statusCode = response.statusCode(); - int pollIntervalSeconds = 0; - if (statusCode == 404) - { - config = configChangeListener.apply(configURI.toURL(), ""); - pollIntervalSeconds = this.pollSeconds; - } - else if (statusCode >= 500 && statusCode <= 599) - { - pollIntervalSeconds = this.pollSeconds; - } - else - { - Optional etagOptional = response.headers().firstValue("Etag"); - String configText = response.body(); - - if (etagOptional.isPresent()) - { - String oldEtag = etags.getOrDefault(configURI, ""); - if (!oldEtag.equals(etagOptional.get())) - { - etags.put(configURI, etagOptional.get()); - config = configChangeListener.apply(configURI.toURL(), configText); - } - else if (response.statusCode() != 304) - { - pollIntervalSeconds = this.pollSeconds; - } - } - else - { - byte[] configHash = configHashes.get(configURI); - byte[] newConfigHash = computeHash(configText); - if (!Arrays.equals(configHash, newConfigHash)) - { - configHashes.put(configURI, newConfigHash); - config = configChangeListener.apply(configURI.toURL(), configText); - } - pollIntervalSeconds = this.pollSeconds; - } - } - futures.remove(configURI); - scheduleRequest(configURI, pollIntervalSeconds); - } - catch (MalformedURLException ex) - { - rethrowUnchecked(ex); - } - return config; - } - - private void scheduleRequest(URI configURI, int pollIntervalSeconds) - { - if (pollIntervalSeconds == 0) - { - configQueue.add(configURI); - } - else - { - executor.schedule(() -> configQueue.add(configURI), pollIntervalSeconds, TimeUnit.SECONDS); - } - } - - private URI toURI( - URL configURL) - { - URI configURI = null; - try - { - configURI = configURL.toURI(); - } - catch (URISyntaxException ex) - { - rethrowUnchecked(ex); - } - return configURI; - } -} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcher.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcher.java deleted file mode 100644 index 1f15cae833..0000000000 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcher.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2021-2023 Aklivity Inc. - * - * Aklivity licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package io.aklivity.zilla.runtime.engine.internal.watcher; - -import java.io.Closeable; -import java.net.URL; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; - -import io.aklivity.zilla.runtime.engine.config.EngineConfig; - -public interface ConfigWatcher extends Closeable -{ - Future submit(); - - CompletableFuture watchConfig(URL configURL); -} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigFileWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java similarity index 87% rename from runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigFileWatcherTask.java rename to runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java index 7fda5d39da..7e48037db9 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigFileWatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java @@ -20,45 +20,42 @@ import java.io.IOException; import java.net.URL; import java.nio.file.ClosedWatchServiceException; -import java.nio.file.FileSystems; +import java.nio.file.FileSystem; import java.nio.file.WatchKey; import java.nio.file.WatchService; import java.util.IdentityHashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; -import java.util.function.BiFunction; import java.util.function.Function; import io.aklivity.zilla.runtime.engine.config.EngineConfig; -public class ConfigFileWatcherTask extends WatcherTask implements ConfigWatcher +public class ConfigWatcherTask extends WatcherTask { private final Map watchedItems; private final WatchService watchService; - private final BiFunction configChangeListener; + private final Function configChangeListener; private final Function readURL; - public ConfigFileWatcherTask( - BiFunction configChangeListener, + public ConfigWatcherTask( + FileSystem fileSystem, + Function configChangeListener, Function readURL) { this.configChangeListener = configChangeListener; this.readURL = readURL; this.watchedItems = new IdentityHashMap<>(); WatchService watchService = null; - try { - watchService = FileSystems.getDefault().newWatchService(); + watchService = fileSystem.newWatchService(); } - catch (IOException ex) + catch (Exception ex) { rethrowUnchecked(ex); } - this.watchService = watchService; - } @Override @@ -92,7 +89,7 @@ public Void call() watchedItem.setHash(newHash); if (configChangeListener != null) { - configChangeListener.apply(watchedItem.getURL(), newText); + configChangeListener.apply(newText); } } } @@ -106,7 +103,6 @@ public Void call() return null; } - @Override public CompletableFuture watchConfig( URL configURL) { @@ -119,7 +115,7 @@ public CompletableFuture watchConfig( CompletableFuture configFuture; try { - EngineConfig config = configChangeListener.apply(configURL, configText); + EngineConfig config = configChangeListener.apply(configText); configFuture = CompletableFuture.completedFuture(config); } catch (Exception ex) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java index 7b7f1b2a80..d5f20fed94 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java @@ -18,6 +18,7 @@ import static org.agrona.LangUtil.rethrowUnchecked; import java.net.URL; +import java.nio.file.FileSystem; import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -29,8 +30,9 @@ public class ResourceWatchManager { private final Map> resources; - private final Map resourceTasks; + private final Map resourceTasks; + private FileSystem fileSystem; private Consumer> resourceChangeListener; private Function readURL; @@ -41,9 +43,11 @@ public ResourceWatchManager() } public void initialize( + FileSystem fileSystem, Consumer> resourceChangeListener, Function readURL) { + this.fileSystem = fileSystem; this.resourceChangeListener = resourceChangeListener; this.readURL = readURL; } @@ -93,7 +97,7 @@ private void startWatchingResource( { try { - ResourceWatcher watcherTask = new ResourceFileWatcherTask(resourceChangeListener, readURL); + ResourceWatcherTask watcherTask = new ResourceWatcherTask(fileSystem, resourceChangeListener, readURL); watcherTask.addNamespace(namespace); watcherTask.submit(); URL resourceURL = Path.of(resource).toUri().toURL(); diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcher.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcher.java deleted file mode 100644 index 20b0dcc9d1..0000000000 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcher.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2021-2023 Aklivity Inc. - * - * Aklivity licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package io.aklivity.zilla.runtime.engine.internal.watcher; - -import java.io.Closeable; -import java.net.URL; -import java.util.concurrent.Future; - -public interface ResourceWatcher extends Closeable -{ - Future submit(); - - void watchResource(URL configURL); - - void addNamespace(String namespace); - - void removeNamespace(String namespace); -} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceFileWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcherTask.java similarity index 93% rename from runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceFileWatcherTask.java rename to runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcherTask.java index f2c8910517..4ea9238fc7 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceFileWatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcherTask.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.net.URL; import java.nio.file.ClosedWatchServiceException; -import java.nio.file.FileSystems; +import java.nio.file.FileSystem; import java.nio.file.WatchKey; import java.nio.file.WatchService; import java.util.HashSet; @@ -31,7 +31,7 @@ import java.util.function.Consumer; import java.util.function.Function; -public class ResourceFileWatcherTask extends WatcherTask implements ResourceWatcher +public class ResourceWatcherTask extends WatcherTask { private final Map watchedItems; private final WatchService watchService; @@ -39,7 +39,8 @@ public class ResourceFileWatcherTask extends WatcherTask implements ResourceWatc private final Function readURL; private final Set namespaces; - public ResourceFileWatcherTask( + public ResourceWatcherTask( + FileSystem fileSystem, Consumer> resourceChangeListener, Function readURL) { @@ -48,16 +49,14 @@ public ResourceFileWatcherTask( this.watchedItems = new IdentityHashMap<>(); this.namespaces = new HashSet<>(); WatchService watchService = null; - try { - watchService = FileSystems.getDefault().newWatchService(); + watchService = fileSystem.newWatchService(); } - catch (IOException ex) + catch (Exception ex) { rethrowUnchecked(ex); } - this.watchService = watchService; } @@ -107,7 +106,6 @@ public Void call() return null; } - @Override public void watchResource( URL resourceURL) { @@ -118,14 +116,12 @@ public void watchResource( watchedItem.setHash(computeHash(resource)); } - @Override public void addNamespace( String namespace) { namespaces.add(namespace); } - @Override public void removeNamespace( String namespace) { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatcherTask.java index 77c532efd7..5dc5834fc4 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatcherTask.java @@ -22,14 +22,17 @@ import java.security.NoSuchAlgorithmException; import java.util.concurrent.Callable; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; -public abstract class WatcherTask implements Callable +public abstract class WatcherTask implements Callable, AutoCloseable { private final MessageDigest md5; protected final ScheduledExecutorService executor; + abstract Future submit(); + protected WatcherTask() { this.md5 = initMessageDigest("MD5"); From 96f070ec53695d2e84055ead4f91d99a5a73a3fe Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 6 Jun 2024 09:54:33 +0200 Subject: [PATCH 05/43] WIP fix --- runtime/engine/pom.xml | 2 +- .../aklivity/zilla/runtime/engine/Engine.java | 34 ++++++++++------- .../watcher/ResourceWatchManager.java | 38 +++++++++++++++++-- .../engine/internal/ReconfigureHttpIT.java | 6 +++ 4 files changed, 61 insertions(+), 19 deletions(-) diff --git a/runtime/engine/pom.xml b/runtime/engine/pom.xml index 219c430e54..fd099257ac 100644 --- a/runtime/engine/pom.xml +++ b/runtime/engine/pom.xml @@ -26,7 +26,7 @@ 11 11 - 0.76 + 0.74 5 diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index c9c90bde1c..204ed0c075 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -16,14 +16,16 @@ package io.aklivity.zilla.runtime.engine; import static io.aklivity.zilla.runtime.engine.internal.layouts.metrics.HistogramsLayout.BUCKETS; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.Executors.newFixedThreadPool; import static java.util.stream.Collectors.toList; import static org.agrona.LangUtil.rethrowUnchecked; +import java.io.InputStream; import java.net.URI; import java.net.URL; +import java.net.URLConnection; import java.nio.file.FileSystem; -import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; @@ -204,9 +206,8 @@ public final class Engine implements Collector, AutoCloseable resourceWatchManager); this.configURL = config.configURL(); - FileSystem fileSystem = getFileSystem(configURL.toString()); - this.configWatcherTask = new ConfigWatcherTask(fileSystem, manager::reconfigure, this::readURL); - this.resourceWatchManager.initialize(fileSystem, manager::reloadNamespacesWithChangedResources, this::readURL); + this.configWatcherTask = new ConfigWatcherTask(getFileSystem(configURL), manager::reconfigure, this::readURL); + this.resourceWatchManager.initialize(configURL, manager::reloadNamespacesWithChangedResources, this::readURL); this.bindings = bindings; this.tasks = tasks; @@ -304,35 +305,40 @@ public static EngineBuilder builder() private String readURL( String location) { - String result = null; + String result; try { - URI uri = new URI(location); - if ("file".equals(uri.getScheme())) + URL url = new URL(configURL, location); + URLConnection connection = url.openConnection(); + try (InputStream input = connection.getInputStream()) { - location = uri.getSchemeSpecificPart(); + result = new String(input.readAllBytes(), UTF_8); } - // TODO: Ati - check this after adding hfs; this should just work fine for http - result = Files.readString(Path.of(location)); } catch (Exception ex) { - rethrowUnchecked(ex); + result = ""; } return result; } - private FileSystem getFileSystem( - String location) + // TODO: Ati - this is a code duplication, remove this + private static FileSystem getFileSystem( + URL url) { FileSystem result = null; try { - URI uri = new URI(location); + URI uri = url.toURI(); + String location; if ("file".equals(uri.getScheme())) { location = uri.getSchemeSpecificPart(); } + else + { + location = uri.toString(); + } // TODO: Ati - check this after adding hfs; this should just work fine for http result = Path.of(location).getFileSystem(); } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java index d5f20fed94..594788e890 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java @@ -17,6 +17,7 @@ import static org.agrona.LangUtil.rethrowUnchecked; +import java.net.URI; import java.net.URL; import java.nio.file.FileSystem; import java.nio.file.Path; @@ -32,6 +33,7 @@ public class ResourceWatchManager private final Map> resources; private final Map resourceTasks; + private URL configURL; private FileSystem fileSystem; private Consumer> resourceChangeListener; private Function readURL; @@ -43,11 +45,12 @@ public ResourceWatchManager() } public void initialize( - FileSystem fileSystem, + URL configURL, Consumer> resourceChangeListener, Function readURL) { - this.fileSystem = fileSystem; + this.configURL = configURL; + this.fileSystem = resolveFileSystem(configURL); this.resourceChangeListener = resourceChangeListener; this.readURL = readURL; } @@ -100,10 +103,10 @@ private void startWatchingResource( ResourceWatcherTask watcherTask = new ResourceWatcherTask(fileSystem, resourceChangeListener, readURL); watcherTask.addNamespace(namespace); watcherTask.submit(); - URL resourceURL = Path.of(resource).toUri().toURL(); + URL resourceURL = new URL(configURL, resource); watcherTask.watchResource(resourceURL); resourceTasks.put(resource, watcherTask); - System.out.println("started watching resource: " + resource); // TODO: Ati + System.out.printf("started watching resource: %s resourceURL: %s\n", resource, resourceURL); // TODO: Ati } catch (Exception ex) { @@ -146,4 +149,31 @@ public void close() } }); } + + // TODO: Ati - chk if this can be simplified + private static FileSystem resolveFileSystem( + URL url) + { + FileSystem result = null; + try + { + URI uri = url.toURI(); + String location; + if ("file".equals(uri.getScheme())) + { + location = uri.getSchemeSpecificPart(); + } + else + { + location = uri.toString(); + } + // TODO: Ati - check this after adding hfs; this should just work fine for http + result = Path.of(location).getFileSystem(); + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } + return result; + } } diff --git a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/ReconfigureHttpIT.java b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/ReconfigureHttpIT.java index c4dea0780b..5ce47cbef4 100644 --- a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/ReconfigureHttpIT.java +++ b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/ReconfigureHttpIT.java @@ -23,6 +23,7 @@ import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.DisableOnDebug; @@ -67,6 +68,7 @@ public void setupRegisterLatch() throws Exception EngineTest.TestEngineExt.registerLatch = new CountDownLatch(1); } + @Ignore // TODO: Ati @Test @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "1") @Configuration("http://localhost:8080/") @@ -83,6 +85,7 @@ public void shouldReconfigureWhenModifiedHttp() throws Exception k3po.finish(); } + @Ignore // TODO: Ati @Test @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "1") @Configuration("http://localhost:8080/") @@ -98,6 +101,7 @@ public void shouldReconfigureWhenCreatedHttp() throws Exception k3po.finish(); } + @Ignore // TODO: Ati @Test @Configuration("http://localhost:8080/") @Specification({ @@ -112,6 +116,7 @@ public void shouldReconfigureWhenDeletedHttp() throws Exception k3po.finish(); } + @Ignore // TODO: Ati @Test @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "1") @Configuration("http://localhost:8080/") @@ -128,6 +133,7 @@ public void shouldReconfigureWhenModifiedHttpEtagNotSupported() throws Exception k3po.finish(); } + @Ignore // TODO: Ati @Test @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "1") @Configuration("http://localhost:8080/") From df0b998f538fa0ddcc15c16423e3a10fce51e8fc Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 6 Jun 2024 14:10:18 +0200 Subject: [PATCH 06/43] WIP EngineConfigWatcher --- .../aklivity/zilla/runtime/engine/Engine.java | 56 +++---------------- .../internal/registry/EngineManager.java | 10 ++-- .../internal/registry/EngineRegistry.java | 10 ++-- .../internal/registry/EngineWorker.java | 6 +- .../internal/registry/NamespaceRegistry.java | 12 ++-- ...hManager.java => EngineConfigWatcher.java} | 23 +++++++- 6 files changed, 49 insertions(+), 68 deletions(-) rename runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/{ResourceWatchManager.java => EngineConfigWatcher.java} (88%) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index 204ed0c075..cd64a8ce22 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -22,11 +22,8 @@ import static org.agrona.LangUtil.rethrowUnchecked; import java.io.InputStream; -import java.net.URI; import java.net.URL; import java.net.URLConnection; -import java.nio.file.FileSystem; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -36,7 +33,6 @@ import java.util.ServiceLoader.Provider; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -67,8 +63,7 @@ import io.aklivity.zilla.runtime.engine.internal.registry.EngineManager; import io.aklivity.zilla.runtime.engine.internal.registry.EngineWorker; import io.aklivity.zilla.runtime.engine.internal.types.event.EventFW; -import io.aklivity.zilla.runtime.engine.internal.watcher.ConfigWatcherTask; -import io.aklivity.zilla.runtime.engine.internal.watcher.ResourceWatchManager; +import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.MetricGroup; import io.aklivity.zilla.runtime.engine.model.Model; @@ -86,15 +81,12 @@ public final class Engine implements Collector, AutoCloseable private final AtomicInteger nextTaskId; private final ThreadFactory factory; - private final ConfigWatcherTask configWatcherTask; private final URL configURL; private final List workers; private final boolean readonly; private final EngineConfiguration config; private final EngineManager manager; - private final ResourceWatchManager resourceWatchManager; - - private Future configWatcherTaskRef; + private final EngineConfigWatcher watcher; Engine( EngineConfiguration config, @@ -155,14 +147,14 @@ public final class Engine implements Collector, AutoCloseable } this.tuning = tuning; - this.resourceWatchManager = new ResourceWatchManager(); + this.watcher = new EngineConfigWatcher(); List workers = new ArrayList<>(workerCount); for (int workerIndex = 0; workerIndex < workerCount; workerIndex++) { EngineWorker worker = new EngineWorker(config, tasks, labels, errorHandler, tuning::affinity, bindings, exporters, guards, vaults, catalogs, models, metricGroups, this, this::supplyEventReader, - eventFormatterFactory, resourceWatchManager, workerIndex, readonly, this::process); + eventFormatterFactory, watcher, workerIndex, readonly, this::process); workers.add(worker); } this.workers = workers; @@ -203,11 +195,11 @@ public final class Engine implements Collector, AutoCloseable config, extensions, this::readURL, - resourceWatchManager); + watcher); this.configURL = config.configURL(); - this.configWatcherTask = new ConfigWatcherTask(getFileSystem(configURL), manager::reconfigure, this::readURL); - this.resourceWatchManager.initialize(configURL, manager::reloadNamespacesWithChangedResources, this::readURL); + this.watcher.initialize(configURL, manager::reconfigure, manager::reloadNamespacesWithChangedResources, + this::readURL); this.bindings = bindings; this.tasks = tasks; @@ -240,11 +232,10 @@ public void start() throws Exception worker.doStart(); } - configWatcherTaskRef = configWatcherTask.submit(); if (!readonly) { // ignore the config file in read-only mode; no config will be read so no namespaces, bindings, etc will be attached - configWatcherTask.watchConfig(configURL).get(); + watcher.startWatchingConfig(); } } @@ -258,9 +249,7 @@ public void close() throws Exception final List errors = new ArrayList<>(); - resourceWatchManager.close(); - configWatcherTask.close(); - configWatcherTaskRef.get(); + watcher.close(); for (EngineWorker worker : workers) { @@ -322,33 +311,6 @@ private String readURL( return result; } - // TODO: Ati - this is a code duplication, remove this - private static FileSystem getFileSystem( - URL url) - { - FileSystem result = null; - try - { - URI uri = url.toURI(); - String location; - if ("file".equals(uri.getScheme())) - { - location = uri.getSchemeSpecificPart(); - } - else - { - location = uri.toString(); - } - // TODO: Ati - check this after adding hfs; this should just work fine for http - result = Path.of(location).getFileSystem(); - } - catch (Exception ex) - { - rethrowUnchecked(ex); - } - return result; - } - private Thread newTaskThread( Runnable r) { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index a10fa4f9f5..da07478473 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -62,7 +62,7 @@ import io.aklivity.zilla.runtime.engine.guard.Guard; import io.aklivity.zilla.runtime.engine.internal.Tuning; import io.aklivity.zilla.runtime.engine.internal.config.NamespaceAdapter; -import io.aklivity.zilla.runtime.engine.internal.watcher.ResourceWatchManager; +import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; import io.aklivity.zilla.runtime.engine.namespace.NamespacedId; import io.aklivity.zilla.runtime.engine.resolver.Resolver; @@ -84,7 +84,7 @@ public class EngineManager private final List extensions; private final Function readURL; private final Resolver expressions; - private final ResourceWatchManager resourceWatchManager; + private final EngineConfigWatcher watcher; private EngineConfig current; @@ -102,7 +102,7 @@ public EngineManager( EngineConfiguration config, List extensions, Function readURL, - ResourceWatchManager resourceWatchManager) + EngineConfigWatcher watcher) { this.schemaTypes = schemaTypes; this.bindingByType = bindingByType; @@ -118,7 +118,7 @@ public EngineManager( this.extensions = extensions; this.readURL = readURL; this.expressions = Resolver.instantiate(config); - this.resourceWatchManager = resourceWatchManager; + this.watcher = watcher; } public EngineConfig reconfigure( @@ -419,7 +419,7 @@ private void unregister( { System.out.println("unregister: " + namespace.name); // TODO: Ati unregister(namespace); - resourceWatchManager.removeNamespace(namespace.name); + watcher.removeNamespace(namespace.name); } } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java index 036dce2778..e75832acbd 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java @@ -28,7 +28,7 @@ import io.aklivity.zilla.runtime.engine.config.NamespaceConfig; import io.aklivity.zilla.runtime.engine.exporter.ExporterContext; import io.aklivity.zilla.runtime.engine.guard.GuardContext; -import io.aklivity.zilla.runtime.engine.internal.watcher.ResourceWatchManager; +import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.Metric; import io.aklivity.zilla.runtime.engine.metrics.MetricContext; @@ -52,7 +52,7 @@ public class EngineRegistry private final LongConsumer detachBinding; private final Collector collector; private final Consumer process; - private final ResourceWatchManager resourceWatchManager; + private final EngineConfigWatcher watcher; public EngineRegistry( Function bindingsByType, @@ -68,7 +68,7 @@ public EngineRegistry( LongConsumer detachBinding, Collector collector, Consumer process, - ResourceWatchManager resourceWatchManager) + EngineConfigWatcher watcher) { this.bindingsByType = bindingsByType; this.guardsByType = guardsByType; @@ -84,7 +84,7 @@ public EngineRegistry( this.detachBinding = detachBinding; this.collector = collector; this.process = process; - this.resourceWatchManager = resourceWatchManager; + this.watcher = watcher; } public void process( @@ -200,7 +200,7 @@ private void attachNamespace( NamespaceRegistry registry = new NamespaceRegistry(namespace, bindingsByType, guardsByType, vaultsByType, catalogsByType, metricsByName, exportersByType, supplyLabelId, this::resolveMetric, exporterAttached, exporterDetached, - supplyMetricRecorder, detachBinding, collector, resourceWatchManager); + supplyMetricRecorder, detachBinding, collector, watcher); namespacesById.put(registry.namespaceId(), registry); registry.attach(); } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java index f1acac9221..e795e640f8 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java @@ -137,7 +137,7 @@ import io.aklivity.zilla.runtime.engine.internal.types.stream.ResetFW; import io.aklivity.zilla.runtime.engine.internal.types.stream.SignalFW; import io.aklivity.zilla.runtime.engine.internal.types.stream.WindowFW; -import io.aklivity.zilla.runtime.engine.internal.watcher.ResourceWatchManager; +import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.Metric; import io.aklivity.zilla.runtime.engine.metrics.MetricContext; @@ -254,7 +254,7 @@ public EngineWorker( Collector collector, Supplier supplyEventReader, EventFormatterFactory eventFormatterFactory, - ResourceWatchManager resourceWatchManager, + EngineConfigWatcher watcher, int index, boolean readonly, Consumer process) @@ -429,7 +429,7 @@ public EngineWorker( this.registry = new EngineRegistry( bindingsByType::get, guardsByType::get, vaultsByType::get, catalogsByType::get, metricsByName::get, exportersByType::get, labels::supplyLabelId, this::onExporterAttached, this::onExporterDetached, - this::supplyMetricWriter, this::detachStreams, collector, process, resourceWatchManager); + this::supplyMetricWriter, this::detachStreams, collector, process, watcher); this.taskQueue = new ConcurrentLinkedDeque<>(); this.correlations = new Long2ObjectHashMap<>(); diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java index 2d90e82f2d..7b2a5cf63a 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java @@ -44,7 +44,7 @@ import io.aklivity.zilla.runtime.engine.exporter.ExporterContext; import io.aklivity.zilla.runtime.engine.exporter.ExporterHandler; import io.aklivity.zilla.runtime.engine.guard.GuardContext; -import io.aklivity.zilla.runtime.engine.internal.watcher.ResourceWatchManager; +import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.Metric; import io.aklivity.zilla.runtime.engine.metrics.MetricContext; @@ -75,7 +75,7 @@ public class NamespaceRegistry private final ObjectLongLongFunction supplyMetricRecorder; private final LongConsumer detachBinding; private final Collector collector; - private final ResourceWatchManager resourceWatchManager; + private final EngineConfigWatcher watcher; public NamespaceRegistry( NamespaceConfig namespace, @@ -92,7 +92,7 @@ public NamespaceRegistry( ObjectLongLongFunction supplyMetricRecorder, LongConsumer detachBinding, Collector collector, - ResourceWatchManager resourceWatchManager) + EngineConfigWatcher watcher) { this.namespace = namespace; this.bindingsByType = bindingsByType; @@ -115,7 +115,7 @@ public NamespaceRegistry( this.metricsById = new Int2ObjectHashMap<>(); this.exportersById = new Int2ObjectHashMap<>(); this.collector = collector; - this.resourceWatchManager = resourceWatchManager; + this.watcher = watcher; } public int namespaceId() @@ -269,7 +269,7 @@ private void attachVault( VaultRegistry registry = new VaultRegistry(config, context); vaultsById.put(vaultId, registry); registry.attach(); - resourceWatchManager.addResources(registry.handler().resources(), config.namespace); + watcher.addResources(registry.handler().resources(), config.namespace); } private void detachVault( @@ -316,7 +316,7 @@ private void attachCatalog( CatalogRegistry registry = new CatalogRegistry(config, context); catalogsById.put(catalogId, registry); registry.attach(); - resourceWatchManager.addResources(registry.handler().resources(), config.namespace); + watcher.addResources(registry.handler().resources(), config.namespace); } private void detachCatalog( diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java similarity index 88% rename from runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java rename to runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java index 594788e890..b93ea4cb51 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatchManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java @@ -28,7 +28,9 @@ import java.util.function.Consumer; import java.util.function.Function; -public class ResourceWatchManager +import io.aklivity.zilla.runtime.engine.config.EngineConfig; + +public class EngineConfigWatcher { private final Map> resources; private final Map resourceTasks; @@ -37,8 +39,9 @@ public class ResourceWatchManager private FileSystem fileSystem; private Consumer> resourceChangeListener; private Function readURL; + private ConfigWatcherTask configWatcherTask; - public ResourceWatchManager() + public EngineConfigWatcher() { this.resources = new ConcurrentHashMap<>(); this.resourceTasks = new ConcurrentHashMap<>(); @@ -46,6 +49,7 @@ public ResourceWatchManager() public void initialize( URL configURL, + Function configChangeListener, Consumer> resourceChangeListener, Function readURL) { @@ -53,6 +57,13 @@ public void initialize( this.fileSystem = resolveFileSystem(configURL); this.resourceChangeListener = resourceChangeListener; this.readURL = readURL; + this.configWatcherTask = new ConfigWatcherTask(this.fileSystem, configChangeListener, readURL); + } + + public void startWatchingConfig() throws Exception + { + configWatcherTask.submit(); + configWatcherTask.watchConfig(configURL).get(); } public void addResources( @@ -148,6 +159,14 @@ public void close() rethrowUnchecked(ex); } }); + try + { + configWatcherTask.close(); + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } } // TODO: Ati - chk if this can be simplified From c0a554fbd5ab1390dc2e8e7d2410d69b1263645a Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 6 Jun 2024 16:28:31 +0200 Subject: [PATCH 07/43] WIP ResourceResolver --- .../aklivity/zilla/runtime/engine/Engine.java | 3 + .../engine/config/EngineConfigReader.java | 4 + .../engine/config/ResourceResolver.java | 100 ++++++++++++++++++ .../internal/registry/EngineManager.java | 5 + 4 files changed, 112 insertions(+) create mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index cd64a8ce22..f858dcd7bb 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -51,6 +51,7 @@ import io.aklivity.zilla.runtime.engine.catalog.Catalog; import io.aklivity.zilla.runtime.engine.config.KindConfig; import io.aklivity.zilla.runtime.engine.config.NamespaceConfig; +import io.aklivity.zilla.runtime.engine.config.ResourceResolver; import io.aklivity.zilla.runtime.engine.event.EventFormatterFactory; import io.aklivity.zilla.runtime.engine.exporter.Exporter; import io.aklivity.zilla.runtime.engine.ext.EngineExtContext; @@ -181,6 +182,7 @@ public final class Engine implements Collector, AutoCloseable final Map guardsByType = guards.stream() .collect(Collectors.toMap(g -> g.name(), g -> g)); + ResourceResolver resources = new ResourceResolver(catalogs, vaults); EngineManager manager = new EngineManager( schemaTypes, bindingsByType::get, @@ -195,6 +197,7 @@ public final class Engine implements Collector, AutoCloseable config, extensions, this::readURL, + resources, watcher); this.configURL = config.configURL(); diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java index 65f1b0852f..3bf95c1f9b 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java @@ -63,6 +63,7 @@ public final class EngineConfigReader private final ConfigAdapterContext context; private final Resolver expressions; private final Collection schemaTypes; + private final ResourceResolver resources; private final Consumer logger; private final MessageDigest md5; @@ -71,12 +72,14 @@ public EngineConfigReader( ConfigAdapterContext context, Resolver expressions, Collection schemaTypes, + ResourceResolver resources, Consumer logger) { this.config = config; this.context = context; this.expressions = expressions; this.schemaTypes = schemaTypes; + this.resources = resources; this.logger = logger; this.md5 = initMessageDigest("MD5"); } @@ -179,6 +182,7 @@ public EngineConfig read( reader.skip(configsAt.get(i)); NamespaceConfig namespace = jsonb.fromJson(reader, NamespaceConfig.class); namespace.hash = hashes.get(i); + System.out.println(resources.resolve(namespace)); // TODO: Ati builder.namespace(namespace); if (!errors.isEmpty()) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java new file mode 100644 index 0000000000..724c2cd5d7 --- /dev/null +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java @@ -0,0 +1,100 @@ +/* + * Copyright 2021-2023 Aklivity Inc. + * + * Aklivity licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.aklivity.zilla.runtime.engine.config; + +import static java.util.Objects.requireNonNull; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; + +import io.aklivity.zilla.runtime.engine.catalog.Catalog; +import io.aklivity.zilla.runtime.engine.catalog.CatalogContext; +import io.aklivity.zilla.runtime.engine.catalog.CatalogHandler; +import io.aklivity.zilla.runtime.engine.vault.Vault; +import io.aklivity.zilla.runtime.engine.vault.VaultContext; +import io.aklivity.zilla.runtime.engine.vault.VaultHandler; + +public class ResourceResolver +{ + private static final String FILESYSTEM = "filesystem"; + + private final Catalog fsCatalog; + private final Vault fsVault; + + public ResourceResolver( + Collection catalogs, + Collection vaults) + { + Catalog fsCatalog = null; + for (Catalog catalog : catalogs) + { + if (FILESYSTEM.equals(catalog.name())) + { + fsCatalog = catalog; + break; + } + } + this.fsCatalog = fsCatalog; + + Vault fsVault = null; + for (Vault vault : vaults) + { + if (FILESYSTEM.equals(vault.name())) + { + fsVault = vault; + break; + } + } + this.fsVault = fsVault; + } + + public List resolve( + NamespaceConfig namespace) + { + requireNonNull(namespace); + List result = new LinkedList<>(); + if (fsCatalog != null && namespace.catalogs != null) + { + for (CatalogConfig catalog : namespace.catalogs) + { + if (FILESYSTEM.equals(catalog.type)) + { + System.out.println("fs catalog " + catalog); + // TODO: Ati - this causes NPE + CatalogContext context = fsCatalog.supply(null); + CatalogHandler handler = context.attach(catalog); + result.addAll(handler.resources()); + } + } + } + if (fsVault != null && namespace.vaults != null) + { + for (VaultConfig vault : namespace.vaults) + { + if (FILESYSTEM.equals(vault.type)) + { + System.out.println("fs vault " + vault); + // TODO: Ati - this causes NPE + VaultContext context = fsVault.supply(null); + VaultHandler handler = context.attach(vault); + result.addAll(handler.resources()); + } + } + } + return result; + } +} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index da07478473..161c530881 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -54,6 +54,7 @@ import io.aklivity.zilla.runtime.engine.config.MetricRefConfig; import io.aklivity.zilla.runtime.engine.config.ModelConfig; import io.aklivity.zilla.runtime.engine.config.NamespaceConfig; +import io.aklivity.zilla.runtime.engine.config.ResourceResolver; import io.aklivity.zilla.runtime.engine.config.RouteConfig; import io.aklivity.zilla.runtime.engine.config.TelemetryRefConfig; import io.aklivity.zilla.runtime.engine.config.VaultConfig; @@ -84,6 +85,7 @@ public class EngineManager private final List extensions; private final Function readURL; private final Resolver expressions; + private final ResourceResolver resources; private final EngineConfigWatcher watcher; private EngineConfig current; @@ -102,6 +104,7 @@ public EngineManager( EngineConfiguration config, List extensions, Function readURL, + ResourceResolver resources, EngineConfigWatcher watcher) { this.schemaTypes = schemaTypes; @@ -118,6 +121,7 @@ public EngineManager( this.extensions = extensions; this.readURL = readURL; this.expressions = Resolver.instantiate(config); + this.resources = resources; this.watcher = watcher; } @@ -215,6 +219,7 @@ private EngineConfig parse( new NamespaceConfigAdapterContext(readURL), expressions, schemaTypes, + resources, logger); engine = reader.read(configText); From b7262671bf4123bc0d6c24fde1cd15790b3df04a Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 6 Jun 2024 18:51:34 +0200 Subject: [PATCH 08/43] WIP fix ResourceResolver --- .../internal/FilesystemCatalogHandler.java | 13 ---- .../config/FilesystemOptionsConfig.java | 13 ++++ .../http/config/HttpOptionsConfig.java | 68 ++++++++++--------- .../kafka/config/KafkaOptionsConfig.java | 19 ++++-- .../mqtt/config/MqttOptionsConfig.java | 23 ++++--- .../aklivity/zilla/runtime/engine/Engine.java | 20 ++---- .../engine/catalog/CatalogHandler.java | 7 -- .../engine/config/EngineConfigReader.java | 6 +- .../runtime/engine/config/OptionsConfig.java | 7 +- .../engine/config/ResourceResolver.java | 16 +---- .../internal/registry/EngineManager.java | 44 +++++++----- .../internal/registry/EngineRegistry.java | 8 +-- .../internal/registry/EngineWorker.java | 4 +- .../internal/registry/NamespaceRegistry.java | 8 +-- .../internal/watcher/EngineConfigWatcher.java | 29 ++++---- .../runtime/engine/vault/VaultHandler.java | 6 -- .../config/FileSystemOptionsConfig.java | 19 ++++++ .../internal/FileSystemVaultHandler.java | 17 ----- 18 files changed, 160 insertions(+), 167 deletions(-) diff --git a/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java b/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java index 3f5542bafc..4716cff249 100644 --- a/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java +++ b/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java @@ -17,7 +17,6 @@ import java.io.InputStream; import java.net.URL; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -36,7 +35,6 @@ public class FilesystemCatalogHandler implements CatalogHandler private final FilesystemEventContext event; private final long catalogId; private final Function resolvePath; - private final List resources; public FilesystemCatalogHandler( FilesystemOptionsConfig config, @@ -50,11 +48,6 @@ public FilesystemCatalogHandler( this.resolvePath = context::resolvePath; this.catalogId = catalogId; registerSchema(config.subjects); - this.resources = new LinkedList<>(); - for (FilesystemSchemaConfig subject : config.subjects) - { - resources.add(subject.path); - } } @Override @@ -95,12 +88,6 @@ private void registerSchema( } } - @Override - public List resources() - { - return resources; - } - private int generateCRC32C( String schema) { diff --git a/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/config/FilesystemOptionsConfig.java b/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/config/FilesystemOptionsConfig.java index f396581829..182995f34b 100644 --- a/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/config/FilesystemOptionsConfig.java +++ b/incubator/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/config/FilesystemOptionsConfig.java @@ -14,6 +14,7 @@ */ package io.aklivity.zilla.runtime.catalog.filesystem.internal.config; +import java.util.LinkedList; import java.util.List; import java.util.function.Function; @@ -37,6 +38,18 @@ public static FilesystemOptionsConfigBuilder builder( public FilesystemOptionsConfig( List subjects) { + super(List.of(), resolveResources(subjects)); this.subjects = subjects; } + + private static List resolveResources( + List subjects) + { + List resources = new LinkedList<>(); + for (FilesystemSchemaConfig subject : subjects) + { + resources.add(subject.path); + } + return resources; + } } diff --git a/runtime/binding-http/src/main/java/io/aklivity/zilla/runtime/binding/http/config/HttpOptionsConfig.java b/runtime/binding-http/src/main/java/io/aklivity/zilla/runtime/binding/http/config/HttpOptionsConfig.java index 40f0152347..f7e318deea 100644 --- a/runtime/binding-http/src/main/java/io/aklivity/zilla/runtime/binding/http/config/HttpOptionsConfig.java +++ b/runtime/binding-http/src/main/java/io/aklivity/zilla/runtime/binding/http/config/HttpOptionsConfig.java @@ -27,6 +27,7 @@ import io.aklivity.zilla.runtime.binding.http.internal.types.String16FW; import io.aklivity.zilla.runtime.binding.http.internal.types.String8FW; +import io.aklivity.zilla.runtime.engine.config.ModelConfig; import io.aklivity.zilla.runtime.engine.config.OptionsConfig; public final class HttpOptionsConfig extends OptionsConfig @@ -55,41 +56,46 @@ public static HttpOptionsConfigBuilder builder( HttpAuthorizationConfig authorization, List requests) { - super(requests != null && !requests.isEmpty() - ? requests.stream() - .flatMap(request -> Stream.concat( - Stream.of(request.content), - Stream.concat( - request.headers != null - ? request.headers.stream().flatMap(header -> Stream.of(header != null ? header.model : null)) - : Stream.empty(), - Stream.concat( - request.pathParams != null - ? request.pathParams.stream().flatMap(param -> Stream.of(param != null ? param.model : null)) - : Stream.empty(), - Stream.concat( - request.queryParams != null - ? request.queryParams.stream().flatMap(param -> Stream.of(param != null ? param.model : null)) - : Stream.empty(), - Stream.concat(request.responses != null - ? request.responses.stream().flatMap(param -> Stream.of(param != null - ? param.content - : null)) - : Stream.empty(), request.responses != null - ? request.responses.stream() - .flatMap(response -> response.headers != null - ? response.headers.stream() - .flatMap(param -> Stream.of(param != null ? param.model : null)) - : Stream.empty()) - : Stream.empty()) - )))).filter(Objects::nonNull)) - .collect(Collectors.toList()) - : emptyList()); - + super(resolveModels(requests), List.of()); this.versions = versions; this.overrides = overrides; this.access = access; this.authorization = authorization; this.requests = requests; } + + private static List resolveModels( + List requests) + { + return requests != null && !requests.isEmpty() + ? requests.stream() + .flatMap(request -> Stream.concat( + Stream.of(request.content), + Stream.concat( + request.headers != null + ? request.headers.stream().flatMap(header -> Stream.of(header != null ? header.model : null)) + : Stream.empty(), + Stream.concat( + request.pathParams != null + ? request.pathParams.stream().flatMap(param -> Stream.of(param != null ? param.model : null)) + : Stream.empty(), + Stream.concat( + request.queryParams != null + ? request.queryParams.stream().flatMap(param -> Stream.of(param != null ? param.model : null)) + : Stream.empty(), + Stream.concat(request.responses != null + ? request.responses.stream().flatMap(param -> Stream.of(param != null + ? param.content + : null)) + : Stream.empty(), request.responses != null + ? request.responses.stream() + .flatMap(response -> response.headers != null + ? response.headers.stream() + .flatMap(param -> Stream.of(param != null ? param.model : null)) + : Stream.empty()) + : Stream.empty()) + )))).filter(Objects::nonNull)) + .collect(Collectors.toList()) + : emptyList(); + } } diff --git a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/config/KafkaOptionsConfig.java b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/config/KafkaOptionsConfig.java index e597ce09ac..e060be22a8 100644 --- a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/config/KafkaOptionsConfig.java +++ b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/config/KafkaOptionsConfig.java @@ -23,6 +23,7 @@ import java.util.function.Function; import java.util.stream.Stream; +import io.aklivity.zilla.runtime.engine.config.ModelConfig; import io.aklivity.zilla.runtime.engine.config.OptionsConfig; public final class KafkaOptionsConfig extends OptionsConfig @@ -49,15 +50,21 @@ public static KafkaOptionsConfigBuilder builder( List servers, KafkaSaslConfig sasl) { - super(topics != null && !topics.isEmpty() - ? topics.stream() - .flatMap(t -> Stream.of(t.key, t.value)) - .filter(Objects::nonNull) - .collect(toList()) - : emptyList()); + super(resolveModels(topics), List.of()); this.bootstrap = bootstrap; this.topics = topics; this.servers = servers; this.sasl = sasl; } + + private static List resolveModels( + List topics) + { + return topics != null && !topics.isEmpty() + ? topics.stream() + .flatMap(t -> Stream.of(t.key, t.value)) + .filter(Objects::nonNull) + .collect(toList()) + : emptyList(); + } } diff --git a/runtime/binding-mqtt/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/config/MqttOptionsConfig.java b/runtime/binding-mqtt/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/config/MqttOptionsConfig.java index 1b1e9479f8..c73de19f61 100644 --- a/runtime/binding-mqtt/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/config/MqttOptionsConfig.java +++ b/runtime/binding-mqtt/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/config/MqttOptionsConfig.java @@ -26,6 +26,7 @@ import java.util.stream.Stream; import io.aklivity.zilla.runtime.binding.mqtt.internal.config.MqttVersion; +import io.aklivity.zilla.runtime.engine.config.ModelConfig; import io.aklivity.zilla.runtime.engine.config.OptionsConfig; public class MqttOptionsConfig extends OptionsConfig @@ -50,18 +51,24 @@ public MqttOptionsConfig( List topics, List versions) { - super(topics != null && !topics.isEmpty() + super(resolveModels(topics), List.of()); + this.authorization = authorization; + this.topics = topics; + this.versions = versions; + } + + private static List resolveModels( + List topics) + { + return topics != null && !topics.isEmpty() ? topics.stream() .flatMap(topic -> Stream.concat( - Stream.of(topic.content), + Stream.of(topic.content), Optional.ofNullable(topic.userProperties).orElseGet(Collections::emptyList).stream() - .flatMap(p -> Stream.of(p.value)) - .filter(Objects::nonNull)) + .flatMap(p -> Stream.of(p.value)) + .filter(Objects::nonNull)) .filter(Objects::nonNull)) .collect(Collectors.toList()) - : emptyList()); - this.authorization = authorization; - this.topics = topics; - this.versions = versions; + : emptyList(); } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index f858dcd7bb..779de620d8 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -64,7 +64,6 @@ import io.aklivity.zilla.runtime.engine.internal.registry.EngineManager; import io.aklivity.zilla.runtime.engine.internal.registry.EngineWorker; import io.aklivity.zilla.runtime.engine.internal.types.event.EventFW; -import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.MetricGroup; import io.aklivity.zilla.runtime.engine.model.Model; @@ -87,7 +86,6 @@ public final class Engine implements Collector, AutoCloseable private final boolean readonly; private final EngineConfiguration config; private final EngineManager manager; - private final EngineConfigWatcher watcher; Engine( EngineConfiguration config, @@ -148,14 +146,13 @@ public final class Engine implements Collector, AutoCloseable } this.tuning = tuning; - this.watcher = new EngineConfigWatcher(); List workers = new ArrayList<>(workerCount); for (int workerIndex = 0; workerIndex < workerCount; workerIndex++) { EngineWorker worker = new EngineWorker(config, tasks, labels, errorHandler, tuning::affinity, bindings, exporters, guards, vaults, catalogs, models, metricGroups, this, this::supplyEventReader, - eventFormatterFactory, watcher, workerIndex, readonly, this::process); + eventFormatterFactory, workerIndex, readonly, this::process); workers.add(worker); } this.workers = workers; @@ -183,6 +180,7 @@ public final class Engine implements Collector, AutoCloseable .collect(Collectors.toMap(g -> g.name(), g -> g)); ResourceResolver resources = new ResourceResolver(catalogs, vaults); + this.configURL = config.configURL(); EngineManager manager = new EngineManager( schemaTypes, bindingsByType::get, @@ -196,13 +194,9 @@ public final class Engine implements Collector, AutoCloseable context, config, extensions, + this.configURL, this::readURL, - resources, - watcher); - - this.configURL = config.configURL(); - this.watcher.initialize(configURL, manager::reconfigure, manager::reloadNamespacesWithChangedResources, - this::readURL); + resources); this.bindings = bindings; this.tasks = tasks; @@ -235,10 +229,10 @@ public void start() throws Exception worker.doStart(); } + // ignore the config file in read-only mode; no config will be read so no namespaces, bindings, etc. will be attached if (!readonly) { - // ignore the config file in read-only mode; no config will be read so no namespaces, bindings, etc will be attached - watcher.startWatchingConfig(); + manager.startWatcher(); } } @@ -252,7 +246,7 @@ public void close() throws Exception final List errors = new ArrayList<>(); - watcher.close(); + manager.closeWatcher(); for (EngineWorker worker : workers) { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/catalog/CatalogHandler.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/catalog/CatalogHandler.java index bfdb5a44e7..d6b730e261 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/catalog/CatalogHandler.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/catalog/CatalogHandler.java @@ -15,8 +15,6 @@ */ package io.aklivity.zilla.runtime.engine.catalog; -import java.util.List; - import org.agrona.DirectBuffer; import io.aklivity.zilla.runtime.engine.model.function.ValueConsumer; @@ -116,9 +114,4 @@ default int decodePadding( { return 0; } - - default List resources() - { - return List.of(); - } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java index 3bf95c1f9b..8ad277ceb0 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java @@ -55,6 +55,7 @@ import io.aklivity.zilla.runtime.engine.EngineConfiguration; import io.aklivity.zilla.runtime.engine.internal.config.NamespaceAdapter; import io.aklivity.zilla.runtime.engine.internal.config.schema.UniquePropertyKeysSchema; +import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; import io.aklivity.zilla.runtime.engine.resolver.Resolver; public final class EngineConfigReader @@ -64,6 +65,7 @@ public final class EngineConfigReader private final Resolver expressions; private final Collection schemaTypes; private final ResourceResolver resources; + private final EngineConfigWatcher watcher; private final Consumer logger; private final MessageDigest md5; @@ -73,6 +75,7 @@ public EngineConfigReader( Resolver expressions, Collection schemaTypes, ResourceResolver resources, + EngineConfigWatcher watcher, Consumer logger) { this.config = config; @@ -80,6 +83,7 @@ public EngineConfigReader( this.expressions = expressions; this.schemaTypes = schemaTypes; this.resources = resources; + this.watcher = watcher; this.logger = logger; this.md5 = initMessageDigest("MD5"); } @@ -182,7 +186,7 @@ public EngineConfig read( reader.skip(configsAt.get(i)); NamespaceConfig namespace = jsonb.fromJson(reader, NamespaceConfig.class); namespace.hash = hashes.get(i); - System.out.println(resources.resolve(namespace)); // TODO: Ati + watcher.addResources(resources.resolve(namespace), namespace.name); builder.namespace(namespace); if (!errors.isEmpty()) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/OptionsConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/OptionsConfig.java index 2e83dcc8f6..7eba0cd095 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/OptionsConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/OptionsConfig.java @@ -21,15 +21,18 @@ public class OptionsConfig { public final List models; + public final List resources; public OptionsConfig() { - this(Collections.emptyList()); + this(Collections.emptyList(), Collections.emptyList()); } public OptionsConfig( - List models) + List models, + List resources) { this.models = models; + this.resources = resources; } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java index 724c2cd5d7..2485dab431 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java @@ -22,11 +22,7 @@ import java.util.List; import io.aklivity.zilla.runtime.engine.catalog.Catalog; -import io.aklivity.zilla.runtime.engine.catalog.CatalogContext; -import io.aklivity.zilla.runtime.engine.catalog.CatalogHandler; import io.aklivity.zilla.runtime.engine.vault.Vault; -import io.aklivity.zilla.runtime.engine.vault.VaultContext; -import io.aklivity.zilla.runtime.engine.vault.VaultHandler; public class ResourceResolver { @@ -73,11 +69,7 @@ public List resolve( { if (FILESYSTEM.equals(catalog.type)) { - System.out.println("fs catalog " + catalog); - // TODO: Ati - this causes NPE - CatalogContext context = fsCatalog.supply(null); - CatalogHandler handler = context.attach(catalog); - result.addAll(handler.resources()); + result.addAll(catalog.options.resources); } } } @@ -87,11 +79,7 @@ public List resolve( { if (FILESYSTEM.equals(vault.type)) { - System.out.println("fs vault " + vault); - // TODO: Ati - this causes NPE - VaultContext context = fsVault.supply(null); - VaultHandler handler = context.attach(vault); - result.addAll(handler.resources()); + result.addAll(vault.options.resources); } } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index 161c530881..dc6e676337 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -103,9 +103,9 @@ public EngineManager( EngineExtContext context, EngineConfiguration config, List extensions, + URL configURL, Function readURL, - ResourceResolver resources, - EngineConfigWatcher watcher) + ResourceResolver resources) { this.schemaTypes = schemaTypes; this.bindingByType = bindingByType; @@ -122,10 +122,31 @@ public EngineManager( this.readURL = readURL; this.expressions = Resolver.instantiate(config); this.resources = resources; - this.watcher = watcher; + this.watcher = new EngineConfigWatcher(configURL, readURL, this::reconfigure, this::reloadNamespacesWithChangedResources); } - public EngineConfig reconfigure( + public void process( + NamespaceConfig namespace) + { + final List guards = current.namespaces.stream() + .map(n -> n.guards) + .flatMap(gs -> gs.stream()) + .collect(toList()); + + process(guards, namespace); + } + + public void startWatcher() throws Exception + { + watcher.startWatchingConfig(); + } + + public void closeWatcher() + { + watcher.close(); + } + + private EngineConfig reconfigure( String configText) { EngineConfig newConfig = null; @@ -177,7 +198,7 @@ public EngineConfig reconfigure( return newConfig; } - public void reloadNamespacesWithChangedResources( + private void reloadNamespacesWithChangedResources( Set namespaces) { if (namespaces != null && !namespaces.isEmpty()) @@ -189,17 +210,6 @@ public void reloadNamespacesWithChangedResources( } } - public void process( - NamespaceConfig namespace) - { - final List guards = current.namespaces.stream() - .map(n -> n.guards) - .flatMap(gs -> gs.stream()) - .collect(toList()); - - process(guards, namespace); - } - private EngineConfig parse( String configText) { @@ -220,6 +230,7 @@ private EngineConfig parse( expressions, schemaTypes, resources, + watcher, logger); engine = reader.read(configText); @@ -404,6 +415,7 @@ private void register( if (!identical.test(namespace.name)) { System.out.println("register: " + namespace.name); // TODO: Ati + watcher.addResources(resources.resolve(namespace), namespace.name); register(namespace); } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java index e75832acbd..452cc3b183 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineRegistry.java @@ -28,7 +28,6 @@ import io.aklivity.zilla.runtime.engine.config.NamespaceConfig; import io.aklivity.zilla.runtime.engine.exporter.ExporterContext; import io.aklivity.zilla.runtime.engine.guard.GuardContext; -import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.Metric; import io.aklivity.zilla.runtime.engine.metrics.MetricContext; @@ -52,7 +51,6 @@ public class EngineRegistry private final LongConsumer detachBinding; private final Collector collector; private final Consumer process; - private final EngineConfigWatcher watcher; public EngineRegistry( Function bindingsByType, @@ -67,8 +65,7 @@ public EngineRegistry( ObjectLongLongFunction supplyMetricRecorder, LongConsumer detachBinding, Collector collector, - Consumer process, - EngineConfigWatcher watcher) + Consumer process) { this.bindingsByType = bindingsByType; this.guardsByType = guardsByType; @@ -84,7 +81,6 @@ public EngineRegistry( this.detachBinding = detachBinding; this.collector = collector; this.process = process; - this.watcher = watcher; } public void process( @@ -200,7 +196,7 @@ private void attachNamespace( NamespaceRegistry registry = new NamespaceRegistry(namespace, bindingsByType, guardsByType, vaultsByType, catalogsByType, metricsByName, exportersByType, supplyLabelId, this::resolveMetric, exporterAttached, exporterDetached, - supplyMetricRecorder, detachBinding, collector, watcher); + supplyMetricRecorder, detachBinding, collector); namespacesById.put(registry.namespaceId(), registry); registry.attach(); } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java index e795e640f8..0d535e2d8a 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java @@ -137,7 +137,6 @@ import io.aklivity.zilla.runtime.engine.internal.types.stream.ResetFW; import io.aklivity.zilla.runtime.engine.internal.types.stream.SignalFW; import io.aklivity.zilla.runtime.engine.internal.types.stream.WindowFW; -import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.Metric; import io.aklivity.zilla.runtime.engine.metrics.MetricContext; @@ -254,7 +253,6 @@ public EngineWorker( Collector collector, Supplier supplyEventReader, EventFormatterFactory eventFormatterFactory, - EngineConfigWatcher watcher, int index, boolean readonly, Consumer process) @@ -429,7 +427,7 @@ public EngineWorker( this.registry = new EngineRegistry( bindingsByType::get, guardsByType::get, vaultsByType::get, catalogsByType::get, metricsByName::get, exportersByType::get, labels::supplyLabelId, this::onExporterAttached, this::onExporterDetached, - this::supplyMetricWriter, this::detachStreams, collector, process, watcher); + this::supplyMetricWriter, this::detachStreams, collector, process); this.taskQueue = new ConcurrentLinkedDeque<>(); this.correlations = new Long2ObjectHashMap<>(); diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java index 7b2a5cf63a..ef49c2c383 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/NamespaceRegistry.java @@ -44,7 +44,6 @@ import io.aklivity.zilla.runtime.engine.exporter.ExporterContext; import io.aklivity.zilla.runtime.engine.exporter.ExporterHandler; import io.aklivity.zilla.runtime.engine.guard.GuardContext; -import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; import io.aklivity.zilla.runtime.engine.metrics.Collector; import io.aklivity.zilla.runtime.engine.metrics.Metric; import io.aklivity.zilla.runtime.engine.metrics.MetricContext; @@ -75,7 +74,6 @@ public class NamespaceRegistry private final ObjectLongLongFunction supplyMetricRecorder; private final LongConsumer detachBinding; private final Collector collector; - private final EngineConfigWatcher watcher; public NamespaceRegistry( NamespaceConfig namespace, @@ -91,8 +89,7 @@ public NamespaceRegistry( LongConsumer exporterDetached, ObjectLongLongFunction supplyMetricRecorder, LongConsumer detachBinding, - Collector collector, - EngineConfigWatcher watcher) + Collector collector) { this.namespace = namespace; this.bindingsByType = bindingsByType; @@ -115,7 +112,6 @@ public NamespaceRegistry( this.metricsById = new Int2ObjectHashMap<>(); this.exportersById = new Int2ObjectHashMap<>(); this.collector = collector; - this.watcher = watcher; } public int namespaceId() @@ -269,7 +265,6 @@ private void attachVault( VaultRegistry registry = new VaultRegistry(config, context); vaultsById.put(vaultId, registry); registry.attach(); - watcher.addResources(registry.handler().resources(), config.namespace); } private void detachVault( @@ -316,7 +311,6 @@ private void attachCatalog( CatalogRegistry registry = new CatalogRegistry(config, context); catalogsById.put(catalogId, registry); registry.attach(); - watcher.addResources(registry.handler().resources(), config.namespace); } private void detachCatalog( diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java index b93ea4cb51..057f76849a 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java @@ -32,31 +32,26 @@ public class EngineConfigWatcher { + private final URL configURL; + private final FileSystem fileSystem; + private final Function readURL; private final Map> resources; private final Map resourceTasks; + private final Consumer> resourceChangeListener; + private final ConfigWatcherTask configWatcherTask; - private URL configURL; - private FileSystem fileSystem; - private Consumer> resourceChangeListener; - private Function readURL; - private ConfigWatcherTask configWatcherTask; - - public EngineConfigWatcher() - { - this.resources = new ConcurrentHashMap<>(); - this.resourceTasks = new ConcurrentHashMap<>(); - } - - public void initialize( + public EngineConfigWatcher( URL configURL, + Function readURL, Function configChangeListener, - Consumer> resourceChangeListener, - Function readURL) + Consumer> resourceChangeListener) { this.configURL = configURL; this.fileSystem = resolveFileSystem(configURL); - this.resourceChangeListener = resourceChangeListener; this.readURL = readURL; + this.resources = new ConcurrentHashMap<>(); + this.resourceTasks = new ConcurrentHashMap<>(); + this.resourceChangeListener = resourceChangeListener; this.configWatcherTask = new ConfigWatcherTask(this.fileSystem, configChangeListener, readURL); } @@ -170,6 +165,7 @@ public void close() } // TODO: Ati - chk if this can be simplified + // TODO: Ati - check this after adding hfs; this should just work fine for http private static FileSystem resolveFileSystem( URL url) { @@ -186,7 +182,6 @@ private static FileSystem resolveFileSystem( { location = uri.toString(); } - // TODO: Ati - check this after adding hfs; this should just work fine for http result = Path.of(location).getFileSystem(); } catch (Exception ex) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/vault/VaultHandler.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/vault/VaultHandler.java index c9086a27e3..6c67d6c856 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/vault/VaultHandler.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/vault/VaultHandler.java @@ -16,7 +16,6 @@ package io.aklivity.zilla.runtime.engine.vault; import java.security.KeyStore; -import java.util.List; public interface VaultHandler { @@ -28,9 +27,4 @@ KeyStore.TrustedCertificateEntry certificate( KeyStore.PrivateKeyEntry[] keys( String signerRef); - - default List resources() - { - return List.of(); - } } diff --git a/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/config/FileSystemOptionsConfig.java b/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/config/FileSystemOptionsConfig.java index e124c1ec23..9a6d4fe725 100644 --- a/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/config/FileSystemOptionsConfig.java +++ b/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/config/FileSystemOptionsConfig.java @@ -15,6 +15,8 @@ */ package io.aklivity.zilla.runtime.vault.filesystem.config; +import java.util.LinkedList; +import java.util.List; import java.util.function.Function; import io.aklivity.zilla.runtime.engine.config.OptionsConfig; @@ -41,8 +43,25 @@ public static FileSystemOptionsConfigBuilder builder( FileSystemStoreConfig trust, FileSystemStoreConfig signers) { + super(List.of(), resolveResources(keys, trust)); this.keys = keys; this.trust = trust; this.signers = signers; } + + private static List resolveResources( + FileSystemStoreConfig keys, + FileSystemStoreConfig trust) + { + List resources = new LinkedList<>(); + if (keys != null && keys.store != null) + { + resources.add(keys.store); + } + if (trust != null && trust.store != null) + { + resources.add(trust.store); + } + return resources; + } } diff --git a/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java b/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java index 97b2ab7093..c465063d99 100644 --- a/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java +++ b/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java @@ -26,7 +26,6 @@ import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.Optional; import java.util.function.Function; @@ -48,7 +47,6 @@ public class FileSystemVaultHandler implements VaultHandler private final Function lookupTrust; private final Function lookupSigner; private final Function, KeyStore.PrivateKeyEntry[]> lookupKeys; - private final List resources; public FileSystemVaultHandler( FileSystemOptionsConfig options, @@ -58,15 +56,6 @@ public FileSystemVaultHandler( lookupTrust = supplyLookupTrustedCertificateEntry(resolvePath, options.trust); lookupSigner = supplyLookupTrustedCertificateEntry(resolvePath, options.signers); lookupKeys = supplyLookupPrivateKeyEntries(resolvePath, options.keys); - resources = new LinkedList<>(); - if (options.keys != null && options.keys.store != null) - { - resources.add(options.keys.store); - } - if (options.trust != null && options.trust.store != null) - { - resources.add(options.trust.store); - } } @Override @@ -104,12 +93,6 @@ public PrivateKeyEntry[] keys( return keys; } - @Override - public List resources() - { - return resources; - } - private static Function supplyLookupPrivateKeyEntry( Function resolvePath, FileSystemStoreConfig aliases) From 9c2cbbee82058caf5bdc189a383cc2ad56734442 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Mon, 10 Jun 2024 10:05:31 +0200 Subject: [PATCH 09/43] fix indentation --- .../runtime/binding/kafka/config/KafkaOptionsConfig.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/config/KafkaOptionsConfig.java b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/config/KafkaOptionsConfig.java index e060be22a8..0d7da18c2d 100644 --- a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/config/KafkaOptionsConfig.java +++ b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/config/KafkaOptionsConfig.java @@ -62,9 +62,9 @@ private static List resolveModels( { return topics != null && !topics.isEmpty() ? topics.stream() - .flatMap(t -> Stream.of(t.key, t.value)) - .filter(Objects::nonNull) - .collect(toList()) + .flatMap(t -> Stream.of(t.key, t.value)) + .filter(Objects::nonNull) + .collect(toList()) : emptyList(); } } From abb923f2aa21fe952572f511b33254a4dc206a82 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Mon, 10 Jun 2024 11:01:45 +0200 Subject: [PATCH 10/43] fix EngineManager method names --- .../main/java/io/aklivity/zilla/runtime/engine/Engine.java | 4 ++-- .../zilla/runtime/engine/internal/registry/EngineManager.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index 779de620d8..10c92a72e5 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -232,7 +232,7 @@ public void start() throws Exception // ignore the config file in read-only mode; no config will be read so no namespaces, bindings, etc. will be attached if (!readonly) { - manager.startWatcher(); + manager.start(); } } @@ -246,7 +246,7 @@ public void close() throws Exception final List errors = new ArrayList<>(); - manager.closeWatcher(); + manager.close(); for (EngineWorker worker : workers) { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index dc6e676337..e985734628 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -136,12 +136,12 @@ public void process( process(guards, namespace); } - public void startWatcher() throws Exception + public void start() throws Exception { watcher.startWatchingConfig(); } - public void closeWatcher() + public void close() { watcher.close(); } From b9355c619a966602ae27976191e7b93cebc4030a Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Mon, 10 Jun 2024 17:02:59 +0200 Subject: [PATCH 11/43] rm ResourceResolver --- .../aklivity/zilla/runtime/engine/Engine.java | 5 +- .../engine/config/EngineConfigReader.java | 5 +- .../engine/config/NamespaceConfig.java | 25 ++++++ .../engine/config/ResourceResolver.java | 88 ------------------- .../internal/registry/EngineManager.java | 9 +- 5 files changed, 29 insertions(+), 103 deletions(-) delete mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index 10c92a72e5..061e27bf5d 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -51,7 +51,6 @@ import io.aklivity.zilla.runtime.engine.catalog.Catalog; import io.aklivity.zilla.runtime.engine.config.KindConfig; import io.aklivity.zilla.runtime.engine.config.NamespaceConfig; -import io.aklivity.zilla.runtime.engine.config.ResourceResolver; import io.aklivity.zilla.runtime.engine.event.EventFormatterFactory; import io.aklivity.zilla.runtime.engine.exporter.Exporter; import io.aklivity.zilla.runtime.engine.ext.EngineExtContext; @@ -179,7 +178,6 @@ public final class Engine implements Collector, AutoCloseable final Map guardsByType = guards.stream() .collect(Collectors.toMap(g -> g.name(), g -> g)); - ResourceResolver resources = new ResourceResolver(catalogs, vaults); this.configURL = config.configURL(); EngineManager manager = new EngineManager( schemaTypes, @@ -195,8 +193,7 @@ public final class Engine implements Collector, AutoCloseable config, extensions, this.configURL, - this::readURL, - resources); + this::readURL); this.bindings = bindings; this.tasks = tasks; diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java index 8ad277ceb0..71a7f8fe0c 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java @@ -64,7 +64,6 @@ public final class EngineConfigReader private final ConfigAdapterContext context; private final Resolver expressions; private final Collection schemaTypes; - private final ResourceResolver resources; private final EngineConfigWatcher watcher; private final Consumer logger; private final MessageDigest md5; @@ -74,7 +73,6 @@ public EngineConfigReader( ConfigAdapterContext context, Resolver expressions, Collection schemaTypes, - ResourceResolver resources, EngineConfigWatcher watcher, Consumer logger) { @@ -82,7 +80,6 @@ public EngineConfigReader( this.context = context; this.expressions = expressions; this.schemaTypes = schemaTypes; - this.resources = resources; this.watcher = watcher; this.logger = logger; this.md5 = initMessageDigest("MD5"); @@ -186,7 +183,7 @@ public EngineConfig read( reader.skip(configsAt.get(i)); NamespaceConfig namespace = jsonb.fromJson(reader, NamespaceConfig.class); namespace.hash = hashes.get(i); - watcher.addResources(resources.resolve(namespace), namespace.name); + watcher.addResources(namespace.resources, namespace.name); builder.namespace(namespace); if (!errors.isEmpty()) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java index f3489fed3f..b9ac42fb3d 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java @@ -18,11 +18,14 @@ import static java.util.Objects.requireNonNull; import static java.util.function.Function.identity; +import java.util.LinkedList; import java.util.List; import java.util.function.Function; public class NamespaceConfig { + public static final String FILESYSTEM = "filesystem"; + public transient int id; public transient Function readURL; public transient String hash; @@ -33,6 +36,7 @@ public class NamespaceConfig public final List guards; public final List vaults; public final List catalogs; + public final List resources; public static NamespaceConfigBuilder builder() { @@ -53,5 +57,26 @@ public static NamespaceConfigBuilder builder() this.guards = requireNonNull(guards); this.vaults = requireNonNull(vaults); this.catalogs = requireNonNull(catalogs); + this.resources = resolveResources(); + } + + private List resolveResources() + { + List resources = new LinkedList<>(); + for (CatalogConfig catalog : catalogs) + { + if (FILESYSTEM.equals(catalog.type)) + { + resources.addAll(catalog.options.resources); + } + } + for (VaultConfig vault : vaults) + { + if (FILESYSTEM.equals(vault.type)) + { + resources.addAll(vault.options.resources); + } + } + return resources; } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java deleted file mode 100644 index 2485dab431..0000000000 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ResourceResolver.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2021-2023 Aklivity Inc. - * - * Aklivity licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package io.aklivity.zilla.runtime.engine.config; - -import static java.util.Objects.requireNonNull; - -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; - -import io.aklivity.zilla.runtime.engine.catalog.Catalog; -import io.aklivity.zilla.runtime.engine.vault.Vault; - -public class ResourceResolver -{ - private static final String FILESYSTEM = "filesystem"; - - private final Catalog fsCatalog; - private final Vault fsVault; - - public ResourceResolver( - Collection catalogs, - Collection vaults) - { - Catalog fsCatalog = null; - for (Catalog catalog : catalogs) - { - if (FILESYSTEM.equals(catalog.name())) - { - fsCatalog = catalog; - break; - } - } - this.fsCatalog = fsCatalog; - - Vault fsVault = null; - for (Vault vault : vaults) - { - if (FILESYSTEM.equals(vault.name())) - { - fsVault = vault; - break; - } - } - this.fsVault = fsVault; - } - - public List resolve( - NamespaceConfig namespace) - { - requireNonNull(namespace); - List result = new LinkedList<>(); - if (fsCatalog != null && namespace.catalogs != null) - { - for (CatalogConfig catalog : namespace.catalogs) - { - if (FILESYSTEM.equals(catalog.type)) - { - result.addAll(catalog.options.resources); - } - } - } - if (fsVault != null && namespace.vaults != null) - { - for (VaultConfig vault : namespace.vaults) - { - if (FILESYSTEM.equals(vault.type)) - { - result.addAll(vault.options.resources); - } - } - } - return result; - } -} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index e985734628..8fc7321c2e 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -54,7 +54,6 @@ import io.aklivity.zilla.runtime.engine.config.MetricRefConfig; import io.aklivity.zilla.runtime.engine.config.ModelConfig; import io.aklivity.zilla.runtime.engine.config.NamespaceConfig; -import io.aklivity.zilla.runtime.engine.config.ResourceResolver; import io.aklivity.zilla.runtime.engine.config.RouteConfig; import io.aklivity.zilla.runtime.engine.config.TelemetryRefConfig; import io.aklivity.zilla.runtime.engine.config.VaultConfig; @@ -85,7 +84,6 @@ public class EngineManager private final List extensions; private final Function readURL; private final Resolver expressions; - private final ResourceResolver resources; private final EngineConfigWatcher watcher; private EngineConfig current; @@ -104,8 +102,7 @@ public EngineManager( EngineConfiguration config, List extensions, URL configURL, - Function readURL, - ResourceResolver resources) + Function readURL) { this.schemaTypes = schemaTypes; this.bindingByType = bindingByType; @@ -121,7 +118,6 @@ public EngineManager( this.extensions = extensions; this.readURL = readURL; this.expressions = Resolver.instantiate(config); - this.resources = resources; this.watcher = new EngineConfigWatcher(configURL, readURL, this::reconfigure, this::reloadNamespacesWithChangedResources); } @@ -229,7 +225,6 @@ private EngineConfig parse( new NamespaceConfigAdapterContext(readURL), expressions, schemaTypes, - resources, watcher, logger); @@ -415,7 +410,7 @@ private void register( if (!identical.test(namespace.name)) { System.out.println("register: " + namespace.name); // TODO: Ati - watcher.addResources(resources.resolve(namespace), namespace.name); + watcher.addResources(namespace.resources, namespace.name); register(namespace); } } From c58ebc9749dcc1a28780339dea316fc0b3f4749c Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Mon, 10 Jun 2024 17:14:40 +0200 Subject: [PATCH 12/43] WIP EngineManager addResources --- .../runtime/engine/config/EngineConfigReader.java | 11 +++++------ .../engine/internal/registry/EngineManager.java | 10 ++++++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java index 71a7f8fe0c..86b3c9c298 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java @@ -55,7 +55,6 @@ import io.aklivity.zilla.runtime.engine.EngineConfiguration; import io.aklivity.zilla.runtime.engine.internal.config.NamespaceAdapter; import io.aklivity.zilla.runtime.engine.internal.config.schema.UniquePropertyKeysSchema; -import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; import io.aklivity.zilla.runtime.engine.resolver.Resolver; public final class EngineConfigReader @@ -64,8 +63,8 @@ public final class EngineConfigReader private final ConfigAdapterContext context; private final Resolver expressions; private final Collection schemaTypes; - private final EngineConfigWatcher watcher; private final Consumer logger; + private final Consumer addResources; private final MessageDigest md5; public EngineConfigReader( @@ -73,15 +72,15 @@ public EngineConfigReader( ConfigAdapterContext context, Resolver expressions, Collection schemaTypes, - EngineConfigWatcher watcher, - Consumer logger) + Consumer logger, + Consumer addResources) { this.config = config; this.context = context; this.expressions = expressions; this.schemaTypes = schemaTypes; - this.watcher = watcher; this.logger = logger; + this.addResources = addResources; this.md5 = initMessageDigest("MD5"); } @@ -183,7 +182,7 @@ public EngineConfig read( reader.skip(configsAt.get(i)); NamespaceConfig namespace = jsonb.fromJson(reader, NamespaceConfig.class); namespace.hash = hashes.get(i); - watcher.addResources(namespace.resources, namespace.name); + addResources.accept(namespace); builder.namespace(namespace); if (!errors.isEmpty()) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index 8fc7321c2e..5d309d77db 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -225,8 +225,8 @@ private EngineConfig parse( new NamespaceConfigAdapterContext(readURL), expressions, schemaTypes, - watcher, - logger); + logger, + this::addResources); engine = reader.read(configText); @@ -249,6 +249,12 @@ private EngineConfig parse( return engine; } + private void addResources( + NamespaceConfig namespace) + { + watcher.addResources(namespace.resources, namespace.name); + } + private void process( List guards, NamespaceConfig namespace) From 2285123cb3aeca6cb721b5e17d0b2543d7264119 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Tue, 11 Jun 2024 09:27:57 +0200 Subject: [PATCH 13/43] Revert "WIP reconfigure only changed namespaces" This reverts commit a370a7bef9d0c9c9f0c7610b5077b100c6c94f5b. --- .../runtime/engine/config/EngineConfig.java | 13 +---- .../engine/config/EngineConfigBuilder.java | 16 +----- .../engine/config/EngineConfigReader.java | 48 +---------------- .../engine/config/NamespaceConfig.java | 1 - .../internal/registry/EngineManager.java | 51 ++++++------------- 5 files changed, 20 insertions(+), 109 deletions(-) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfig.java index 34ab6785d0..450002ff2e 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfig.java @@ -19,12 +19,9 @@ import static java.util.function.Function.identity; import java.util.List; -import java.util.Map; public class EngineConfig { - private final Map hashes; - public final List namespaces; public static EngineConfigBuilder builder() @@ -33,16 +30,8 @@ public static EngineConfigBuilder builder() } EngineConfig( - List namespaces, - Map hashes) + List namespaces) { this.namespaces = requireNonNull(namespaces); - this.hashes = requireNonNull(hashes); - } - - public String hash( - String name) - { - return hashes.get(name); } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigBuilder.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigBuilder.java index bd7b6f7fca..914ff3ca76 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigBuilder.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigBuilder.java @@ -15,16 +15,13 @@ */ package io.aklivity.zilla.runtime.engine.config; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.function.Function; public final class EngineConfigBuilder extends ConfigBuilder> { private final Function mapper; - private final Map hashes; private List namespaces; @@ -32,7 +29,6 @@ public final class EngineConfigBuilder extends ConfigBuilder mapper) { this.mapper = mapper; - this.hashes = new HashMap<>(); } @Override @@ -54,15 +50,7 @@ public EngineConfigBuilder namespace( { namespaces = new LinkedList<>(); } - if (!hashes.containsKey(namespace.name)) - { - namespaces.add(namespace); - hashes.put(namespace.name, namespace.hash); - } - else - { - System.out.printf("Warning: duplicate namespace ignored: %s%n", namespace.name); - } + namespaces.add(namespace); return this; } @@ -80,6 +68,6 @@ public T build() namespaces = new LinkedList<>(); } - return mapper.apply(new EngineConfig(namespaces, hashes)); + return mapper.apply(new EngineConfig(namespaces)); } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java index 86b3c9c298..6caa3a872a 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java @@ -25,10 +25,6 @@ import java.io.StringReader; import java.io.StringWriter; import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -44,7 +40,6 @@ import jakarta.json.spi.JsonProvider; import jakarta.json.stream.JsonParser; -import org.agrona.BitUtil; import org.agrona.collections.IntArrayList; import org.leadpony.justify.api.JsonSchema; import org.leadpony.justify.api.JsonSchemaReader; @@ -65,7 +60,6 @@ public final class EngineConfigReader private final Collection schemaTypes; private final Consumer logger; private final Consumer addResources; - private final MessageDigest md5; public EngineConfigReader( EngineConfiguration config, @@ -81,7 +75,6 @@ public EngineConfigReader( this.schemaTypes = schemaTypes; this.logger = logger; this.addResources = addResources; - this.md5 = initMessageDigest("MD5"); } public EngineConfig read( @@ -157,16 +150,6 @@ public EngineConfig read( } } - List hashes = new ArrayList<>(); - for (int i = 0; i < configsAt.size(); i++) - { - int start = configsAt.get(i); - int end = i < configsAt.size() - 1 ? configsAt.get(i + 1) : readable.length(); - byte[] bytes = readable.substring(start, end).stripTrailing().getBytes(StandardCharsets.UTF_8); - String hash = calculateHash(bytes, 0, bytes.length); - hashes.add(hash); - } - JsonbConfig config = new JsonbConfig() .withAdapters(new NamespaceAdapter(context)); Jsonb jsonb = JsonbBuilder.newBuilder() @@ -176,12 +159,11 @@ public EngineConfig read( Reader reader = new StringReader(readable); EngineConfigBuilder builder = EngineConfig.builder(); - for (int i = 0; i < configsAt.size(); i++) + for (int configAt : configsAt) { reader.reset(); - reader.skip(configsAt.get(i)); + reader.skip(configAt); NamespaceConfig namespace = jsonb.fromJson(reader, NamespaceConfig.class); - namespace.hash = hashes.get(i); addResources.accept(namespace); builder.namespace(namespace); @@ -284,30 +266,4 @@ private boolean validateAnnotatedSchema( return valid; } - - private String calculateHash( - byte[] input, - int offset, - int length) - { - md5.reset(); - md5.update(input, offset, length); - byte[] hash = md5.digest(); - return BitUtil.toHex(hash); - } - - private static MessageDigest initMessageDigest( - String algorithm) - { - MessageDigest messageDigest = null; - try - { - messageDigest = MessageDigest.getInstance(algorithm); - } - catch (NoSuchAlgorithmException ex) - { - rethrowUnchecked(ex); - } - return messageDigest; - } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java index b9ac42fb3d..10ba592dac 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java @@ -28,7 +28,6 @@ public class NamespaceConfig public transient int id; public transient Function readURL; - public transient String hash; public final String name; public final TelemetryConfig telemetry; diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index 5d309d77db..e8e64b39cf 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -30,7 +30,6 @@ import java.util.function.IntFunction; import java.util.function.LongFunction; import java.util.function.LongPredicate; -import java.util.function.Predicate; import java.util.function.ToIntFunction; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -118,7 +117,7 @@ public EngineManager( this.extensions = extensions; this.readURL = readURL; this.expressions = Resolver.instantiate(config); - this.watcher = new EngineConfigWatcher(configURL, readURL, this::reconfigure, this::reloadNamespacesWithChangedResources); + this.watcher = new EngineConfigWatcher(configURL, readURL, this::reconfigure, this::reconfigure); } public void process( @@ -153,26 +152,19 @@ private EngineConfig reconfigure( if (newConfig != null) { final EngineConfig oldConfig = current; - EngineConfig newConfig0 = newConfig; - Predicate identical = name -> - { - String hash1 = oldConfig == null ? null : oldConfig.hash(name); - String hash2 = newConfig0.hash(name); - return hash1 != null && hash1.equals(hash2); - }; - unregister(oldConfig, identical); + unregister(oldConfig); try { current = newConfig; - register(newConfig, identical); + register(newConfig); } catch (Exception ex) { context.onError(ex); current = oldConfig; - register(oldConfig, identical); + register(oldConfig); rethrowUnchecked(ex); } @@ -194,16 +186,11 @@ private EngineConfig reconfigure( return newConfig; } - private void reloadNamespacesWithChangedResources( + private void reconfigure( Set namespaces) { - if (namespaces != null && !namespaces.isEmpty()) - { - Set namespaces0 = new HashSet<>(namespaces); - Predicate identical = name -> !namespaces0.contains(name); - unregister(current, identical); - register(current, identical); - } + unregister(current); + register(current); } private EngineConfig parse( @@ -406,19 +393,15 @@ private void process( } private void register( - EngineConfig config, - Predicate identical) + EngineConfig config) { if (config != null) { for (NamespaceConfig namespace : config.namespaces) { - if (!identical.test(namespace.name)) - { - System.out.println("register: " + namespace.name); // TODO: Ati - watcher.addResources(namespace.resources, namespace.name); - register(namespace); - } + System.out.println("register: " + namespace.name); // TODO: Ati + watcher.addResources(namespace.resources, namespace.name); + register(namespace); } } @@ -426,19 +409,15 @@ private void register( } private void unregister( - EngineConfig config, - Predicate identical) + EngineConfig config) { if (config != null) { for (NamespaceConfig namespace : config.namespaces) { - if (!identical.test(namespace.name)) - { - System.out.println("unregister: " + namespace.name); // TODO: Ati - unregister(namespace); - watcher.removeNamespace(namespace.name); - } + System.out.println("unregister: " + namespace.name); // TODO: Ati + unregister(namespace); + watcher.removeNamespace(namespace.name); } } From 39f34d0702e76b8ea744628a78c3238c4735f0a1 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Tue, 11 Jun 2024 10:05:44 +0200 Subject: [PATCH 14/43] fix EngineConfigWatcher addResources --- .../engine/internal/registry/EngineManager.java | 4 ++-- .../internal/watcher/EngineConfigWatcher.java | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index e8e64b39cf..ce7a147f52 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -239,7 +239,7 @@ private EngineConfig parse( private void addResources( NamespaceConfig namespace) { - watcher.addResources(namespace.resources, namespace.name); + watcher.addResources(namespace); } private void process( @@ -400,7 +400,7 @@ private void register( for (NamespaceConfig namespace : config.namespaces) { System.out.println("register: " + namespace.name); // TODO: Ati - watcher.addResources(namespace.resources, namespace.name); + watcher.addResources(namespace); register(namespace); } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java index 057f76849a..c6b83d643f 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java @@ -21,7 +21,6 @@ import java.net.URL; import java.nio.file.FileSystem; import java.nio.file.Path; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -29,6 +28,7 @@ import java.util.function.Function; import io.aklivity.zilla.runtime.engine.config.EngineConfig; +import io.aklivity.zilla.runtime.engine.config.NamespaceConfig; public class EngineConfigWatcher { @@ -62,18 +62,17 @@ public void startWatchingConfig() throws Exception } public void addResources( - List additionalResources, - String namespace) + NamespaceConfig namespace) { - additionalResources.forEach(resource -> + namespace.resources.forEach(resource -> { resources.computeIfAbsent(resource, i -> { - startWatchingResource(resource, namespace); + startWatchingResource(resource, namespace.name); return ConcurrentHashMap.newKeySet(); } - ).add(namespace); - resourceTasks.get(resource).addNamespace(namespace); + ).add(namespace.name); + resourceTasks.get(resource).addNamespace(namespace.name); } ); } From 2da3603eb940efe4460fd3d0b9333fe917a4ede2 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Tue, 11 Jun 2024 10:40:27 +0200 Subject: [PATCH 15/43] WIP readPath resolvePath readLocation --- .../config/AsyncapiBindingConfig.java | 4 +- .../echo/internal/bench/EchoWorker.java | 6 +- .../config/GrpcOptionsConfigAdapter.java | 6 +- .../config/GrpcOptionsConfigAdapterTest.java | 2 +- .../config/OpenapiAsyncapiBindingConfig.java | 2 +- .../internal/config/OpenapiBindingConfig.java | 2 +- .../binding/tls/internal/bench/TlsWorker.java | 23 +--- .../internal/FilesystemCatalogHandler.java | 9 +- .../FilesystemCatalogFactoryTest.java | 6 +- .../filesystem/internal/FilesystemIT.java | 6 +- .../aklivity/zilla/runtime/engine/Engine.java | 39 ++++--- .../runtime/engine/EngineConfiguration.java | 29 +++++ .../zilla/runtime/engine/EngineContext.java | 6 +- .../runtime/engine/config/BindingConfig.java | 2 +- .../engine/config/ConfigAdapterContext.java | 2 +- .../runtime/engine/config/GuardConfig.java | 3 +- .../engine/config/NamespaceConfig.java | 2 +- .../internal/registry/EngineManager.java | 35 +++--- .../internal/registry/EngineWorker.java | 24 ++-- .../internal/watcher/ConfigWatcherTask.java | 89 ++++++++------ .../internal/watcher/EngineConfigWatcher.java | 55 +++------ .../internal/watcher/ResourceWatcherTask.java | 80 +++++++------ .../engine/internal/watcher/WatchedItem.java | 109 +++++++++--------- .../zilla/runtime/engine/test/EngineRule.java | 23 ++++ runtime/guard-jwt/pom.xml | 5 + .../guard/jwt/internal/JwtGuardContext.java | 2 +- .../guard/jwt/internal/JwtGuardHandler.java | 15 ++- .../jwt/internal/JwtGuardHandlerTest.java | 3 +- .../internal/FileSystemContext.java | 4 +- .../internal/FileSystemVaultHandler.java | 24 ++-- .../internal/FileSystemVaultTest.java | 24 +++- 31 files changed, 359 insertions(+), 282 deletions(-) diff --git a/runtime/binding-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/asyncapi/internal/config/AsyncapiBindingConfig.java b/runtime/binding-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/asyncapi/internal/config/AsyncapiBindingConfig.java index 5151ab36c1..d6f6fa5b56 100644 --- a/runtime/binding-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/asyncapi/internal/config/AsyncapiBindingConfig.java +++ b/runtime/binding-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/asyncapi/internal/config/AsyncapiBindingConfig.java @@ -227,7 +227,7 @@ private void attachProxyBinding( namespaceGenerator.init(binding); final List labels = configs.stream().map(c -> c.apiLabel).collect(toList()); final NamespaceConfig composite = namespaceGenerator.generateProxy(binding, asyncapis, schemaIdsByApiId::get, labels); - composite.readURL = binding.readURL; + composite.readLocation = binding.readLocation; attach.accept(composite); updateNamespace(configs, composite, new ArrayList<>(asyncapis.values())); } @@ -256,7 +256,7 @@ private void attachServerClientBinding( namespaceConfig.servers.forEach(s -> s.setAsyncapiProtocol( namespaceGenerator.resolveProtocol(s.protocol(), options, namespaceConfig.asyncapis, namespaceConfig.servers))); final NamespaceConfig composite = namespaceGenerator.generate(binding, namespaceConfig); - composite.readURL = binding.readURL; + composite.readLocation = binding.readLocation; attach.accept(composite); updateNamespace(namespaceConfig.configs, composite, namespaceConfig.asyncapis); } diff --git a/runtime/binding-echo/src/test/java/io/aklivity/zilla/runtime/binding/echo/internal/bench/EchoWorker.java b/runtime/binding-echo/src/test/java/io/aklivity/zilla/runtime/binding/echo/internal/bench/EchoWorker.java index a2e35d4ca8..ef76fa18d2 100644 --- a/runtime/binding-echo/src/test/java/io/aklivity/zilla/runtime/binding/echo/internal/bench/EchoWorker.java +++ b/runtime/binding-echo/src/test/java/io/aklivity/zilla/runtime/binding/echo/internal/bench/EchoWorker.java @@ -16,8 +16,8 @@ package io.aklivity.zilla.runtime.binding.echo.internal.bench; import java.net.InetAddress; -import java.net.URL; import java.nio.channels.SelectableChannel; +import java.nio.file.Path; import java.time.Clock; import java.util.function.LongSupplier; @@ -319,8 +319,8 @@ public ConverterHandler supplyWriteConverter( } @Override - public URL resolvePath( - String path) + public Path resolvePath( + String location) { return null; } diff --git a/runtime/binding-grpc/src/main/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapter.java b/runtime/binding-grpc/src/main/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapter.java index 41d2d9131b..93f38131b2 100644 --- a/runtime/binding-grpc/src/main/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapter.java +++ b/runtime/binding-grpc/src/main/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapter.java @@ -41,7 +41,7 @@ public final class GrpcOptionsConfigAdapter implements OptionsConfigAdapterSpi, private final GrpcProtobufParser parser = new GrpcProtobufParser(); - private Function readURL; + private Function readLocation; @Override public Kind kind() @@ -88,7 +88,7 @@ public OptionsConfig adaptFromJson( public void adaptContext( ConfigAdapterContext context) { - this.readURL = context::readURL; + this.readLocation = context::readLocation; } private List asListProtobufs( @@ -103,7 +103,7 @@ private GrpcProtobufConfig asProtobuf( JsonValue value) { final String location = ((JsonString) value).getString(); - final String protobuf = readURL.apply(location); + final String protobuf = readLocation.apply(location); return parser.parse(location, protobuf); } diff --git a/runtime/binding-grpc/src/test/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapterTest.java b/runtime/binding-grpc/src/test/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapterTest.java index aeb7efe820..1e29b6c5a8 100644 --- a/runtime/binding-grpc/src/test/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapterTest.java +++ b/runtime/binding-grpc/src/test/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapterTest.java @@ -66,7 +66,7 @@ public void initJson() throws IOException { content = new String(resource.readAllBytes(), UTF_8); } - Mockito.doReturn(content).when(context).readURL("protobuf/echo.proto"); + Mockito.doReturn(content).when(context).readLocation("protobuf/echo.proto"); adapter = new OptionsConfigAdapter(OptionsConfigAdapterSpi.Kind.BINDING, context); adapter.adaptType("grpc"); JsonbConfig config = new JsonbConfig() diff --git a/runtime/binding-openapi-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/asyncapi/internal/config/OpenapiAsyncapiBindingConfig.java b/runtime/binding-openapi-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/asyncapi/internal/config/OpenapiAsyncapiBindingConfig.java index d7ae8c04bd..e7e66289d9 100644 --- a/runtime/binding-openapi-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/asyncapi/internal/config/OpenapiAsyncapiBindingConfig.java +++ b/runtime/binding-openapi-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/asyncapi/internal/config/OpenapiAsyncapiBindingConfig.java @@ -144,7 +144,7 @@ public void attach( Object2ObjectHashMap::new)); this.composite = namespaceGenerator.generate(binding, openapis, asyncapis, openapiSchemaIdsByApiId::get); - this.composite.readURL = binding.readURL; + this.composite.readLocation = binding.readLocation; attach.accept(this.composite); BindingConfig mappingBinding = composite.bindings.stream() diff --git a/runtime/binding-openapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/internal/config/OpenapiBindingConfig.java b/runtime/binding-openapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/internal/config/OpenapiBindingConfig.java index 27f91d386f..a7cc361413 100644 --- a/runtime/binding-openapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/internal/config/OpenapiBindingConfig.java +++ b/runtime/binding-openapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/internal/config/OpenapiBindingConfig.java @@ -138,7 +138,7 @@ public void attach( for (OpenapiNamespaceConfig namespaceConfig : namespaceConfigs.values()) { final NamespaceConfig composite = namespaceGenerator.generate(binding, namespaceConfig); - composite.readURL = binding.readURL; + composite.readLocation = binding.readLocation; attach.accept(composite); namespaceConfig.configs.forEach(c -> { diff --git a/runtime/binding-tls/src/test/java/io/aklivity/zilla/runtime/binding/tls/internal/bench/TlsWorker.java b/runtime/binding-tls/src/test/java/io/aklivity/zilla/runtime/binding/tls/internal/bench/TlsWorker.java index 308cdd011d..9f90718b95 100644 --- a/runtime/binding-tls/src/test/java/io/aklivity/zilla/runtime/binding/tls/internal/bench/TlsWorker.java +++ b/runtime/binding-tls/src/test/java/io/aklivity/zilla/runtime/binding/tls/internal/bench/TlsWorker.java @@ -18,12 +18,10 @@ import static io.aklivity.zilla.runtime.engine.internal.stream.StreamId.isInitial; import static java.lang.ThreadLocal.withInitial; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.agrona.LangUtil.rethrowUnchecked; import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.URL; import java.nio.channels.SelectableChannel; +import java.nio.file.Path; import java.time.Clock; import java.util.function.IntConsumer; import java.util.function.LongSupplier; @@ -85,7 +83,7 @@ public class TlsWorker implements EngineContext private final BindingFactory factory; private final VaultFactory vaultFactory; private final Configuration config; - private final URL configURL; + private final Path configPath; private final TlsSignaler signaler; @@ -105,7 +103,7 @@ public TlsWorker( .readonly(false) .build() .bufferPool(); - this.configURL = config.configURL(); + this.configPath = config.configPath(); this.signaler = new TlsSignaler(); @@ -387,19 +385,10 @@ public ConverterHandler supplyWriteConverter( } @Override - public URL resolvePath( - String path) + public Path resolvePath( + String location) { - URL resolved = null; - try - { - resolved = new URL(configURL, path); - } - catch (MalformedURLException ex) - { - rethrowUnchecked(ex); - } - return resolved; + return configPath.resolveSibling(location); } @Override diff --git a/runtime/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java b/runtime/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java index 4716cff249..681e340594 100644 --- a/runtime/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java +++ b/runtime/catalog-filesystem/src/main/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogHandler.java @@ -15,7 +15,8 @@ package io.aklivity.zilla.runtime.catalog.filesystem.internal; import java.io.InputStream; -import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,7 +35,7 @@ public class FilesystemCatalogHandler implements CatalogHandler private final CRC32C crc32c; private final FilesystemEventContext event; private final long catalogId; - private final Function resolvePath; + private final Function resolvePath; public FilesystemCatalogHandler( FilesystemOptionsConfig config, @@ -72,8 +73,8 @@ private void registerSchema( { try { - URL storeURL = resolvePath.apply(config.path); - try (InputStream input = storeURL.openStream()) + Path storePath = resolvePath.apply(config.path); + try (InputStream input = Files.newInputStream(storePath)) { String schema = new String(input.readAllBytes()); int schemaId = generateCRC32C(schema); diff --git a/runtime/catalog-filesystem/src/test/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogFactoryTest.java b/runtime/catalog-filesystem/src/test/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogFactoryTest.java index 042367e125..cf3f010b66 100644 --- a/runtime/catalog-filesystem/src/test/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogFactoryTest.java +++ b/runtime/catalog-filesystem/src/test/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemCatalogFactoryTest.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.mock; import java.net.URL; +import java.nio.file.Path; import org.junit.Test; import org.mockito.Mockito; @@ -38,7 +39,7 @@ public class FilesystemCatalogFactoryTest { @Test - public void shouldLoadAndCreate() + public void shouldLoadAndCreate() throws Exception { Configuration config = new Configuration(); CatalogFactory factory = CatalogFactory.instantiate(); @@ -50,7 +51,8 @@ public void shouldLoadAndCreate() EngineContext engineContext = mock(EngineContext.class); URL url = FilesystemCatalogFactoryTest.class .getResource("../../../../specs/catalog/filesystem/config/asyncapi/mqtt.yaml"); - Mockito.doReturn(url).when(engineContext).resolvePath("asyncapi/mqtt.yaml"); + Path path = Path.of(url.toURI()); + Mockito.doReturn(path).when(engineContext).resolvePath("asyncapi/mqtt.yaml"); CatalogContext context = catalog.supply(engineContext); assertThat(context, instanceOf(FilesystemCatalogContext.class)); diff --git a/runtime/catalog-filesystem/src/test/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemIT.java b/runtime/catalog-filesystem/src/test/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemIT.java index 58b6675c2d..9dff743ef2 100644 --- a/runtime/catalog-filesystem/src/test/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemIT.java +++ b/runtime/catalog-filesystem/src/test/java/io/aklivity/zilla/runtime/catalog/filesystem/internal/FilesystemIT.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock; import java.net.URL; +import java.nio.file.Path; import org.agrona.DirectBuffer; import org.agrona.concurrent.UnsafeBuffer; @@ -41,13 +42,14 @@ public class FilesystemIT private EngineContext context = mock(EngineContext.class); @Before - public void setup() + public void setup() throws Exception { config = new FilesystemOptionsConfig(singletonList( new FilesystemSchemaConfig("subject1", "asyncapi/mqtt.yaml"))); URL url = FilesystemIT.class.getResource("../../../../specs/catalog/filesystem/config/asyncapi/mqtt.yaml"); - Mockito.doReturn(url).when(context).resolvePath("asyncapi/mqtt.yaml"); + Path path = Path.of(url.toURI()); + Mockito.doReturn(path).when(context).resolvePath("asyncapi/mqtt.yaml"); } @Test diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index 061e27bf5d..84d40aba16 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -16,14 +16,13 @@ package io.aklivity.zilla.runtime.engine; import static io.aklivity.zilla.runtime.engine.internal.layouts.metrics.HistogramsLayout.BUCKETS; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.Executors.newFixedThreadPool; import static java.util.stream.Collectors.toList; import static org.agrona.LangUtil.rethrowUnchecked; -import java.io.InputStream; import java.net.URL; -import java.net.URLConnection; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -80,7 +79,7 @@ public final class Engine implements Collector, AutoCloseable private final AtomicInteger nextTaskId; private final ThreadFactory factory; - private final URL configURL; + private final Path configPath; private final List workers; private final boolean readonly; private final EngineConfiguration config; @@ -149,7 +148,7 @@ public final class Engine implements Collector, AutoCloseable for (int workerIndex = 0; workerIndex < workerCount; workerIndex++) { EngineWorker worker = - new EngineWorker(config, tasks, labels, errorHandler, tuning::affinity, bindings, exporters, + new EngineWorker(config, tasks, labels, errorHandler, tuning::affinity, this::resolvePath, bindings, exporters, guards, vaults, catalogs, models, metricGroups, this, this::supplyEventReader, eventFormatterFactory, workerIndex, readonly, this::process); workers.add(worker); @@ -178,7 +177,7 @@ public final class Engine implements Collector, AutoCloseable final Map guardsByType = guards.stream() .collect(Collectors.toMap(g -> g.name(), g -> g)); - this.configURL = config.configURL(); + this.configPath = config.configPath(); EngineManager manager = new EngineManager( schemaTypes, bindingsByType::get, @@ -192,8 +191,9 @@ public final class Engine implements Collector, AutoCloseable context, config, extensions, - this.configURL, - this::readURL); + this.configPath, + this::readPath, + this::readLocation); this.bindings = bindings; this.tasks = tasks; @@ -285,18 +285,13 @@ public static EngineBuilder builder() return new EngineBuilder(); } - private String readURL( - String location) + private String readPath( + Path path) { String result; try { - URL url = new URL(configURL, location); - URLConnection connection = url.openConnection(); - try (InputStream input = connection.getInputStream()) - { - result = new String(input.readAllBytes(), UTF_8); - } + result = Files.readString(path); } catch (Exception ex) { @@ -305,6 +300,18 @@ private String readURL( return result; } + public Path resolvePath( + String location) + { + return configPath.resolveSibling(location); + } + + private String readLocation( + String location) + { + return readPath(resolvePath(location)); + } + private Thread newTaskThread( Runnable r) { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java index 5f9522446d..2786d63a16 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java @@ -18,6 +18,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.agrona.LangUtil.rethrowUnchecked; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; @@ -146,6 +147,34 @@ public URL configURL() return ENGINE_CONFIG_URL.get(this); } + public Path configPath() + { + Path configPath = null; + try + { + URI configUri = configURL().toURI(); + if ("file".equals(configUri.getScheme()) && !Path.of(configUri.getSchemeSpecificPart()).isAbsolute()) + { + // this works for relative file e.g. file:zilla.yaml + Path basePath = Path.of("").toAbsolutePath(); + configPath = basePath.resolve(configUri.getSchemeSpecificPart()); + } + else + { + // this works for absolute file e.g. file:/path/dir/zilla.yaml + // this works for http e.g. http://localhost:7115/zilla.yaml + // this works for jar e.g. jar:file:/path/engine.jar!/package/zilla.yaml + // (the jar filesystem is opened and closed by EngineRule) + configPath = Path.of(configUri); + } + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } + return configPath; + } + public int configPollIntervalSeconds() { return ENGINE_CONFIG_POLL_INTERVAL_SECONDS.getAsInt(this); diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineContext.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineContext.java index 3c6f81931e..c810d29998 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineContext.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineContext.java @@ -16,8 +16,8 @@ package io.aklivity.zilla.runtime.engine; import java.net.InetAddress; -import java.net.URL; import java.nio.channels.SelectableChannel; +import java.nio.file.Path; import java.time.Clock; import java.util.function.LongSupplier; @@ -157,8 +157,8 @@ ConverterHandler supplyReadConverter( ConverterHandler supplyWriteConverter( ModelConfig config); - URL resolvePath( - String path); + Path resolvePath( + String location); Metric resolveMetric( String name); diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/BindingConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/BindingConfig.java index 33faaec460..1c799d2ecd 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/BindingConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/BindingConfig.java @@ -27,7 +27,7 @@ public class BindingConfig public transient long id; public transient long entryId; public transient ToLongFunction resolveId; - public transient Function readURL; + public transient Function readLocation; public transient long vaultId; public transient String qvault; diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ConfigAdapterContext.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ConfigAdapterContext.java index fa4f5161e9..91f8349981 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ConfigAdapterContext.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ConfigAdapterContext.java @@ -17,5 +17,5 @@ public interface ConfigAdapterContext { - String readURL(String location); + String readLocation(String location); } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/GuardConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/GuardConfig.java index 5804edd815..e7d6d4669f 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/GuardConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/GuardConfig.java @@ -18,12 +18,13 @@ import static java.util.Objects.requireNonNull; import static java.util.function.Function.identity; +import java.nio.file.Path; import java.util.function.Function; public class GuardConfig { public transient long id; - public transient Function readURL; + public transient Function readPath; public final String namespace; public final String name; diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java index 10ba592dac..4e791b8104 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java @@ -27,7 +27,7 @@ public class NamespaceConfig public static final String FILESYSTEM = "filesystem"; public transient int id; - public transient Function readURL; + public transient Function readLocation; public final String name; public final TelemetryConfig telemetry; diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index ce7a147f52..60a4cda8b2 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -19,6 +19,7 @@ import static org.agrona.LangUtil.rethrowUnchecked; import java.net.URL; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -81,7 +82,8 @@ public class EngineManager private final EngineExtContext context; private final EngineConfiguration config; private final List extensions; - private final Function readURL; + private final Function readPath; + private final Function readLocation; private final Resolver expressions; private final EngineConfigWatcher watcher; @@ -100,8 +102,9 @@ public EngineManager( EngineExtContext context, EngineConfiguration config, List extensions, - URL configURL, - Function readURL) + Path configPath, + Function readPath, + Function readLocation) { this.schemaTypes = schemaTypes; this.bindingByType = bindingByType; @@ -115,9 +118,10 @@ public EngineManager( this.context = context; this.config = config; this.extensions = extensions; - this.readURL = readURL; + this.readPath = readPath; + this.readLocation = readLocation; this.expressions = Resolver.instantiate(config); - this.watcher = new EngineConfigWatcher(configURL, readURL, this::reconfigure, this::reconfigure); + this.watcher = new EngineConfigWatcher(configPath, readPath, this::reconfigure, this::reconfigure); } public void process( @@ -209,7 +213,7 @@ private EngineConfig parse( { EngineConfigReader reader = new EngineConfigReader( config, - new NamespaceConfigAdapterContext(readURL), + new NamespaceConfigAdapterContext(readLocation), expressions, schemaTypes, logger, @@ -224,7 +228,7 @@ private EngineConfig parse( for (NamespaceConfig namespace : engine.namespaces) { - namespace.readURL = readURL; + namespace.readLocation = readLocation; process(guards, namespace); } } @@ -246,7 +250,8 @@ private void process( List guards, NamespaceConfig namespace) { - assert namespace.readURL != null; + assert readPath != null; + assert namespace.readLocation != null; namespace.id = supplyId.applyAsInt(namespace.name); @@ -255,7 +260,7 @@ private void process( for (GuardConfig guard : namespace.guards) { guard.id = resolver.resolve(guard.name); - guard.readURL = namespace.readURL; + guard.readPath = readPath; } for (VaultConfig vault : namespace.vaults) @@ -287,7 +292,7 @@ private void process( binding.id = resolver.resolve(binding.name); binding.entryId = resolver.resolve(binding.entry); binding.resolveId = resolver::resolve; - binding.readURL = namespace.readURL; + binding.readLocation = namespace.readLocation; binding.typeId = supplyId.applyAsInt(binding.type); binding.kindId = supplyId.applyAsInt(binding.kind.name().toLowerCase()); @@ -486,19 +491,19 @@ private String format( private static final class NamespaceConfigAdapterContext implements ConfigAdapterContext { - private final Function readURL; + private final Function readLocation; NamespaceConfigAdapterContext( - Function readURL) + Function readLocation) { - this.readURL = readURL; + this.readLocation = readLocation; } @Override - public String readURL( + public String readLocation( String location) { - return readURL.apply(location); + return readLocation.apply(location); } } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java index bc4b731109..f29251f5e7 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java @@ -37,13 +37,11 @@ import static java.lang.ThreadLocal.withInitial; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.agrona.CloseHelper.quietClose; -import static org.agrona.LangUtil.rethrowUnchecked; import static org.agrona.concurrent.AgentRunner.startOnThread; import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.URL; import java.nio.channels.SelectableChannel; +import java.nio.file.Path; import java.time.Clock; import java.time.Duration; import java.util.BitSet; @@ -173,7 +171,6 @@ public class EngineWorker implements EngineContext, Agent private final int localIndex; private final EngineConfiguration config; - private final URL configURL; private final LabelManager labels; private final String agentName; private final Function resolveHost; @@ -217,6 +214,7 @@ public class EngineWorker implements EngineContext, Agent private final EngineRegistry registry; private final Deque taskQueue; private final LongUnaryOperator affinityMask; + private final Function resolvePath; private final AgentRunner runner; private final IdleStrategy idleStrategy; private final ErrorHandler errorHandler; @@ -244,6 +242,7 @@ public EngineWorker( LabelManager labels, ErrorHandler errorHandler, LongUnaryOperator affinityMask, + Function resolvePath, Collection bindings, Collection exporters, Collection guards, @@ -260,7 +259,7 @@ public EngineWorker( { this.localIndex = index; this.config = config; - this.configURL = config.configURL(); + this.resolvePath = resolvePath; this.labels = labels; this.affinityMask = affinityMask; @@ -741,19 +740,10 @@ public ConverterHandler supplyWriteConverter( } @Override - public URL resolvePath( - String path) + public Path resolvePath( + String location) { - URL resolved = null; - try - { - resolved = new URL(configURL, path); - } - catch (MalformedURLException ex) - { - rethrowUnchecked(ex); - } - return resolved; + return resolvePath.apply(location); } @Override diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java index 7e48037db9..0d66784a8e 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java @@ -18,9 +18,9 @@ import static org.agrona.LangUtil.rethrowUnchecked; import java.io.IOException; -import java.net.URL; import java.nio.file.ClosedWatchServiceException; import java.nio.file.FileSystem; +import java.nio.file.Path; import java.nio.file.WatchKey; import java.nio.file.WatchService; import java.util.IdentityHashMap; @@ -36,24 +36,27 @@ public class ConfigWatcherTask extends WatcherTask private final Map watchedItems; private final WatchService watchService; private final Function configChangeListener; - private final Function readURL; + private final Function readPath; public ConfigWatcherTask( FileSystem fileSystem, Function configChangeListener, - Function readURL) + Function readPath) { this.configChangeListener = configChangeListener; - this.readURL = readURL; + this.readPath = readPath; this.watchedItems = new IdentityHashMap<>(); WatchService watchService = null; - try + if (!"jar".equals(fileSystem.provider().getScheme())) // we can't watch in jar fs { - watchService = fileSystem.newWatchService(); - } - catch (Exception ex) - { - rethrowUnchecked(ex); + try + { + watchService = fileSystem.newWatchService(); + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } } this.watchService = watchService; } @@ -67,36 +70,39 @@ public Future submit() @Override public Void call() { - while (true) + if (watchService != null) { - try + while (true) { - final WatchKey key = watchService.take(); + try + { + final WatchKey key = watchService.take(); - WatchedItem watchedItem = watchedItems.get(key); + WatchedItem watchedItem = watchedItems.get(key); - if (watchedItem != null && watchedItem.isWatchedKey(key)) - { - // Even if no reconfigure needed, recalculation is necessary, since symlinks might have changed. - watchedItem.keys().forEach(watchedItems::remove); - watchedItem.unregister(); - watchedItem.register(); - watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); - String newText = readURL.apply(watchedItem.getURL().toString()); - byte[] newHash = computeHash(newText); - if (watchedItem.isReconfigureNeeded(newHash)) + if (watchedItem != null && watchedItem.isWatchedKey(key)) { - watchedItem.setHash(newHash); - if (configChangeListener != null) + // Even if no reconfigure needed, recalculation is necessary, since symlinks might have changed. + watchedItem.keys().forEach(watchedItems::remove); + watchedItem.unregister(); + watchedItem.register(); + watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); + String newText = readPath.apply(watchedItem.getPath()); + byte[] newHash = computeHash(newText); + if (watchedItem.isReconfigureNeeded(newHash)) { - configChangeListener.apply(newText); + watchedItem.setHash(newHash); + if (configChangeListener != null) + { + configChangeListener.apply(newText); + } } } } - } - catch (InterruptedException | ClosedWatchServiceException ex) - { - break; + catch (InterruptedException | ClosedWatchServiceException ex) + { + break; + } } } @@ -104,13 +110,17 @@ public Void call() } public CompletableFuture watchConfig( - URL configURL) + Path configPath) { - WatchedItem watchedItem = new WatchedItem(configURL, watchService); - watchedItem.register(); - watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); - String configText = readURL.apply(configURL.toString()); - watchedItem.setHash(computeHash(configText)); + if (!"jar".equals(configPath.getFileSystem().provider().getScheme())) + { + WatchedItem watchedItem = new WatchedItem(configPath, watchService); + watchedItem.register(); + watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); + String configText = readPath.apply(configPath); + watchedItem.setHash(computeHash(configText)); + } + String configText = readPath.apply(configPath); CompletableFuture configFuture; try @@ -129,6 +139,9 @@ public CompletableFuture watchConfig( @Override public void close() throws IOException { - watchService.close(); + if (watchService != null) + { + watchService.close(); + } } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java index c6b83d643f..968fa36708 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java @@ -17,8 +17,6 @@ import static org.agrona.LangUtil.rethrowUnchecked; -import java.net.URI; -import java.net.URL; import java.nio.file.FileSystem; import java.nio.file.Path; import java.util.Map; @@ -32,33 +30,33 @@ public class EngineConfigWatcher { - private final URL configURL; + private final Path configPath; private final FileSystem fileSystem; - private final Function readURL; + private final Function readPath; private final Map> resources; private final Map resourceTasks; private final Consumer> resourceChangeListener; private final ConfigWatcherTask configWatcherTask; public EngineConfigWatcher( - URL configURL, - Function readURL, + Path configPath, + Function readPath, Function configChangeListener, Consumer> resourceChangeListener) { - this.configURL = configURL; - this.fileSystem = resolveFileSystem(configURL); - this.readURL = readURL; + this.configPath = configPath; + this.fileSystem = configPath.getFileSystem(); + this.readPath = readPath; this.resources = new ConcurrentHashMap<>(); this.resourceTasks = new ConcurrentHashMap<>(); this.resourceChangeListener = resourceChangeListener; - this.configWatcherTask = new ConfigWatcherTask(this.fileSystem, configChangeListener, readURL); + this.configWatcherTask = new ConfigWatcherTask(this.fileSystem, configChangeListener, readPath); } public void startWatchingConfig() throws Exception { configWatcherTask.submit(); - configWatcherTask.watchConfig(configURL).get(); + configWatcherTask.watchConfig(configPath).get(); } public void addResources( @@ -105,13 +103,13 @@ private void startWatchingResource( { try { - ResourceWatcherTask watcherTask = new ResourceWatcherTask(fileSystem, resourceChangeListener, readURL); + ResourceWatcherTask watcherTask = new ResourceWatcherTask(fileSystem, resourceChangeListener, readPath); watcherTask.addNamespace(namespace); watcherTask.submit(); - URL resourceURL = new URL(configURL, resource); - watcherTask.watchResource(resourceURL); + Path resourcePath = configPath.resolveSibling(resource); + watcherTask.watchResource(resourcePath); resourceTasks.put(resource, watcherTask); - System.out.printf("started watching resource: %s resourceURL: %s\n", resource, resourceURL); // TODO: Ati + System.out.printf("started watching resource: %s resourcePath: %s\n", resource, resourcePath); // TODO: Ati } catch (Exception ex) { @@ -162,31 +160,4 @@ public void close() rethrowUnchecked(ex); } } - - // TODO: Ati - chk if this can be simplified - // TODO: Ati - check this after adding hfs; this should just work fine for http - private static FileSystem resolveFileSystem( - URL url) - { - FileSystem result = null; - try - { - URI uri = url.toURI(); - String location; - if ("file".equals(uri.getScheme())) - { - location = uri.getSchemeSpecificPart(); - } - else - { - location = uri.toString(); - } - result = Path.of(location).getFileSystem(); - } - catch (Exception ex) - { - rethrowUnchecked(ex); - } - return result; - } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcherTask.java index 4ea9238fc7..bbd4b1c8a3 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcherTask.java @@ -18,9 +18,9 @@ import static org.agrona.LangUtil.rethrowUnchecked; import java.io.IOException; -import java.net.URL; import java.nio.file.ClosedWatchServiceException; import java.nio.file.FileSystem; +import java.nio.file.Path; import java.nio.file.WatchKey; import java.nio.file.WatchService; import java.util.HashSet; @@ -36,26 +36,29 @@ public class ResourceWatcherTask extends WatcherTask private final Map watchedItems; private final WatchService watchService; private final Consumer> resourceChangeListener; - private final Function readURL; + private final Function readPath; private final Set namespaces; public ResourceWatcherTask( FileSystem fileSystem, Consumer> resourceChangeListener, - Function readURL) + Function readPath) { this.resourceChangeListener = resourceChangeListener; - this.readURL = readURL; + this.readPath = readPath; this.watchedItems = new IdentityHashMap<>(); this.namespaces = new HashSet<>(); WatchService watchService = null; - try + if (!"jar".equals(fileSystem.provider().getScheme())) // we can't watch in jar fs { - watchService = fileSystem.newWatchService(); - } - catch (Exception ex) - { - rethrowUnchecked(ex); + try + { + watchService = fileSystem.newWatchService(); + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } } this.watchService = watchService; @@ -70,49 +73,51 @@ public Future submit() @Override public Void call() { - while (true) + if (watchService != null) { - try + while (true) { - final WatchKey key = watchService.take(); + try + { + final WatchKey key = watchService.take(); - WatchedItem watchedItem = watchedItems.get(key); + WatchedItem watchedItem = watchedItems.get(key); - if (watchedItem != null && watchedItem.isWatchedKey(key)) - { - // Even if no reconfigure needed, recalculation is necessary, since symlinks might have changed. - watchedItem.keys().forEach(watchedItems::remove); - watchedItem.unregister(); - watchedItem.register(); - watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); - String newText = readURL.apply(watchedItem.getURL().toString()); - byte[] newHash = computeHash(newText); - if (watchedItem.isReconfigureNeeded(newHash)) + if (watchedItem != null && watchedItem.isWatchedKey(key)) { - watchedItem.setHash(newHash); - if (resourceChangeListener != null) + // Even if no reconfigure needed, recalculation is necessary, since symlinks might have changed. + watchedItem.keys().forEach(watchedItems::remove); + watchedItem.unregister(); + watchedItem.register(); + watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); + String newText = readPath.apply(watchedItem.getPath()); + byte[] newHash = computeHash(newText); + if (watchedItem.isReconfigureNeeded(newHash)) { - resourceChangeListener.accept(namespaces); + watchedItem.setHash(newHash); + if (resourceChangeListener != null) + { + resourceChangeListener.accept(namespaces); + } } } } - } - catch (InterruptedException | ClosedWatchServiceException ex) - { - break; + catch (InterruptedException | ClosedWatchServiceException ex) + { + break; + } } } - return null; } public void watchResource( - URL resourceURL) + Path resourcePath) { - WatchedItem watchedItem = new WatchedItem(resourceURL, watchService); + WatchedItem watchedItem = new WatchedItem(resourcePath, watchService); watchedItem.register(); watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); - String resource = readURL.apply(resourceURL.toString()); + String resource = readPath.apply(resourcePath); watchedItem.setHash(computeHash(resource)); } @@ -131,6 +136,9 @@ public void removeNamespace( @Override public void close() throws IOException { - watchService.close(); + if (watchService != null) + { + watchService.close(); + } } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatchedItem.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatchedItem.java index 5bbacf7ef3..b51077b13c 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatchedItem.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatchedItem.java @@ -21,10 +21,8 @@ import static org.agrona.LangUtil.rethrowUnchecked; import java.io.IOException; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.WatchKey; import java.nio.file.WatchService; import java.util.Arrays; @@ -37,16 +35,16 @@ public class WatchedItem { private final WatchService watchService; private final Set watchKeys; - private final URL watchedURL; + private final Path watchedPath; private byte[] hash; public WatchedItem( - URL watchedURL, + Path watchedPath, WatchService watchService) { this.watchService = watchService; this.watchKeys = new HashSet<>(); - this.watchedURL = watchedURL; + this.watchedPath = watchedPath; } public Set keys() @@ -56,56 +54,74 @@ public Set keys() public void register() { - Path configPath = Paths.get(watchedURL.getPath()).toAbsolutePath(); try { - Set watchedPaths = new HashSet<>(); + String scheme = watchedPath.getFileSystem().provider().getScheme(); + if ("file".equals(scheme)) + { + registerFilePath(); + } + else if ("http".equals(scheme)) + { + watchKeys.add(watchedPath.register(watchService)); + } + } + catch (IOException ex) + { + rethrowUnchecked(ex); + } + } - Deque observablePaths = new LinkedList<>(); - observablePaths.addLast(configPath); + private void registerFilePath() throws IOException + { + Set watchedPaths = new HashSet<>(); + Deque observablePaths = new LinkedList<>(); + observablePaths.addLast(watchedPath); - while (!observablePaths.isEmpty()) - { - Path observablePath = observablePaths.removeFirst(); + while (!observablePaths.isEmpty()) + { + Path observablePath = observablePaths.removeFirst(); - if (watchedPaths.add(observablePath)) + if (watchedPaths.add(observablePath)) + { + if (Files.isSymbolicLink(observablePath)) { - if (Files.isSymbolicLink(observablePath)) - { - Path targetPath = Files.readSymbolicLink(observablePath); - targetPath = configPath.resolveSibling(targetPath).normalize(); - observablePaths.addLast(targetPath); - } + Path targetPath = Files.readSymbolicLink(observablePath); + targetPath = watchedPath.resolveSibling(targetPath).normalize(); + observablePaths.addLast(targetPath); + } - for (Path ancestorPath = observablePath.getParent(); - ancestorPath != null; - ancestorPath = ancestorPath.getParent()) + for (Path ancestorPath = observablePath.getParent(); + ancestorPath != null; + ancestorPath = ancestorPath.getParent()) + { + if (Files.isSymbolicLink(ancestorPath)) { - if (Files.isSymbolicLink(ancestorPath)) + if (watchedPaths.add(ancestorPath)) { - if (watchedPaths.add(ancestorPath)) - { - Path targetPath = Files.readSymbolicLink(ancestorPath); - observablePaths.addLast(ancestorPath.resolve(targetPath).normalize()); - } + Path targetPath = Files.readSymbolicLink(ancestorPath); + observablePaths.addLast(ancestorPath.resolve(targetPath).normalize()); } } } } - - for (Path watchedPath : watchedPaths) + } + for (Path watchedPath : watchedPaths) + { + if (Files.exists(watchedPath.getParent())) { - if (Files.exists(watchedPath.getParent())) + WatchKey key = null; + try { - WatchKey key = registerPath(watchedPath.getParent()); - watchKeys.add(key); + key = watchedPath.getParent().register(watchService, ENTRY_MODIFY, ENTRY_CREATE, ENTRY_DELETE); } + catch (IOException ex) + { + rethrowUnchecked(ex); + } + watchKeys.add(key); } } - catch (IOException ex) - { - rethrowUnchecked(ex); - } } public void unregister() @@ -132,23 +148,8 @@ public void setHash( hash = newHash; } - public URL getURL() - { - return watchedURL; - } - - private WatchKey registerPath( - Path path) + public Path getPath() { - WatchKey key = null; - try - { - key = path.register(watchService, ENTRY_MODIFY, ENTRY_CREATE, ENTRY_DELETE); - } - catch (IOException ex) - { - rethrowUnchecked(ex); - } - return key; + return watchedPath; } } diff --git a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/EngineRule.java b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/EngineRule.java index 44278f08cf..05b1b4947b 100644 --- a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/EngineRule.java +++ b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/EngineRule.java @@ -31,11 +31,14 @@ import java.io.IOException; import java.net.URI; import java.net.URL; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.function.LongConsumer; import java.util.function.LongSupplier; @@ -265,6 +268,7 @@ public Statement apply( { Class testClass = description.getTestClass(); final String testMethod = description.getMethodName().replaceAll("\\[.*\\]", ""); + URI jarUri = null; try { Configure[] configures = testClass @@ -286,6 +290,13 @@ else if (configurationRoot != null) String resourceName = String.format("%s/%s", configurationRoot, config.value()); URL configURL = testClass.getClassLoader().getResource(resourceName); configure(ENGINE_CONFIG_URL, configURL); + if ("jar".equals(configURL.getProtocol())) + { + String jarLocation = Path.of( + configURL.toString().replace("jar:file:", "").split("!")[0] + ).toUri().toString(); + jarUri = new URI("jar", jarLocation, null); + } } else { @@ -303,6 +314,7 @@ else if (configurationRoot != null) } boolean allowErrors = exceptions.test(testMethod); + URI jarUri0 = jarUri; return new Statement() { @@ -318,6 +330,13 @@ public void evaluate() throws Throwable errors.add(ex); baseThread.interrupt(); }; + + FileSystem fs = null; + if (jarUri0 != null) + { + fs = FileSystems.newFileSystem(jarUri0, Map.of()); + } + engine = builder.config(config) .errorHandler(errorHandler) .build(); @@ -348,6 +367,10 @@ public void evaluate() throws Throwable { assertEmpty(errors); } + if (fs != null) + { + fs.close(); + } } } } diff --git a/runtime/guard-jwt/pom.xml b/runtime/guard-jwt/pom.xml index 538015ff34..ebba56e5b5 100644 --- a/runtime/guard-jwt/pom.xml +++ b/runtime/guard-jwt/pom.xml @@ -68,6 +68,11 @@ ${project.version} test + + ${project.groupId} + filesystem-http + test + junit junit diff --git a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardContext.java b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardContext.java index 55c141be78..551aa66dce 100644 --- a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardContext.java +++ b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardContext.java @@ -44,7 +44,7 @@ public JwtGuardHandler attach( GuardConfig guard) { JwtOptionsConfig options = (JwtOptionsConfig) guard.options; - JwtGuardHandler handler = new JwtGuardHandler(options, context, supplyAuthorizedId, guard.readURL); + JwtGuardHandler handler = new JwtGuardHandler(options, context, supplyAuthorizedId, guard.readPath); handlersById.put(guard.id, handler); return handler; } diff --git a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java index 50ec6342bb..69f5940185 100644 --- a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java +++ b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java @@ -16,6 +16,9 @@ import static org.agrona.LangUtil.rethrowUnchecked; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.Arrays; @@ -66,7 +69,7 @@ public JwtGuardHandler( JwtOptionsConfig options, EngineContext context, LongSupplier supplyAuthorizedId, - Function readURL) + Function readPath) { this.issuer = options.issuer; this.audience = options.audience; @@ -81,7 +84,15 @@ public JwtGuardHandler( .withConfig(config) .build(); - String keysText = readURL.apply(options.keysURL.get()); + String keysText = null; + try + { + keysText = readPath.apply(Path.of(new URI(options.keysURL.get()))); + } + catch (URISyntaxException ex) + { + rethrowUnchecked(ex); + } JwtKeySetConfig jwks = jsonb.fromJson(keysText, JwtKeySetConfig.class); keysConfig = jwks.keys; } diff --git a/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandlerTest.java b/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandlerTest.java index 90bd4add79..0780ac914d 100644 --- a/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandlerTest.java +++ b/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandlerTest.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.nio.file.Path; import java.security.KeyPair; import java.time.Clock; import java.time.Duration; @@ -46,7 +47,7 @@ public class JwtGuardHandlerTest { - private static final Function READ_KEYS_URL = url -> "{}"; + private static final Function READ_KEYS_URL = url -> "{}"; private EngineContext context; diff --git a/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemContext.java b/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemContext.java index 0b303d4bc6..7b7904d5b9 100644 --- a/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemContext.java +++ b/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemContext.java @@ -15,7 +15,7 @@ */ package io.aklivity.zilla.runtime.vault.filesystem.internal; -import java.net.URL; +import java.nio.file.Path; import java.util.function.Function; import io.aklivity.zilla.runtime.engine.Configuration; @@ -26,7 +26,7 @@ final class FileSystemContext implements VaultContext { - private final Function resolvePath; + private final Function resolvePath; FileSystemContext( Configuration config, diff --git a/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java b/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java index c465063d99..80fc4d3fbc 100644 --- a/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java +++ b/runtime/vault-filesystem/src/main/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultHandler.java @@ -16,8 +16,8 @@ package io.aklivity.zilla.runtime.vault.filesystem.internal; import java.io.InputStream; -import java.net.URL; -import java.net.URLConnection; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.KeyStore; import java.security.KeyStore.Entry; import java.security.KeyStore.PrivateKeyEntry; @@ -50,7 +50,7 @@ public class FileSystemVaultHandler implements VaultHandler public FileSystemVaultHandler( FileSystemOptionsConfig options, - Function resolvePath) + Function resolvePath) { lookupKey = supplyLookupPrivateKeyEntry(resolvePath, options.keys); lookupTrust = supplyLookupTrustedCertificateEntry(resolvePath, options.trust); @@ -94,21 +94,21 @@ public PrivateKeyEntry[] keys( } private static Function supplyLookupPrivateKeyEntry( - Function resolvePath, + Function resolvePath, FileSystemStoreConfig aliases) { return supplyLookupAlias(resolvePath, aliases, FileSystemVaultHandler::lookupPrivateKeyEntry); } private static Function supplyLookupTrustedCertificateEntry( - Function resolvePath, + Function resolvePath, FileSystemStoreConfig aliases) { return supplyLookupAlias(resolvePath, aliases, FileSystemVaultHandler::lookupTrustedCertificateEntry); } private Function, KeyStore.PrivateKeyEntry[]> supplyLookupPrivateKeyEntries( - Function resolvePath, + Function resolvePath, FileSystemStoreConfig entries) { Function, KeyStore.PrivateKeyEntry[]> lookupKeys = p -> null; @@ -117,9 +117,8 @@ private Function, KeyStore.PrivateKeyEntry[]> supplyLoo { try { - URL storeURL = resolvePath.apply(entries.store); - URLConnection connection = storeURL.openConnection(); - try (InputStream input = connection.getInputStream()) + Path storePath = resolvePath.apply(entries.store); + try (InputStream input = Files.newInputStream(storePath)) { String type = Optional.ofNullable(entries.type).orElse(TYPE_DEFAULT); char[] password = Optional.ofNullable(entries.password).map(String::toCharArray).orElse(null); @@ -165,7 +164,7 @@ private Function, KeyStore.PrivateKeyEntry[]> supplyLoo } private static Function supplyLookupAlias( - Function resolvePath, + Function resolvePath, FileSystemStoreConfig aliases, Lookup lookup) { @@ -175,9 +174,8 @@ private static Function supplyLookupAlias( { try { - URL storeURL = resolvePath.apply(aliases.store); - URLConnection connection = storeURL.openConnection(); - try (InputStream input = connection.getInputStream()) + Path storePath = resolvePath.apply(aliases.store); + try (InputStream input = Files.newInputStream(storePath)) { String type = Optional.ofNullable(aliases.type).orElse(TYPE_DEFAULT); char[] password = Optional.ofNullable(aliases.password).map(String::toCharArray).orElse(null); diff --git a/runtime/vault-filesystem/src/test/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultTest.java b/runtime/vault-filesystem/src/test/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultTest.java index fecdd376f6..945e2b535a 100644 --- a/runtime/vault-filesystem/src/test/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultTest.java +++ b/runtime/vault-filesystem/src/test/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultTest.java @@ -15,11 +15,14 @@ */ package io.aklivity.zilla.runtime.vault.filesystem.internal; +import static org.agrona.LangUtil.rethrowUnchecked; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import java.net.URL; +import java.nio.file.Path; import java.security.KeyStore.PrivateKeyEntry; import java.security.KeyStore.TrustedCertificateEntry; @@ -45,7 +48,7 @@ public void shouldResolveServer() throws Exception .build() .build(); - FileSystemVaultHandler vault = new FileSystemVaultHandler(options, FileSystemVaultTest.class::getResource); + FileSystemVaultHandler vault = new FileSystemVaultHandler(options, FileSystemVaultTest::getResourcePath); PrivateKeyEntry key = vault.key("localhost"); TrustedCertificateEntry certificate = vault.certificate("clientca"); @@ -70,7 +73,7 @@ public void shouldResolveClient() throws Exception .build() .build(); - FileSystemVaultHandler vault = new FileSystemVaultHandler(options, FileSystemVaultTest.class::getResource); + FileSystemVaultHandler vault = new FileSystemVaultHandler(options, FileSystemVaultTest::getResourcePath); PrivateKeyEntry key = vault.key("client1"); PrivateKeyEntry[] signedKeys = vault.keys("clientca"); @@ -80,4 +83,21 @@ public void shouldResolveClient() throws Exception assertThat(signedKeys.length, equalTo(1)); assertThat(signedKeys[0], not(nullValue())); } + + public static Path getResourcePath( + String resource) + { + URL url = FileSystemVaultTest.class.getResource(resource); + assert url != null; + Path path = null; + try + { + path = Path.of(url.toURI()); + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } + return path; + } } From c85bdc1580bafcc5fa025a1c592e86d27fcccd2d Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Wed, 5 Jun 2024 17:35:11 +0200 Subject: [PATCH 16/43] WIP filesystem-http --- cloud/docker-image/pom.xml | 6 + .../docker-image/src/main/docker/assembly.xml | 1 + .../src/main/docker/zpm.json.template | 1 + runtime/filesystem-http/COPYRIGHT | 12 + runtime/filesystem-http/LICENSE | 114 +++++ runtime/filesystem-http/NOTICE | 14 + runtime/filesystem-http/NOTICE.template | 13 + runtime/filesystem-http/mvnw | 310 +++++++++++++ runtime/filesystem-http/mvnw.cmd | 182 ++++++++ runtime/filesystem-http/pom.xml | 205 ++++++++ .../http/HttpBaseFileSystemProvider.java | 361 +++++++++++++++ .../filesystem/http/HttpFileSystem.java | 152 ++++++ .../http/HttpFileSystemProvider.java | 26 ++ .../runtime/filesystem/http/HttpPath.java | 291 ++++++++++++ .../filesystem/http/HttpWatchService.java | 438 ++++++++++++++++++ .../http/HttpsFileSystemProvider.java | 26 ++ .../http/ReadOnlyByteArrayChannel.java | 115 +++++ .../src/main/moditect/module-info.java | 23 + .../java.nio.file.spi.FileSystemProvider | 2 + .../filesystem/http/HttpFileSystemIT.java | 255 ++++++++++ .../filesystem/http/TestHttpFileSystem.java | 122 +++++ runtime/pom.xml | 6 + specs/filesystem-http.spec/COPYRIGHT | 12 + specs/filesystem-http.spec/LICENSE | 114 +++++ specs/filesystem-http.spec/NOTICE | 13 + specs/filesystem-http.spec/NOTICE.template | 13 + specs/filesystem-http.spec/mvnw | 310 +++++++++++++ specs/filesystem-http.spec/mvnw.cmd | 182 ++++++++ specs/filesystem-http.spec/pom.xml | 123 +++++ .../src/main/moditect/module-info.java | 17 + .../http/application/notfound/client.rpt | 23 + .../http/application/notfound/server.rpt | 25 + .../http/application/success/client.rpt | 25 + .../http/application/success/server.rpt | 27 ++ .../watch.error.success/client.rpt | 37 ++ .../watch.error.success/server.rpt | 38 ++ .../http/application/watch/client.rpt | 39 ++ .../http/application/watch/server.rpt | 42 ++ .../specs/filesystem/http/ApplicationIT.java | 74 +++ specs/pom.xml | 1 + 40 files changed, 3790 insertions(+) create mode 100644 runtime/filesystem-http/COPYRIGHT create mode 100644 runtime/filesystem-http/LICENSE create mode 100644 runtime/filesystem-http/NOTICE create mode 100644 runtime/filesystem-http/NOTICE.template create mode 100755 runtime/filesystem-http/mvnw create mode 100644 runtime/filesystem-http/mvnw.cmd create mode 100644 runtime/filesystem-http/pom.xml create mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpBaseFileSystemProvider.java create mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java create mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java create mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java create mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java create mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpsFileSystemProvider.java create mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/ReadOnlyByteArrayChannel.java create mode 100644 runtime/filesystem-http/src/main/moditect/module-info.java create mode 100644 runtime/filesystem-http/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider create mode 100644 runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java create mode 100644 runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/TestHttpFileSystem.java create mode 100644 specs/filesystem-http.spec/COPYRIGHT create mode 100644 specs/filesystem-http.spec/LICENSE create mode 100644 specs/filesystem-http.spec/NOTICE create mode 100644 specs/filesystem-http.spec/NOTICE.template create mode 100755 specs/filesystem-http.spec/mvnw create mode 100644 specs/filesystem-http.spec/mvnw.cmd create mode 100644 specs/filesystem-http.spec/pom.xml create mode 100644 specs/filesystem-http.spec/src/main/moditect/module-info.java create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/client.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/server.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/client.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/server.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/client.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/server.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/server.rpt create mode 100644 specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java diff --git a/cloud/docker-image/pom.xml b/cloud/docker-image/pom.xml index 0b37392348..a3ff265ca9 100644 --- a/cloud/docker-image/pom.xml +++ b/cloud/docker-image/pom.xml @@ -253,6 +253,12 @@ ${project.version} runtime + + ${project.groupId} + filesystem-http + ${project.version} + runtime + ${project.groupId} metrics-stream diff --git a/cloud/docker-image/src/main/docker/assembly.xml b/cloud/docker-image/src/main/docker/assembly.xml index ca3954717b..7e0d989f1a 100644 --- a/cloud/docker-image/src/main/docker/assembly.xml +++ b/cloud/docker-image/src/main/docker/assembly.xml @@ -30,6 +30,7 @@ io/aklivity/zilla/binding-*/** io/aklivity/zilla/catalog-*/** io/aklivity/zilla/exporter-*/** + io/aklivity/zilla/filesystem-*/** io/aklivity/zilla/guard-*/** io/aklivity/zilla/metrics-*/** io/aklivity/zilla/model-*/** diff --git a/cloud/docker-image/src/main/docker/zpm.json.template b/cloud/docker-image/src/main/docker/zpm.json.template index 46961f11bb..7a792ce44d 100644 --- a/cloud/docker-image/src/main/docker/zpm.json.template +++ b/cloud/docker-image/src/main/docker/zpm.json.template @@ -50,6 +50,7 @@ "io.aklivity.zilla:exporter-otlp", "io.aklivity.zilla:exporter-prometheus", "io.aklivity.zilla:exporter-stdout", + "io.aklivity.zilla:filesystem-http", "io.aklivity.zilla:guard-jwt", "io.aklivity.zilla:metrics-stream", "io.aklivity.zilla:metrics-http", diff --git a/runtime/filesystem-http/COPYRIGHT b/runtime/filesystem-http/COPYRIGHT new file mode 100644 index 0000000000..0cb10b6f62 --- /dev/null +++ b/runtime/filesystem-http/COPYRIGHT @@ -0,0 +1,12 @@ +Copyright ${copyrightYears} Aklivity Inc + +Licensed under the Aklivity Community License (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + + https://www.aklivity.io/aklivity-community-license/ + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/runtime/filesystem-http/LICENSE b/runtime/filesystem-http/LICENSE new file mode 100644 index 0000000000..f6abb6327b --- /dev/null +++ b/runtime/filesystem-http/LICENSE @@ -0,0 +1,114 @@ + Aklivity Community License Agreement + Version 1.0 + +This Aklivity Community License Agreement Version 1.0 (the “Agreement”) sets +forth the terms on which Aklivity, Inc. (“Aklivity”) makes available certain +software made available by Aklivity under this Agreement (the “Software”). BY +INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY OF THE SOFTWARE, +YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO +SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. IF YOU ARE RECEIVING +THE SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU +HAVE THE ACTUAL AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS +AGREEMENT ON BEHALF OF SUCH ENTITY. “Licensee” means you, an individual, or +the entity on whose behalf you are receiving the Software. + + 1. LICENSE GRANT AND CONDITIONS. + + 1.1 License. Subject to the terms and conditions of this Agreement, + Aklivity hereby grants to Licensee a non-exclusive, royalty-free, + worldwide, non-transferable, non-sublicenseable license during the term + of this Agreement to: (a) use the Software; (b) prepare modifications and + derivative works of the Software; (c) distribute the Software (including + without limitation in source code or object code form); and (d) reproduce + copies of the Software (the “License”). Licensee is not granted the + right to, and Licensee shall not, exercise the License for an Excluded + Purpose. For purposes of this Agreement, “Excluded Purpose” means making + available any software-as-a-service, platform-as-a-service, + infrastructure-as-a-service or other similar online service that competes + with Aklivity products or services that provide the Software. + + 1.2 Conditions. In consideration of the License, Licensee’s distribution + of the Software is subject to the following conditions: + + (a) Licensee must cause any Software modified by Licensee to carry + prominent notices stating that Licensee modified the Software. + + (b) On each Software copy, Licensee shall reproduce and not remove or + alter all Aklivity or third party copyright or other proprietary + notices contained in the Software, and Licensee must provide the + notice below with each copy. + + “This software is made available by Aklivity, Inc., under the + terms of the Aklivity Community License Agreement, Version 1.0 + located at http://www.Aklivity.io/Aklivity-community-license. BY + INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY OF + THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.” + + 1.3 Licensee Modifications. Licensee may add its own copyright notices + to modifications made by Licensee and may provide additional or different + license terms and conditions for use, reproduction, or distribution of + Licensee’s modifications. While redistributing the Software or + modifications thereof, Licensee may choose to offer, for a fee or free of + charge, support, warranty, indemnity, or other obligations. Licensee, and + not Aklivity, will be responsible for any such obligations. + + 1.4 No Sublicensing. The License does not include the right to + sublicense the Software, however, each recipient to which Licensee + provides the Software may exercise the Licenses so long as such recipient + agrees to the terms and conditions of this Agreement. + + 2. TERM AND TERMINATION. This Agreement will continue unless and until + earlier terminated as set forth herein. If Licensee breaches any of its + conditions or obligations under this Agreement, this Agreement will + terminate automatically and the License will terminate automatically and + permanently. + + 3. INTELLECTUAL PROPERTY. As between the parties, Aklivity will retain all + right, title, and interest in the Software, and all intellectual property + rights therein. Aklivity hereby reserves all rights not expressly granted + to Licensee in this Agreement. Aklivity hereby reserves all rights in its + trademarks and service marks, and no licenses therein are granted in this + Agreement. + + 4. DISCLAIMER. Aklivity HEREBY DISCLAIMS ANY AND ALL WARRANTIES AND + CONDITIONS, EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, AND SPECIFICALLY + DISCLAIMS ANY WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR + PURPOSE, WITH RESPECT TO THE SOFTWARE. + + 5. LIMITATION OF LIABILITY. Aklivity WILL NOT BE LIABLE FOR ANY DAMAGES OF + ANY KIND, INCLUDING BUT NOT LIMITED TO, LOST PROFITS OR ANY CONSEQUENTIAL, + SPECIAL, INCIDENTAL, INDIRECT, OR DIRECT DAMAGES, HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, ARISING OUT OF THIS AGREEMENT. THE FOREGOING SHALL + APPLY TO THE EXTENT PERMITTED BY APPLICABLE LAW. + + 6.GENERAL. + + 6.1 Governing Law. This Agreement will be governed by and interpreted in + accordance with the laws of the state of California, without reference to + its conflict of laws principles. If Licensee is located within the + United States, all disputes arising out of this Agreement are subject to + the exclusive jurisdiction of courts located in Santa Clara County, + California. USA. If Licensee is located outside of the United States, + any dispute, controversy or claim arising out of or relating to this + Agreement will be referred to and finally determined by arbitration in + accordance with the JAMS International Arbitration Rules. The tribunal + will consist of one arbitrator. The place of arbitration will be Palo + Alto, California. The language to be used in the arbitral proceedings + will be English. Judgment upon the award rendered by the arbitrator may + be entered in any court having jurisdiction thereof. + + 6.2 Assignment. Licensee is not authorized to assign its rights under + this Agreement to any third party. Aklivity may freely assign its rights + under this Agreement to any third party. + + 6.3 Other. This Agreement is the entire agreement between the parties + regarding the subject matter hereof. No amendment or modification of + this Agreement will be valid or binding upon the parties unless made in + writing and signed by the duly authorized representatives of both + parties. In the event that any provision, including without limitation + any condition, of this Agreement is held to be unenforceable, this + Agreement and all licenses and rights granted hereunder will immediately + terminate. Waiver by Aklivity of a breach of any provision of this + Agreement or the failure by Aklivity to exercise any right hereunder + will not be construed as a waiver of any subsequent breach of that right + or as a waiver of any other right. \ No newline at end of file diff --git a/runtime/filesystem-http/NOTICE b/runtime/filesystem-http/NOTICE new file mode 100644 index 0000000000..d5dee9ccfc --- /dev/null +++ b/runtime/filesystem-http/NOTICE @@ -0,0 +1,14 @@ +Licensed under the Aklivity Community License (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + + https://www.aklivity.io/aklivity-community-license/ + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +This project includes: + agrona under The Apache License, Version 2.0 + diff --git a/runtime/filesystem-http/NOTICE.template b/runtime/filesystem-http/NOTICE.template new file mode 100644 index 0000000000..209ca12f74 --- /dev/null +++ b/runtime/filesystem-http/NOTICE.template @@ -0,0 +1,13 @@ +Licensed under the Aklivity Community License (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + + https://www.aklivity.io/aklivity-community-license/ + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +This project includes: +#GENERATED_NOTICES# diff --git a/runtime/filesystem-http/mvnw b/runtime/filesystem-http/mvnw new file mode 100755 index 0000000000..d2f0ea3808 --- /dev/null +++ b/runtime/filesystem-http/mvnw @@ -0,0 +1,310 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" + fi + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` + fi + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=`cygpath --path --windows "$javaClass"` + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/runtime/filesystem-http/mvnw.cmd b/runtime/filesystem-http/mvnw.cmd new file mode 100644 index 0000000000..b26ab24f03 --- /dev/null +++ b/runtime/filesystem-http/mvnw.cmd @@ -0,0 +1,182 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" + +FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/runtime/filesystem-http/pom.xml b/runtime/filesystem-http/pom.xml new file mode 100644 index 0000000000..10ab1d2590 --- /dev/null +++ b/runtime/filesystem-http/pom.xml @@ -0,0 +1,205 @@ + + + + 4.0.0 + + io.aklivity.zilla + runtime + develop-SNAPSHOT + ../pom.xml + + + filesystem-http + zilla::runtime::filesystem-http + + + + Aklivity Community License Agreement + https://www.aklivity.io/aklivity-community-license/ + repo + + + + + 11 + 11 + 0.60 + 0 + + + + + ${project.groupId} + filesystem-http.spec + ${project.version} + provided + + + org.agrona + agrona + + + junit + junit + test + + + org.hamcrest + hamcrest + test + + + com.vtence.hamcrest + hamcrest-jpa + test + + + com.github.npathai + hamcrest-optional + test + + + org.mockito + mockito-core + test + + + io.aklivity.k3po + control-junit + test + + + io.aklivity.k3po + lang + test + + + org.openjdk.jmh + jmh-core + test + + + org.openjdk.jmh + jmh-generator-annprocess + test + + + + + + + org.jasig.maven + maven-notice-plugin + + + com.mycila + license-maven-plugin + + + maven-checkstyle-plugin + + + maven-dependency-plugin + + + unpack-test-resources + process-test-resources + + unpack + + + + + ${project.groupId} + filesystem-http.spec + + + ^\Qio/aklivity/zilla/specs/filesystem/http/\E + io/aklivity/zilla/runtime/filesystem/http/internal/ + + + + + + io/aklivity/zilla/specs/filesystem/http/application/**/*, + + ${project.build.directory}/test-classes + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.moditect + moditect-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + io.aklivity.k3po + k3po-maven-plugin + + + org.apache.maven.plugins + maven-failsafe-plugin + + + org.jacoco + jacoco-maven-plugin + + + + BUNDLE + + + INSTRUCTION + COVEREDRATIO + ${jacoco.coverage.ratio} + + + CLASS + MISSEDCOUNT + ${jacoco.missed.count} + + + + + + + + io.gatling + maven-shade-plugin + + + + org.openjdk.jmh:jmh-core + net.sf.jopt-simple:jopt-simple + org.apache.commons:commons-math3 + commons-cli:commons-cli + com.github.biboudis:jmh-profilers + + + + + + + diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpBaseFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpBaseFileSystemProvider.java new file mode 100644 index 0000000000..a12b5f6d97 --- /dev/null +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpBaseFileSystemProvider.java @@ -0,0 +1,361 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http; + +import static java.net.HttpURLConnection.HTTP_OK; +import static java.net.http.HttpClient.Redirect.NORMAL; +import static java.net.http.HttpClient.Version.HTTP_2; +import static org.agrona.LangUtil.rethrowUnchecked; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.channels.AsynchronousFileChannel; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemAlreadyExistsException; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.LinkOption; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.spi.FileSystemProvider; +import java.time.Duration; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; + +public abstract class HttpBaseFileSystemProvider extends FileSystemProvider +{ + private static final Duration TIMEOUT = Duration.ofSeconds(5); + private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() + .version(HTTP_2) + .followRedirects(NORMAL) + .build(); + + private final Map fileSystems = new ConcurrentHashMap<>(); + + @Override + public abstract String getScheme(); + + @Override + public FileSystem newFileSystem( + URI uri, + Map env) + { + checkUri(uri); + HttpFileSystem hfs = fileSystems.get(uri); + if (hfs == null) + { + hfs = new HttpFileSystem(this, uri); + fileSystems.put(uri, hfs); + } + else + { + throw new FileSystemAlreadyExistsException(); + } + return hfs; + } + + @Override + public FileSystem getFileSystem( + URI uri) + { + checkUri(uri); + HttpFileSystem hfs = fileSystems.get(uri); + if (hfs == null) + { + throw new FileSystemNotFoundException(); + } + return hfs; + } + + @Override + public Path getPath( + URI uri) + { + FileSystem hfs = fileSystems.get(uri); + if (hfs == null) + { + hfs = newFileSystem(uri, Map.of()); + } + return hfs.getPath(uri.toString()); + } + + @Override + public FileSystem newFileSystem( + Path path, + Map env) + { + return newFileSystem(path.toUri(), env); + } + + @Override + public InputStream newInputStream( + Path path, + OpenOption... options) + { + return new ByteArrayInputStream(resolveBody(path)); + } + + @Override + public OutputStream newOutputStream( + Path path, + OpenOption... options) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public FileChannel newFileChannel( + Path path, + Set options, + FileAttribute... attrs) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public AsynchronousFileChannel newAsynchronousFileChannel( + Path path, + Set options, + ExecutorService executor, + FileAttribute... attrs) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public SeekableByteChannel newByteChannel( + Path path, + Set options, + FileAttribute... attrs) + { + return new ReadOnlyByteArrayChannel(resolveBody(path)); + } + + @Override + public DirectoryStream newDirectoryStream( + Path dir, + DirectoryStream.Filter filter) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void createDirectory( + Path dir, + FileAttribute... attrs) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void createSymbolicLink( + Path link, + Path target, FileAttribute... attrs) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void createLink( + Path link, + Path existing) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void delete( + Path path) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean deleteIfExists( + Path path) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path readSymbolicLink( + Path link) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void copy( + Path source, + Path target, + CopyOption... options) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void move( + Path source, + Path target, + CopyOption... options) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean isSameFile( + Path path, + Path path2) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean isHidden( + Path path) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public FileStore getFileStore( + Path path) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void checkAccess( + Path path, + AccessMode... modes) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public V getFileAttributeView( + Path path, + Class type, + LinkOption... options) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public A readAttributes( + Path path, + Class type, + LinkOption... options) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Map readAttributes( + Path path, + String attributes, + LinkOption... options) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void setAttribute( + Path path, + String attribute, + Object value, LinkOption... options) + { + throw new UnsupportedOperationException("not implemented"); + } + + void closeFileSystem( + URI uri) + { + fileSystems.remove(uri); + } + + private void checkUri( + URI uri) + { + if (!uri.getScheme().equalsIgnoreCase(getScheme())) + { + throw new IllegalArgumentException("URI does not match this provider"); + } + if (uri.getPath() == null) + { + throw new IllegalArgumentException("Path component is undefined"); + } + } + + private byte[] resolveBody( + Path path) + { + byte[] body; + HttpFileSystem fileSystem = fileSystems.get(path.toUri()); + if (fileSystem != null && fileSystem.body() != null) + { + System.out.println("HBFSP resolveBody fs.body"); // TODO: Ati + body = fileSystem.body(); + } + else + { + System.out.println("HBFSP resolveBody readBody"); // TODO: Ati + body = readBody(path); + } + return body; + } + + private static byte[] readBody( + Path path) + { + byte[] body = new byte[0]; + try + { + HttpRequest request = HttpRequest.newBuilder() + .GET() + .uri(path.toUri()) + .timeout(TIMEOUT) + .build(); + System.out.println("HBFSP readBody path " + path + " request " + request); // TODO: Ati + HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofByteArray()); + System.out.println("HBFSP readBody response.body " + new String(response.body())); // TODO: Ati + if (response.statusCode() == HTTP_OK) + { + body = response.body(); + } + } + catch (Exception ex) + { + System.out.println("HBFSP readBody exception " + ex); // TODO: Ati + rethrowUnchecked(ex); + } + return body; + } +} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java new file mode 100644 index 0000000000..6982ef2372 --- /dev/null +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java @@ -0,0 +1,152 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http; + +import static java.util.Objects.requireNonNull; +import static org.agrona.LangUtil.rethrowUnchecked; + +import java.net.URI; +import java.net.URL; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.WatchService; +import java.nio.file.attribute.UserPrincipalLookupService; +import java.nio.file.spi.FileSystemProvider; +import java.util.Set; + +public class HttpFileSystem extends FileSystem +{ + public static final String SEPARATOR = "/"; + + private final HttpBaseFileSystemProvider provider; + private final URI uri; + + private byte[] body; + + HttpFileSystem( + HttpBaseFileSystemProvider provider, + URI uri) + { + this.provider = provider; + this.uri = uri; + this.body = null; + } + + @Override + public FileSystemProvider provider() + { + return provider; + } + + @Override + public void close() + { + provider.closeFileSystem(uri); + } + + @Override + public boolean isOpen() + { + return true; + } + + @Override + public boolean isReadOnly() + { + return true; + } + + @Override + public String getSeparator() + { + return SEPARATOR; + } + + @Override + public Iterable getRootDirectories() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Iterable getFileStores() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Set supportedFileAttributeViews() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path getPath( + String first, + String... more) + { + requireNonNull(first); + requireNonNull(more); + String second = String.join(SEPARATOR, more); + String path = second.isBlank() ? first : first + SEPARATOR + second; + Path result = null; + try + { + result = new HttpPath(this, new URL(path)); + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } + return result; + } + + @Override + public PathMatcher getPathMatcher( + String syntaxAndPattern) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public UserPrincipalLookupService getUserPrincipalLookupService() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public WatchService newWatchService() + { + return new HttpWatchService(this); + } + + URI baseUri() + { + return this.uri; + } + + byte[] body() + { + return body; + } + + void body( + byte[] body) + { + this.body = body; + } +} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java new file mode 100644 index 0000000000..36b28f3e50 --- /dev/null +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java @@ -0,0 +1,26 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http; + +public class HttpFileSystemProvider extends HttpBaseFileSystemProvider +{ + public static final String SCHEME = "http"; + + @Override + public final String getScheme() + { + return SCHEME; + } +} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java new file mode 100644 index 0000000000..f626882c49 --- /dev/null +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -0,0 +1,291 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http; + +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; +import static java.util.Objects.requireNonNull; +import static org.agrona.LangUtil.rethrowUnchecked; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileSystem; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.ProviderMismatchException; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.Iterator; +import java.util.Objects; + +public class HttpPath implements Path +{ + private final HttpFileSystem fs; + private final URL url; + + HttpPath( + HttpFileSystem fs, + URL url) + { + if (!fs.provider().getScheme().equals(url.getProtocol())) + { + throw new IllegalArgumentException(String.format("invalid protocol: %s", url.getProtocol())); + } + this.fs = fs; + this.url = url; + } + + @Override + public FileSystem getFileSystem() + { + return fs; + } + + @Override + public boolean isAbsolute() + { + return true; + } + + @Override + public Path getRoot() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path getFileName() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path getParent() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public int getNameCount() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path getName( + int index) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path subpath( + int beginIndex, + int endIndex) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean startsWith( + Path other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean startsWith( + String other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean endsWith( + Path other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean endsWith( + String other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path normalize() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path resolve( + Path other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path resolve( + String other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path resolveSibling( + Path other) + { + return resolveSibling(other.toString()); + } + + @Override + public Path resolveSibling( + String other) + { + Path path = null; + try + { + path = new HttpPath(fs, new URL(fs.baseUri().toURL(), other)); + } + catch (Exception ex) + { + rethrowUnchecked(ex); + } + return path; + } + + @Override + public Path relativize( + Path other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public URI toUri() + { + URI result = null; + try + { + result = url.toURI(); + } + catch (URISyntaxException ex) + { + rethrowUnchecked(ex); + } + return result; + } + + @Override + public Path toAbsolutePath() + { + return this; + } + + @Override + public Path toRealPath( + LinkOption... options) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public File toFile() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public WatchKey register( + WatchService watcher, + WatchEvent.Kind[] events, + WatchEvent.Modifier... modifiers) + throws IOException + { + requireNonNull(watcher); + if (!(watcher instanceof HttpWatchService)) + { + throw new ProviderMismatchException(); + } + for (WatchEvent.Kind event : events) + { + if (!event.equals(ENTRY_MODIFY)) + { + throw new IllegalArgumentException("Only ENTRY_MODIFY event kind is supported"); + } + } + if (modifiers.length > 0) + { + throw new IllegalArgumentException("Modifiers are not supported"); + } + return ((HttpWatchService) watcher).register(this, events, modifiers); + } + + @Override + public WatchKey register( + WatchService watcher, + WatchEvent.Kind... events) throws IOException + { + return register(watcher, events, new WatchEvent.Modifier[0]); + } + + @Override + public Iterator iterator() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public int compareTo( + Path other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public String toString() + { + return url.toString(); + } + + @Override + public boolean equals( + Object o) + { + if (this == o) + { + return true; + } + if (o == null || getClass() != o.getClass()) + { + return false; + } + + HttpPath path = (HttpPath) o; + return Objects.equals(url, path.url); + } + + @Override + public int hashCode() + { + return Objects.hashCode(url); + } +} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java new file mode 100644 index 0000000000..b427b518e8 --- /dev/null +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -0,0 +1,438 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http; + +import static java.net.http.HttpClient.Redirect.NORMAL; +import static java.net.http.HttpClient.Version.HTTP_2; +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; +import static org.agrona.LangUtil.rethrowUnchecked; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.ClosedWatchServiceException; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.nio.file.Watchable; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public class HttpWatchService implements WatchService, Callable +{ + private static final Duration TIMEOUT = Duration.ofSeconds(5); + private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() + .version(HTTP_2) + .followRedirects(NORMAL) + .build(); + private static final Path CLOSE_PATH = Path.of(URI.create("http://localhost:12345")); + private static final byte[] EMPTY_BODY = new byte[0]; + + private final WatchKey closeKey = new HttpWatchKey(CLOSE_PATH); + + private final HttpFileSystem fileSystem; + private final ScheduledExecutorService executor; + private final LinkedBlockingQueue pendingKeys; + private final BlockingQueue pathQueue; + private final Map watchKeys; + private final Map etags; + private final Map hashes; + private final Map> futures; + private final MessageDigest md5; + + private int pollSeconds; + private volatile boolean closed; + + public HttpWatchService( + HttpFileSystem fileSystem) + { + this.fileSystem = fileSystem; + this.executor = Executors.newScheduledThreadPool(2); + this.pendingKeys = new LinkedBlockingQueue<>(); + this.watchKeys = new ConcurrentHashMap<>(); + this.pathQueue = new LinkedBlockingQueue<>(); + this.etags = new ConcurrentHashMap<>(); + this.hashes = new ConcurrentHashMap<>(); + this.futures = new ConcurrentHashMap<>(); + this.md5 = initMessageDigest("MD5"); + this.pollSeconds = 30; + this.closed = false; + executor.submit(this); + } + + @Override + public Void call() throws Exception + { + while (true) + { + Path path = pathQueue.take(); + if (path == CLOSE_PATH) + { + break; + } + String etag = etags.getOrDefault(path, ""); + System.out.println("HWS call take path " + path + " etag [" + etag + "]"); // TODO: Ati + sendAsync(path, etag); + } + return null; + } + + @Override + public void close() + { + closed = true; + fileSystem.body(null); + pendingKeys.clear(); + pendingKeys.offer(closeKey); + futures.values().forEach(future -> future.cancel(true)); + pathQueue.add(CLOSE_PATH); + watchKeys.clear(); + } + + @Override + public WatchKey poll() + { + checkOpen(); + WatchKey key = pendingKeys.poll(); + checkKey(key); + return key; + } + + @Override + public WatchKey poll( + long timeout, + TimeUnit unit) throws InterruptedException + { + checkOpen(); + WatchKey key = pendingKeys.poll(timeout, unit); + checkKey(key); + return key; + } + + @Override + public WatchKey take() throws InterruptedException + { + checkOpen(); + WatchKey key = pendingKeys.take(); + checkKey(key); + return key; + } + + public void pollSeconds( + int pollSeconds) + { + this.pollSeconds = pollSeconds; + } + + private void checkOpen() + { + if (closed) + { + throw new ClosedWatchServiceException(); + } + } + + private void checkKey( + WatchKey key) + { + if (key == closeKey) + { + enqueueKey(closeKey); + } + checkOpen(); + } + + private void enqueueKey( + WatchKey key) + { + pendingKeys.offer(key); + } + + WatchKey register( + final HttpPath path, + WatchEvent.Kind[] events, + WatchEvent.Modifier... modifiers) + { + System.out.printf("HWS register path: %s\n", path); // TODO: Ati + WatchKey watchKey = watchKeys.computeIfAbsent(path, i -> new HttpWatchKey(path)); + pathQueue.offer(path); + return watchKey; + } + + private void sendAsync( + Path path, + String etag) + { + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .GET() + .uri(path.toUri()) + .timeout(TIMEOUT); + if (etag != null && !etag.isEmpty()) + { + requestBuilder = requestBuilder.headers("If-None-Match", etag, "Prefer", "wait=86400"); + } + + System.out.println("HWS sendAsync path " + path + " etag " + etag); // TODO: Ati + CompletableFuture future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + .thenAccept(this::handleChange) + .exceptionally(ex -> handleException(ex, path)); + futures.put(path, future); + } + + private Void handleException( + Throwable throwable, + Path path) + { + System.out.println("HWS handleException " + throwable.getMessage()); // TODO: Ati + scheduleRequest(path, pollSeconds); + return null; + } + + private void handleChange( + HttpResponse response) + { + System.out.println("HWS handleChange response: " + response); // TODO: Ati + System.out.println("HWS handleChange response.headers: " + response.headers()); // TODO: Ati + System.out.println("HWS handleChange response.body: " + new String(response.body())); // TODO: Ati + Path path = Path.of(response.request().uri()); + int statusCode = response.statusCode(); + int pollSeconds = 0; + if (statusCode == 404) + { + fileSystem.body(EMPTY_BODY); + addEvent(path); + pollSeconds = this.pollSeconds; + } + else if (statusCode >= 500 && statusCode <= 599) + { + fileSystem.body(EMPTY_BODY); + pollSeconds = this.pollSeconds; + } + else + { + byte[] body = response.body(); + fileSystem.body(body); + Optional etagOptional = response.headers().firstValue("Etag"); + if (etagOptional.isPresent()) + { + String oldEtag = etags.getOrDefault(path, ""); + String newEtag = etagOptional.get(); + if (!oldEtag.equals(newEtag)) + { + etags.put(path, newEtag); + addEvent(path); + + } + else if (response.statusCode() != 304) + { + pollSeconds = this.pollSeconds; + } + } + else + { + byte[] hash = hashes.get(path); + byte[] newHash = computeHash(body); + if (!Arrays.equals(hash, newHash)) + { + hashes.put(path, newHash); + addEvent(path); + } + pollSeconds = this.pollSeconds; + } + } + futures.remove(path); + scheduleRequest(path, pollSeconds); + } + + private void addEvent( + Path path) + { + System.out.println("HWS addEvent path " + path); // TODO: Ati + HttpWatchKey key = (HttpWatchKey) watchKeys.get(path); + if (key != null) + { + key.addEvent(ENTRY_MODIFY, path); + enqueueKey(key); + } + } + + private void scheduleRequest( + Path path, + int pollSeconds) + { + if (pollSeconds == 0) + { + System.out.println("HWS scheduleRequest 0"); // TODO: Ati + pathQueue.add(path); + } + else + { + System.out.println("HWS scheduleRequest " + pollSeconds); // TODO: Ati + executor.schedule(() -> pathQueue.add(path), pollSeconds, TimeUnit.SECONDS); + } + } + + private byte[] computeHash( + byte[] body) + { + return md5.digest(body); + } + + private MessageDigest initMessageDigest( + String algorithm) + { + MessageDigest md5 = null; + try + { + md5 = MessageDigest.getInstance(algorithm); + } + catch (NoSuchAlgorithmException ex) + { + rethrowUnchecked(ex); + } + return md5; + } + + private final class HttpWatchKey implements WatchKey + { + private final Path path; + + private List> events = Collections.synchronizedList(new LinkedList<>()); + + private volatile boolean valid; + + private HttpWatchKey( + Path path) + { + this.path = path; + this.valid = true; + } + + @Override + public boolean isValid() + { + return valid; + } + + @Override + public List> pollEvents() + { + List> result = events; + events = Collections.synchronizedList(new LinkedList<>()); + return result; + } + + @Override + public boolean reset() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void cancel() + { + watchKeys.remove(path); + valid = false; + } + + @Override + public Watchable watchable() + { + return path; + } + + void addEvent( + WatchEvent.Kind kind, + Path context) + { + Event ev = new Event<>(kind, context); + events.add(ev); + } + + @Override + public boolean equals(Object o) + { + if (this == o) + { + return true; + } + if (o == null || getClass() != o.getClass()) + { + return false; + } + + HttpWatchKey watchKey = (HttpWatchKey) o; + return Objects.equals(path, watchKey.path); + } + + @Override + public int hashCode() + { + return Objects.hashCode(path); + } + + private static class Event implements WatchEvent + { + private final WatchEvent.Kind kind; + private final T context; + private final int count; + + Event( + WatchEvent.Kind type, + T context) + { + this.kind = type; + this.context = context; + this.count = 1; + } + + @Override + public WatchEvent.Kind kind() + { + return kind; + } + + @Override + public T context() + { + return context; + } + + @Override + public int count() + { + return count; + } + } + } +} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpsFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpsFileSystemProvider.java new file mode 100644 index 0000000000..8500c96638 --- /dev/null +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpsFileSystemProvider.java @@ -0,0 +1,26 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http; + +public class HttpsFileSystemProvider extends HttpBaseFileSystemProvider +{ + public static final String SCHEME = "https"; + + @Override + public final String getScheme() + { + return SCHEME; + } +} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/ReadOnlyByteArrayChannel.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/ReadOnlyByteArrayChannel.java new file mode 100644 index 0000000000..d075596dcd --- /dev/null +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/ReadOnlyByteArrayChannel.java @@ -0,0 +1,115 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.SeekableByteChannel; + +public class ReadOnlyByteArrayChannel implements SeekableByteChannel +{ + private final byte[] data; + private int position; + private boolean closed; + + public ReadOnlyByteArrayChannel( + byte[] data) + { + this.data = data; + this.position = 0; + this.closed = false; + } + + @Override + public int read( + ByteBuffer dst) throws IOException + { + ensureOpen(); + int bytesRead; + if (position >= data.length) + { + bytesRead = -1; + } + else + { + bytesRead = Math.min(dst.remaining(), data.length - position); + dst.put(data, position, bytesRead); + position += bytesRead; + } + return bytesRead; + } + + @Override + public int write( + ByteBuffer src) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public long position() throws IOException + { + ensureOpen(); + return position; + } + + @Override + public SeekableByteChannel position( + long newPosition) throws IOException + { + ensureOpen(); + if (newPosition < 0 || newPosition > data.length) + { + throw new IllegalArgumentException("Position out of bounds"); + } + this.position = (int) newPosition; + return this; + } + + @Override + public long size() throws IOException + { + ensureOpen(); + return data.length; + } + + @Override + public SeekableByteChannel truncate( + long size) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean isOpen() + { + return !closed; + } + + @Override + public void close() + { + closed = true; + } + + private void ensureOpen() throws IOException + { + if (closed) + { + throw new ClosedChannelException(); + } + } +} diff --git a/runtime/filesystem-http/src/main/moditect/module-info.java b/runtime/filesystem-http/src/main/moditect/module-info.java new file mode 100644 index 0000000000..d35f116893 --- /dev/null +++ b/runtime/filesystem-http/src/main/moditect/module-info.java @@ -0,0 +1,23 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +module io.aklivity.zilla.runtime.filesystem.http +{ + requires java.net.http; + requires org.agrona.core; + + provides java.nio.file.spi.FileSystemProvider with + io.aklivity.zilla.runtime.filesystem.http.HttpFileSystemProvider, + io.aklivity.zilla.runtime.filesystem.http.HttpsFileSystemProvider; +} diff --git a/runtime/filesystem-http/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider b/runtime/filesystem-http/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider new file mode 100644 index 0000000000..5628d02fb6 --- /dev/null +++ b/runtime/filesystem-http/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider @@ -0,0 +1,2 @@ +io.aklivity.zilla.runtime.filesystem.http.HttpFileSystemProvider +io.aklivity.zilla.runtime.filesystem.http.HttpsFileSystemProvider diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java new file mode 100644 index 0000000000..e87edce52f --- /dev/null +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java @@ -0,0 +1,255 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http; + +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.rules.RuleChain.outerRule; + +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.spi.FileSystemProvider; +import java.util.List; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import io.aklivity.k3po.runtime.junit.annotation.Specification; +import io.aklivity.k3po.runtime.junit.rules.K3poRule; + +public class HttpFileSystemIT +{ + private static final String HELLO_BODY = "Hello World!"; + private static final String EMPTY_BODY = ""; + + private final K3poRule k3po = new K3poRule() + .addScriptRoot("app", "io/aklivity/zilla/specs/filesystem/http/application"); + + private final TestRule timeout = new DisableOnDebug(new Timeout(10, SECONDS)); + + @Rule + public final TestRule chain = outerRule(timeout).around(k3po); + + @Test + @Specification({ + "${app}/success/server", + }) + public void shouldReadString() throws Exception + { + // GIVEN + String helloUrl = "http://localhost:8080/hello.txt"; + Path helloPath = Path.of(new URI(helloUrl)); + + // WHEN + k3po.start(); + String helloBody = Files.readString(helloPath); + k3po.finish(); + + // THEN + assertThat(helloBody, equalTo(HELLO_BODY)); + } + + @Test + @Specification({ + "${app}/notfound/server", + }) + public void shouldReadStringNotFound() throws Exception + { + // GIVEN + String notFoundUrl = "http://localhost:8080/notfound.txt"; + Path notFoundPath = Path.of(new URI(notFoundUrl)); + + // WHEN + k3po.start(); + String notFoundBody = Files.readString(notFoundPath); + k3po.finish(); + + // THEN + assertThat(notFoundBody, equalTo(EMPTY_BODY)); + } + + @Test + @Specification({ + "${app}/success/server", + }) + public void shouldReadInputStream() throws Exception + { + // GIVEN + String helloUrl = "http://localhost:8080/hello.txt"; + Path helloPath = Path.of(new URI(helloUrl)); + FileSystemProvider fsp = helloPath.getFileSystem().provider(); + + // WHEN + k3po.start(); + InputStream helloIs = fsp.newInputStream(helloPath); + String helloBody = new String(helloIs.readAllBytes()); + helloIs.close(); + k3po.finish(); + + // THEN + assertThat(helloBody, equalTo(HELLO_BODY)); + } + + @Test + @Specification({ + "${app}/notfound/server", + }) + public void shouldReadInputStreamNotFound() throws Exception + { + // GIVEN + String notFoundUrl = "http://localhost:8080/notfound.txt"; + Path notFoundPath = Path.of(new URI(notFoundUrl)); + FileSystemProvider fsp = notFoundPath.getFileSystem().provider(); + + // WHEN + k3po.start(); + InputStream notFoundIs = fsp.newInputStream(notFoundPath); + String notFoundBody = new String(notFoundIs.readAllBytes()); + notFoundIs.close(); + k3po.finish(); + + // THEN + assertThat(notFoundBody, equalTo(EMPTY_BODY)); + } + + @Test + @Specification({ + "${app}/watch/server", + }) + public void shouldWatch() throws Exception + { + // GIVEN + String url = "http://localhost:8080/hello.txt"; + Path path = Path.of(new URI(url)); + HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); + watchService.pollSeconds(1); + path.register(watchService); + + // WHEN + k3po.start(); + WatchKey key1 = watchService.take(); + List> events1 = key1.pollEvents(); + k3po.notifyBarrier("CHANGED"); + WatchKey key2 = watchService.take(); + List> events2 = key2.pollEvents(); + watchService.close(); + k3po.finish(); + + // THEN + assertThat(events1.size(), equalTo(1)); + assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); + assertThat(events1.get(0).context(), equalTo(path)); + assertThat(events2.size(), equalTo(1)); + assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); + assertThat(events2.get(0).context(), equalTo(path)); + } + + @Test + @Specification({ + "${app}/watch/server", + }) + public void shouldWatchReadString() throws Exception + { + // GIVEN + String url = "http://localhost:8080/hello.txt"; + Path path = Path.of(new URI(url)); + HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); + watchService.pollSeconds(1); + path.register(watchService); + + // WHEN + k3po.start(); + watchService.take(); + String body1 = Files.readString(path); + k3po.notifyBarrier("CHANGED"); + watchService.take(); + String body2 = Files.readString(path); + watchService.close(); + k3po.finish(); + + // THEN + assertThat(body1, equalTo("Hello World!")); + assertThat(body2, equalTo("Hello Universe!")); + } + + @Test + @Specification({ + "${app}/watch.error.success/server", + }) + public void shouldWatchErrorSuccess() throws Exception + { + // GIVEN + String url = "http://localhost:8080/hello.txt"; + Path path = Path.of(new URI(url)); + HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); + watchService.pollSeconds(1); + path.register(watchService); + + // WHEN + k3po.start(); + WatchKey key1 = watchService.take(); + List> events1 = key1.pollEvents(); + k3po.notifyBarrier("FOUND"); + WatchKey key2 = watchService.take(); + List> events2 = key2.pollEvents(); + watchService.close(); + k3po.finish(); + + // THEN + assertThat(events1.size(), equalTo(1)); + assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); + assertThat(events1.get(0).context(), equalTo(path)); + assertThat(events2.size(), equalTo(1)); + assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); + assertThat(events2.get(0).context(), equalTo(path)); + } + + @Test + @Specification({ + "${app}/watch.error.success/server", + }) + public void shouldWatchErrorSuccessReadString() throws Exception + { + // GIVEN + String url = "http://localhost:8080/hello.txt"; + Path path = Path.of(new URI(url)); + HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); + watchService.pollSeconds(1); + path.register(watchService); + + // WHEN + k3po.start(); + watchService.take(); + String body1 = Files.readString(path); + k3po.notifyBarrier("FOUND"); + watchService.take(); + String body2 = Files.readString(path); + watchService.close(); + k3po.finish(); + + // THEN + assertThat(body1, equalTo("")); + assertThat(body2, equalTo("Hello World!")); + } +} diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/TestHttpFileSystem.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/TestHttpFileSystem.java new file mode 100644 index 0000000000..127c1acc2c --- /dev/null +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/TestHttpFileSystem.java @@ -0,0 +1,122 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +import java.net.URI; +import java.nio.file.Path; + +import org.junit.Test; + +public class TestHttpFileSystem +{ + @Test + public void testHttpPath() throws Exception + { + // GIVEN + String httpUrl = "http://localhost:4242/hello.txt"; + + // WHEN + Path path = Path.of(new URI(httpUrl)); + + // THEN + assertThat(path.getFileSystem().getClass().getSimpleName(), equalTo("HttpFileSystem")); + assertThat(path.getFileSystem().provider().getClass().getSimpleName(), equalTo("HttpFileSystemProvider")); + assertThat(path.getFileSystem().provider().getScheme(), equalTo("http")); + } + + @Test + public void testHttpsPath() throws Exception + { + // GIVEN + String httpsUrl = "https://localhost:4242/hello.txt"; + + // WHEN + Path path = Path.of(new URI(httpsUrl)); + + // THEN + assertThat(path.getFileSystem().getClass().getSimpleName(), equalTo("HttpFileSystem")); + assertThat(path.getFileSystem().provider().getClass().getSimpleName(), equalTo("HttpsFileSystemProvider")); + assertThat(path.getFileSystem().provider().getScheme(), equalTo("https")); + } + + @Test + public void testHttpSiblingPath() throws Exception + { + // GIVEN + String httpUrl = "http://localhost:4242/greeting/hello.txt"; + Path path = Path.of(new URI(httpUrl)); + + // WHEN + Path sibling = path.resolveSibling(Path.of("bye.txt")); + + // THEN + assertThat(sibling.getFileSystem().getClass().getSimpleName(), equalTo("HttpFileSystem")); + assertThat(sibling.getFileSystem().provider().getClass().getSimpleName(), equalTo("HttpFileSystemProvider")); + assertThat(sibling.toString(), equalTo("http://localhost:4242/greeting/bye.txt")); + } + + + @Test + public void testHttpsSiblingPath() throws Exception + { + // GIVEN + String httpUrl = "https://localhost:4242/greeting/hello.txt"; + Path path = Path.of(new URI(httpUrl)); + + // WHEN + Path sibling = path.resolveSibling(Path.of("bye.txt")); + + // THEN + assertThat(sibling.getFileSystem().getClass().getSimpleName(), equalTo("HttpFileSystem")); + assertThat(sibling.getFileSystem().provider().getClass().getSimpleName(), equalTo("HttpsFileSystemProvider")); + assertThat(sibling.toString(), equalTo("https://localhost:4242/greeting/bye.txt")); + } + + @Test + public void testHttpSiblingString() throws Exception + { + // GIVEN + String httpUrl = "http://localhost:4242/greeting/hello.txt"; + Path path = Path.of(new URI(httpUrl)); + + // WHEN + Path sibling = path.resolveSibling("bye.txt"); + + // THEN + assertThat(sibling.getFileSystem().getClass().getSimpleName(), equalTo("HttpFileSystem")); + assertThat(sibling.getFileSystem().provider().getClass().getSimpleName(), equalTo("HttpFileSystemProvider")); + assertThat(sibling.toString(), equalTo("http://localhost:4242/greeting/bye.txt")); + } + + + @Test + public void testHttpsSiblingString() throws Exception + { + // GIVEN + String httpUrl = "https://localhost:4242/greeting/hello.txt"; + Path path = Path.of(new URI(httpUrl)); + + // WHEN + Path sibling = path.resolveSibling("bye.txt"); + + // THEN + assertThat(sibling.getFileSystem().getClass().getSimpleName(), equalTo("HttpFileSystem")); + assertThat(sibling.getFileSystem().provider().getClass().getSimpleName(), equalTo("HttpsFileSystemProvider")); + assertThat(sibling.toString(), equalTo("https://localhost:4242/greeting/bye.txt")); + } +} diff --git a/runtime/pom.xml b/runtime/pom.xml index 7da8fd0543..51ee2464ea 100644 --- a/runtime/pom.xml +++ b/runtime/pom.xml @@ -51,6 +51,7 @@ exporter-otlp exporter-prometheus exporter-stdout + filesystem-http guard-jwt metrics-grpc metrics-http @@ -235,6 +236,11 @@ exporter-stdout ${project.version} + + ${project.groupId} + filesystem-http + ${project.version} + ${project.groupId} guard-jwt diff --git a/specs/filesystem-http.spec/COPYRIGHT b/specs/filesystem-http.spec/COPYRIGHT new file mode 100644 index 0000000000..0cb10b6f62 --- /dev/null +++ b/specs/filesystem-http.spec/COPYRIGHT @@ -0,0 +1,12 @@ +Copyright ${copyrightYears} Aklivity Inc + +Licensed under the Aklivity Community License (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + + https://www.aklivity.io/aklivity-community-license/ + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/specs/filesystem-http.spec/LICENSE b/specs/filesystem-http.spec/LICENSE new file mode 100644 index 0000000000..f6abb6327b --- /dev/null +++ b/specs/filesystem-http.spec/LICENSE @@ -0,0 +1,114 @@ + Aklivity Community License Agreement + Version 1.0 + +This Aklivity Community License Agreement Version 1.0 (the “Agreement”) sets +forth the terms on which Aklivity, Inc. (“Aklivity”) makes available certain +software made available by Aklivity under this Agreement (the “Software”). BY +INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY OF THE SOFTWARE, +YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO +SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. IF YOU ARE RECEIVING +THE SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU +HAVE THE ACTUAL AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS +AGREEMENT ON BEHALF OF SUCH ENTITY. “Licensee” means you, an individual, or +the entity on whose behalf you are receiving the Software. + + 1. LICENSE GRANT AND CONDITIONS. + + 1.1 License. Subject to the terms and conditions of this Agreement, + Aklivity hereby grants to Licensee a non-exclusive, royalty-free, + worldwide, non-transferable, non-sublicenseable license during the term + of this Agreement to: (a) use the Software; (b) prepare modifications and + derivative works of the Software; (c) distribute the Software (including + without limitation in source code or object code form); and (d) reproduce + copies of the Software (the “License”). Licensee is not granted the + right to, and Licensee shall not, exercise the License for an Excluded + Purpose. For purposes of this Agreement, “Excluded Purpose” means making + available any software-as-a-service, platform-as-a-service, + infrastructure-as-a-service or other similar online service that competes + with Aklivity products or services that provide the Software. + + 1.2 Conditions. In consideration of the License, Licensee’s distribution + of the Software is subject to the following conditions: + + (a) Licensee must cause any Software modified by Licensee to carry + prominent notices stating that Licensee modified the Software. + + (b) On each Software copy, Licensee shall reproduce and not remove or + alter all Aklivity or third party copyright or other proprietary + notices contained in the Software, and Licensee must provide the + notice below with each copy. + + “This software is made available by Aklivity, Inc., under the + terms of the Aklivity Community License Agreement, Version 1.0 + located at http://www.Aklivity.io/Aklivity-community-license. BY + INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY OF + THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.” + + 1.3 Licensee Modifications. Licensee may add its own copyright notices + to modifications made by Licensee and may provide additional or different + license terms and conditions for use, reproduction, or distribution of + Licensee’s modifications. While redistributing the Software or + modifications thereof, Licensee may choose to offer, for a fee or free of + charge, support, warranty, indemnity, or other obligations. Licensee, and + not Aklivity, will be responsible for any such obligations. + + 1.4 No Sublicensing. The License does not include the right to + sublicense the Software, however, each recipient to which Licensee + provides the Software may exercise the Licenses so long as such recipient + agrees to the terms and conditions of this Agreement. + + 2. TERM AND TERMINATION. This Agreement will continue unless and until + earlier terminated as set forth herein. If Licensee breaches any of its + conditions or obligations under this Agreement, this Agreement will + terminate automatically and the License will terminate automatically and + permanently. + + 3. INTELLECTUAL PROPERTY. As between the parties, Aklivity will retain all + right, title, and interest in the Software, and all intellectual property + rights therein. Aklivity hereby reserves all rights not expressly granted + to Licensee in this Agreement. Aklivity hereby reserves all rights in its + trademarks and service marks, and no licenses therein are granted in this + Agreement. + + 4. DISCLAIMER. Aklivity HEREBY DISCLAIMS ANY AND ALL WARRANTIES AND + CONDITIONS, EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, AND SPECIFICALLY + DISCLAIMS ANY WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR + PURPOSE, WITH RESPECT TO THE SOFTWARE. + + 5. LIMITATION OF LIABILITY. Aklivity WILL NOT BE LIABLE FOR ANY DAMAGES OF + ANY KIND, INCLUDING BUT NOT LIMITED TO, LOST PROFITS OR ANY CONSEQUENTIAL, + SPECIAL, INCIDENTAL, INDIRECT, OR DIRECT DAMAGES, HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, ARISING OUT OF THIS AGREEMENT. THE FOREGOING SHALL + APPLY TO THE EXTENT PERMITTED BY APPLICABLE LAW. + + 6.GENERAL. + + 6.1 Governing Law. This Agreement will be governed by and interpreted in + accordance with the laws of the state of California, without reference to + its conflict of laws principles. If Licensee is located within the + United States, all disputes arising out of this Agreement are subject to + the exclusive jurisdiction of courts located in Santa Clara County, + California. USA. If Licensee is located outside of the United States, + any dispute, controversy or claim arising out of or relating to this + Agreement will be referred to and finally determined by arbitration in + accordance with the JAMS International Arbitration Rules. The tribunal + will consist of one arbitrator. The place of arbitration will be Palo + Alto, California. The language to be used in the arbitral proceedings + will be English. Judgment upon the award rendered by the arbitrator may + be entered in any court having jurisdiction thereof. + + 6.2 Assignment. Licensee is not authorized to assign its rights under + this Agreement to any third party. Aklivity may freely assign its rights + under this Agreement to any third party. + + 6.3 Other. This Agreement is the entire agreement between the parties + regarding the subject matter hereof. No amendment or modification of + this Agreement will be valid or binding upon the parties unless made in + writing and signed by the duly authorized representatives of both + parties. In the event that any provision, including without limitation + any condition, of this Agreement is held to be unenforceable, this + Agreement and all licenses and rights granted hereunder will immediately + terminate. Waiver by Aklivity of a breach of any provision of this + Agreement or the failure by Aklivity to exercise any right hereunder + will not be construed as a waiver of any subsequent breach of that right + or as a waiver of any other right. \ No newline at end of file diff --git a/specs/filesystem-http.spec/NOTICE b/specs/filesystem-http.spec/NOTICE new file mode 100644 index 0000000000..9024d8926d --- /dev/null +++ b/specs/filesystem-http.spec/NOTICE @@ -0,0 +1,13 @@ +Licensed under the Aklivity Community License (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + + https://www.aklivity.io/aklivity-community-license/ + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +This project includes: + diff --git a/specs/filesystem-http.spec/NOTICE.template b/specs/filesystem-http.spec/NOTICE.template new file mode 100644 index 0000000000..209ca12f74 --- /dev/null +++ b/specs/filesystem-http.spec/NOTICE.template @@ -0,0 +1,13 @@ +Licensed under the Aklivity Community License (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + + https://www.aklivity.io/aklivity-community-license/ + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +This project includes: +#GENERATED_NOTICES# diff --git a/specs/filesystem-http.spec/mvnw b/specs/filesystem-http.spec/mvnw new file mode 100755 index 0000000000..d2f0ea3808 --- /dev/null +++ b/specs/filesystem-http.spec/mvnw @@ -0,0 +1,310 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" + fi + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` + fi + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=`cygpath --path --windows "$javaClass"` + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/specs/filesystem-http.spec/mvnw.cmd b/specs/filesystem-http.spec/mvnw.cmd new file mode 100644 index 0000000000..b26ab24f03 --- /dev/null +++ b/specs/filesystem-http.spec/mvnw.cmd @@ -0,0 +1,182 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" + +FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/specs/filesystem-http.spec/pom.xml b/specs/filesystem-http.spec/pom.xml new file mode 100644 index 0000000000..98bf6f77aa --- /dev/null +++ b/specs/filesystem-http.spec/pom.xml @@ -0,0 +1,123 @@ + + + + 4.0.0 + + io.aklivity.zilla + specs + develop-SNAPSHOT + ../pom.xml + + + filesystem-http.spec + zilla::specs::filesystem-http.spec + + + + Aklivity Community License Agreement + https://www.aklivity.io/aklivity-community-license/ + repo + + + + + 1.00 + 0 + + + + + junit + junit + test + + + io.aklivity.k3po + lang + provided + + + io.aklivity.k3po + control-junit + test + + + org.hamcrest + hamcrest-library + test + + + + + + + src/main/resources + + + src/main/scripts + + + + + + org.jasig.maven + maven-notice-plugin + + + com.mycila + license-maven-plugin + + + maven-checkstyle-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.moditect + moditect-maven-plugin + + + io.aklivity.k3po + k3po-maven-plugin + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + org.jacoco + jacoco-maven-plugin + + + + BUNDLE + + + INSTRUCTION + COVEREDRATIO + ${jacoco.coverage.ratio} + + + CLASS + MISSEDCOUNT + ${jacoco.missed.count} + + + + + + + + + diff --git a/specs/filesystem-http.spec/src/main/moditect/module-info.java b/specs/filesystem-http.spec/src/main/moditect/module-info.java new file mode 100644 index 0000000000..a7fac52d06 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/moditect/module-info.java @@ -0,0 +1,17 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +open module io.aklivity.zilla.specs.filesystem.http +{ +} diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/client.rpt new file mode 100644 index 0000000000..c373f679d7 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/client.rpt @@ -0,0 +1,23 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +connect "http://localhost:8080/notfound.txt" +connected + +write http:method "GET" +write close + +read http:status "404" "Not Found" +read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/server.rpt new file mode 100644 index 0000000000..179a436744 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/server.rpt @@ -0,0 +1,25 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "http://localhost:8080/notfound.txt" +accepted +connected + +read http:method "GET" +read closed + +write http:status "404" "Not Found" +write http:content-length +write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/client.rpt new file mode 100644 index 0000000000..c6134a366a --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/client.rpt @@ -0,0 +1,25 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write close + +read http:status "200" "OK" +read http:header "Etag" "AAAAAAA" +read "Hello World!" +read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/server.rpt new file mode 100644 index 0000000000..fc0194bdbd --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/server.rpt @@ -0,0 +1,27 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "http://localhost:8080/hello.txt" +accepted +connected + +read http:method "GET" +read closed + +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "AAAAAAA" +write "Hello World!" +write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/client.rpt new file mode 100644 index 0000000000..8d530015d2 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/client.rpt @@ -0,0 +1,37 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write close + +read http:status "404" "Not Found" +read closed + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write http:header "If-None-Match" "AAAAAAA" +write http:header "Prefer" "wait=86400" +write close + +read notify FOUND +read http:status "200" "OK" +read http:header "Etag" "AAAAAAA" +read "Hello World!" +read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/server.rpt new file mode 100644 index 0000000000..aa5072487c --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/server.rpt @@ -0,0 +1,38 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "http://localhost:8080/hello.txt" +accepted +connected + +read http:method "GET" +read closed + +write http:status "404" "Not Found" +write http:content-length +write close + +accepted +connected + +read http:method "GET" +read closed + +write await FOUND +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "AAAAAAA" +write "Hello World!" +write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt new file mode 100644 index 0000000000..53048d7d00 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt @@ -0,0 +1,39 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write close + +read http:status "200" "OK" +read http:header "Etag" "AAAAAAA" +read "Hello World!" +read closed + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write http:header "If-None-Match" "AAAAAAA" +write http:header "Prefer" "wait=86400" +write close + +read notify CHANGED +read http:status "200" "OK" +read http:header "Etag" "BBBBBBB" +read "Hello Universe!" +read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/server.rpt new file mode 100644 index 0000000000..19eed58ea1 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/server.rpt @@ -0,0 +1,42 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "http://localhost:8080/hello.txt" +accepted +connected + +read http:method "GET" +read closed + +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "AAAAAAA" +write "Hello World!" +write close + +accepted +connected + +read http:method "GET" +read http:header "If-None-Match" "AAAAAAA" +read http:header "Prefer" "wait=86400" +read closed + +write await CHANGED +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "BBBBBBB" +write "Hello Universe!" +write close diff --git a/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java new file mode 100644 index 0000000000..5e75275f4f --- /dev/null +++ b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java @@ -0,0 +1,74 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.specs.filesystem.http; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.rules.RuleChain.outerRule; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import io.aklivity.k3po.runtime.junit.annotation.Specification; +import io.aklivity.k3po.runtime.junit.rules.K3poRule; + +public class ApplicationIT +{ + private final K3poRule k3po = new K3poRule() + .addScriptRoot("app", "io/aklivity/zilla/specs/filesystem/http/application"); + + private final TestRule timeout = new DisableOnDebug(new Timeout(5, SECONDS)); + + @Rule + public final TestRule chain = outerRule(k3po).around(timeout); + + @Test + @Specification({ + "${app}/success/client", + "${app}/success/server" }) + public void shouldReadString() throws Exception + { + k3po.finish(); + } + + @Test + @Specification({ + "${app}/notfound/client", + "${app}/notfound/server" }) + public void shouldReadStringNotFound() throws Exception + { + k3po.finish(); + } + + @Test + @Specification({ + "${app}/watch/client", + "${app}/watch/server" }) + public void shouldWatch() throws Exception + { + k3po.finish(); + } + + @Test + @Specification({ + "${app}/watch.error.success/client", + "${app}/watch.error.success/server" }) + public void shouldWatchErrorSuccess() throws Exception + { + k3po.finish(); + } +} diff --git a/specs/pom.xml b/specs/pom.xml index 29bb1d9f32..1dfc440066 100644 --- a/specs/pom.xml +++ b/specs/pom.xml @@ -46,6 +46,7 @@ exporter-otlp.spec exporter-prometheus.spec exporter-stdout.spec + filesystem-http.spec metrics-stream.spec metrics-http.spec metrics-grpc.spec From ab94dac7216dc80810dec1db62b75cac57275f2b Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Tue, 18 Jun 2024 19:56:22 +0200 Subject: [PATCH 17/43] WIP --- runtime/catalog-apicurio/pom.xml | 2 +- runtime/engine/pom.xml | 5 +++++ .../io/aklivity/zilla/runtime/engine/Engine.java | 2 ++ .../engine/internal/watcher/ConfigWatcherTask.java | 1 + .../zilla/runtime/engine/internal/EngineTest.java | 1 + .../runtime/guard/jwt/internal/JwtGuardHandler.java | 12 +----------- .../filesystem/internal/FileSystemVaultTest.java | 13 ++----------- 7 files changed, 13 insertions(+), 23 deletions(-) diff --git a/runtime/catalog-apicurio/pom.xml b/runtime/catalog-apicurio/pom.xml index c944604e7e..4571e1815b 100644 --- a/runtime/catalog-apicurio/pom.xml +++ b/runtime/catalog-apicurio/pom.xml @@ -22,7 +22,7 @@ - 0.93 + 0.90 0 diff --git a/runtime/engine/pom.xml b/runtime/engine/pom.xml index d39ddb8a76..ee446c0716 100644 --- a/runtime/engine/pom.xml +++ b/runtime/engine/pom.xml @@ -77,6 +77,11 @@ jackson-dataformat-yaml 2.16.1 + + ${project.groupId} + filesystem-http + test + org.jmock jmock-junit4 diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index 84d40aba16..46aba6b3ac 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -291,7 +291,9 @@ private String readPath( String result; try { + System.out.println("ENG readPath path " + path); // TODO: Ati result = Files.readString(path); + System.out.println("ENG readPath result [" + result + "]"); // TODO: Ati } catch (Exception ex) { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java index 0d66784a8e..bfd6436830 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java @@ -117,6 +117,7 @@ public CompletableFuture watchConfig( WatchedItem watchedItem = new WatchedItem(configPath, watchService); watchedItem.register(); watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); + System.out.println("CWT watchConfig readPath " + configPath); // TODO: Ati String configText = readPath.apply(configPath); watchedItem.setHash(computeHash(configText)); } diff --git a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/EngineTest.java b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/EngineTest.java index 4ae65aba4e..ba13dd0d77 100644 --- a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/EngineTest.java +++ b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/EngineTest.java @@ -284,6 +284,7 @@ public static final class TestEngineExt implements EngineExtSpi public void onRegistered( EngineExtContext context) { + System.out.println("TestEngineExt onRegistered"); // TODO: Ati if (registerLatch != null) { registerLatch.countDown(); diff --git a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java index 69f5940185..4223e6454a 100644 --- a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java +++ b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java @@ -17,7 +17,6 @@ import static org.agrona.LangUtil.rethrowUnchecked; import java.net.URI; -import java.net.URISyntaxException; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; @@ -83,16 +82,7 @@ public JwtGuardHandler( Jsonb jsonb = JsonbBuilder.newBuilder() .withConfig(config) .build(); - - String keysText = null; - try - { - keysText = readPath.apply(Path.of(new URI(options.keysURL.get()))); - } - catch (URISyntaxException ex) - { - rethrowUnchecked(ex); - } + String keysText = readPath.apply(Path.of(URI.create(options.keysURL.get()))); JwtKeySetConfig jwks = jsonb.fromJson(keysText, JwtKeySetConfig.class); keysConfig = jwks.keys; } diff --git a/runtime/vault-filesystem/src/test/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultTest.java b/runtime/vault-filesystem/src/test/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultTest.java index 945e2b535a..2f16fdcc88 100644 --- a/runtime/vault-filesystem/src/test/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultTest.java +++ b/runtime/vault-filesystem/src/test/java/io/aklivity/zilla/runtime/vault/filesystem/internal/FileSystemVaultTest.java @@ -15,12 +15,12 @@ */ package io.aklivity.zilla.runtime.vault.filesystem.internal; -import static org.agrona.LangUtil.rethrowUnchecked; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import java.net.URI; import java.net.URL; import java.nio.file.Path; import java.security.KeyStore.PrivateKeyEntry; @@ -89,15 +89,6 @@ public static Path getResourcePath( { URL url = FileSystemVaultTest.class.getResource(resource); assert url != null; - Path path = null; - try - { - path = Path.of(url.toURI()); - } - catch (Exception ex) - { - rethrowUnchecked(ex); - } - return path; + return Path.of(URI.create(url.toString())); } } From 22a2d03fc4a606fa38c5e6ced34771565d4745e5 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Tue, 18 Jun 2024 21:57:17 +0200 Subject: [PATCH 18/43] fix --- runtime/catalog-apicurio/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/catalog-apicurio/pom.xml b/runtime/catalog-apicurio/pom.xml index 4571e1815b..c944604e7e 100644 --- a/runtime/catalog-apicurio/pom.xml +++ b/runtime/catalog-apicurio/pom.xml @@ -22,7 +22,7 @@ - 0.90 + 0.93 0 From e4cf40704ccf484588b2cc2789257c48e030c992 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Tue, 18 Jun 2024 22:12:17 +0200 Subject: [PATCH 19/43] fix --- .../binding/sse/config/SseOptionsConfig.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/runtime/binding-sse/src/main/java/io/aklivity/zilla/runtime/binding/sse/config/SseOptionsConfig.java b/runtime/binding-sse/src/main/java/io/aklivity/zilla/runtime/binding/sse/config/SseOptionsConfig.java index 427db49e49..be1bad73fb 100644 --- a/runtime/binding-sse/src/main/java/io/aklivity/zilla/runtime/binding/sse/config/SseOptionsConfig.java +++ b/runtime/binding-sse/src/main/java/io/aklivity/zilla/runtime/binding/sse/config/SseOptionsConfig.java @@ -23,6 +23,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import io.aklivity.zilla.runtime.engine.config.ModelConfig; import io.aklivity.zilla.runtime.engine.config.OptionsConfig; public final class SseOptionsConfig extends OptionsConfig @@ -46,14 +47,20 @@ public static SseOptionsConfigBuilder builder( int retry, List requests) { - super(requests != null && !requests.isEmpty() + super(resolveModels(requests), List.of()); + this.retry = retry; + this.requests = requests; + } + + private static List resolveModels( + List requests) + { + return requests != null && !requests.isEmpty() ? requests.stream() .flatMap(path -> Stream.of(path.content) .filter(Objects::nonNull)) .collect(Collectors.toList()) - : emptyList()); - this.retry = retry; - this.requests = requests; + : emptyList(); } } From df172d81bb944be0b4a13821b3607c1bfcc603f1 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Wed, 19 Jun 2024 08:51:37 +0200 Subject: [PATCH 20/43] fix EngineConfiguration configPath --- .../runtime/engine/EngineConfiguration.java | 68 +++++++++++-------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java index 2786d63a16..60f772c1c5 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java @@ -18,7 +18,6 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; -import static org.agrona.LangUtil.rethrowUnchecked; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; @@ -45,6 +44,7 @@ public class EngineConfiguration extends Configuration public static final boolean DEBUG_BUDGETS = Boolean.getBoolean("zilla.engine.debug.budgets"); public static final PropertyDef ENGINE_CONFIG_URL; + public static final PropertyDef ENGINE_CONFIG_PATH; public static final IntPropertyDef ENGINE_CONFIG_POLL_INTERVAL_SECONDS; public static final PropertyDef ENGINE_NAME; public static final PropertyDef ENGINE_DIRECTORY; @@ -82,6 +82,8 @@ public class EngineConfiguration extends Configuration { final ConfigurationDef config = new ConfigurationDef("zilla.engine"); ENGINE_CONFIG_URL = config.property(URL.class, "config.url", EngineConfiguration::configURL, "file:zilla.yaml"); + ENGINE_CONFIG_PATH = config.property(Path.class, "config.path", EngineConfiguration::decodeConfigPath, + EngineConfiguration::defaultConfigPath); ENGINE_CONFIG_POLL_INTERVAL_SECONDS = config.property("config.poll.interval.seconds", 60); ENGINE_NAME = config.property("name", EngineConfiguration::defaultName); ENGINE_DIRECTORY = config.property("directory", EngineConfiguration::defaultDirectory); @@ -149,30 +151,7 @@ public URL configURL() public Path configPath() { - Path configPath = null; - try - { - URI configUri = configURL().toURI(); - if ("file".equals(configUri.getScheme()) && !Path.of(configUri.getSchemeSpecificPart()).isAbsolute()) - { - // this works for relative file e.g. file:zilla.yaml - Path basePath = Path.of("").toAbsolutePath(); - configPath = basePath.resolve(configUri.getSchemeSpecificPart()); - } - else - { - // this works for absolute file e.g. file:/path/dir/zilla.yaml - // this works for http e.g. http://localhost:7115/zilla.yaml - // this works for jar e.g. jar:file:/path/engine.jar!/package/zilla.yaml - // (the jar filesystem is opened and closed by EngineRule) - configPath = Path.of(configUri); - } - } - catch (Exception ex) - { - rethrowUnchecked(ex); - } - return configPath; + return ENGINE_CONFIG_PATH.get(this); } public int configPollIntervalSeconds() @@ -343,8 +322,7 @@ private static int defaultBudgetsBufferCapacity( private static URL configURL( Configuration config, - String url - ) + String url) { URL configURL = null; try @@ -445,4 +423,40 @@ private static HostResolver defaultHostResolver( return addresses; }; } + + private static Path decodeConfigPath( + Configuration config, + String value) + { + return resolveConfigPath(value); + } + + private static Path defaultConfigPath( + Configuration config) + { + URL url = ((EngineConfiguration) config).configURL(); + return resolveConfigPath(url.toString()); + } + + private static Path resolveConfigPath( + String config) + { + Path configPath; + URI configUri = URI.create(config); + if ("file".equals(configUri.getScheme()) && !Path.of(configUri.getSchemeSpecificPart()).isAbsolute()) + { + // this works for relative file e.g. file:zilla.yaml + Path basePath = Path.of("").toAbsolutePath(); + configPath = basePath.resolve(configUri.getSchemeSpecificPart()); + } + else + { + // this works for absolute file e.g. file:/path/dir/zilla.yaml + // this works for http e.g. http://localhost:7115/zilla.yaml + // this works for jar e.g. jar:file:/path/engine.jar!/package/zilla.yaml + // (the jar filesystem is opened and closed by EngineRule) + configPath = Path.of(configUri); + } + return configPath; + } } From bef04ad6f8648036b4cabbe65438c8c68cffb4c9 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Wed, 19 Jun 2024 13:42:04 +0200 Subject: [PATCH 21/43] WIP fix --- .../internal/watcher/ConfigWatcherTask.java | 28 +++++-------- .../internal/watcher/EngineConfigWatcher.java | 2 +- .../filesystem/http/HttpWatchService.java | 39 +++++++++++-------- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java index bfd6436830..b9c7aac38b 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java @@ -25,7 +25,6 @@ import java.nio.file.WatchService; import java.util.IdentityHashMap; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import java.util.function.Function; @@ -77,6 +76,7 @@ public Void call() try { final WatchKey key = watchService.take(); + System.out.println("CWT call key " + key); // TODO: Ati WatchedItem watchedItem = watchedItems.get(key); @@ -109,32 +109,24 @@ public Void call() return null; } - public CompletableFuture watchConfig( + public void watchConfig( Path configPath) { - if (!"jar".equals(configPath.getFileSystem().provider().getScheme())) + String configText; + if ("jar".equals(configPath.getFileSystem().provider().getScheme())) + { + configText = readPath.apply(configPath); + } + else { WatchedItem watchedItem = new WatchedItem(configPath, watchService); watchedItem.register(); watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); System.out.println("CWT watchConfig readPath " + configPath); // TODO: Ati - String configText = readPath.apply(configPath); + configText = readPath.apply(configPath); watchedItem.setHash(computeHash(configText)); } - String configText = readPath.apply(configPath); - - CompletableFuture configFuture; - try - { - EngineConfig config = configChangeListener.apply(configText); - configFuture = CompletableFuture.completedFuture(config); - } - catch (Exception ex) - { - configFuture = CompletableFuture.failedFuture(ex); - } - - return configFuture; + configChangeListener.apply(configText); } @Override diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java index 968fa36708..1bc849a429 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java @@ -55,8 +55,8 @@ public EngineConfigWatcher( public void startWatchingConfig() throws Exception { + configWatcherTask.watchConfig(configPath); configWatcherTask.submit(); - configWatcherTask.watchConfig(configPath).get(); } public void addResources( diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java index b427b518e8..52321275e2 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -102,7 +102,15 @@ public Void call() throws Exception } String etag = etags.getOrDefault(path, ""); System.out.println("HWS call take path " + path + " etag [" + etag + "]"); // TODO: Ati - sendAsync(path, etag); + HttpResponse response = send(path, etag); + if (response == null) + { + scheduleRequest(path, pollSeconds); + } + else + { + handleChange(response); + } } return null; } @@ -189,7 +197,7 @@ WatchKey register( return watchKey; } - private void sendAsync( + private HttpResponse send( Path path, String etag) { @@ -202,20 +210,19 @@ private void sendAsync( requestBuilder = requestBuilder.headers("If-None-Match", etag, "Prefer", "wait=86400"); } - System.out.println("HWS sendAsync path " + path + " etag " + etag); // TODO: Ati - CompletableFuture future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) - .thenAccept(this::handleChange) - .exceptionally(ex -> handleException(ex, path)); - futures.put(path, future); - } - - private Void handleException( - Throwable throwable, - Path path) - { - System.out.println("HWS handleException " + throwable.getMessage()); // TODO: Ati - scheduleRequest(path, pollSeconds); - return null; + HttpResponse response; + try + { + response = HTTP_CLIENT.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()); + //System.out.println("HWS send response " + response); // TODO: Ati + //System.out.println("HWS send response.body " + new String(response.body())); // TODO: Ati + } + catch (Exception e) + { + response = null; + //System.out.println("HWS send exception " + e); // TODO: Ati + } + return response; } private void handleChange( From 3177c8043a94c9fbf672c3e6fde28592001bcfb9 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 10:29:53 +0200 Subject: [PATCH 22/43] WIP hfs 1 --- ...va => AbstractHttpFileSystemProvider.java} | 27 +++-- .../filesystem/http/HttpFileSystem.java | 31 ++--- .../http/HttpFileSystemProvider.java | 6 +- .../runtime/filesystem/http/HttpPath.java | 27 +++-- .../filesystem/http/HttpWatchService.java | 107 +++++++++++++----- .../http/HttpsFileSystemProvider.java | 6 +- 6 files changed, 137 insertions(+), 67 deletions(-) rename runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/{HttpBaseFileSystemProvider.java => AbstractHttpFileSystemProvider.java} (90%) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpBaseFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java similarity index 90% rename from runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpBaseFileSystemProvider.java rename to runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java index a12b5f6d97..9e674b1310 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpBaseFileSystemProvider.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java @@ -49,7 +49,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -public abstract class HttpBaseFileSystemProvider extends FileSystemProvider +public abstract class AbstractHttpFileSystemProvider extends FileSystemProvider { private static final Duration TIMEOUT = Duration.ofSeconds(5); private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() @@ -119,6 +119,7 @@ public InputStream newInputStream( Path path, OpenOption... options) { + // TODO: Ati - chk if path is http/s -> cast it to HttpPath return new ByteArrayInputStream(resolveBody(path)); } @@ -155,6 +156,7 @@ public SeekableByteChannel newByteChannel( Set options, FileAttribute... attrs) { + // TODO: Ati - chk if path is http/s -> cast it to HttpPath return new ReadOnlyByteArrayChannel(resolveBody(path)); } @@ -314,19 +316,30 @@ private void checkUri( } } + // TODO: Ati - this should be the new resolveBody + /*private byte[] resolveBody( + Path path) + //HttpPath path) + { + // TODO: Ati - this should delegate the call to HttpPath path.resolveBody() + return readBody(path); + }*/ + + // TODO: Ati - remove this private byte[] resolveBody( Path path) { byte[] body; - HttpFileSystem fileSystem = fileSystems.get(path.toUri()); + //HttpFileSystem fileSystem = fileSystems.get(path.toUri()); + HttpFileSystem fileSystem = (HttpFileSystem) path.getFileSystem(); if (fileSystem != null && fileSystem.body() != null) { - System.out.println("HBFSP resolveBody fs.body"); // TODO: Ati + System.out.println("AHFSP resolveBody fs.body"); // TODO: Ati body = fileSystem.body(); } else { - System.out.println("HBFSP resolveBody readBody"); // TODO: Ati + System.out.println("AHFSP resolveBody readBody"); // TODO: Ati body = readBody(path); } return body; @@ -343,9 +356,9 @@ private static byte[] readBody( .uri(path.toUri()) .timeout(TIMEOUT) .build(); - System.out.println("HBFSP readBody path " + path + " request " + request); // TODO: Ati + System.out.println("AHFSP readBody path " + path + " request " + request); // TODO: Ati HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofByteArray()); - System.out.println("HBFSP readBody response.body " + new String(response.body())); // TODO: Ati + System.out.println("AHFSP readBody response.body " + new String(response.body())); // TODO: Ati if (response.statusCode() == HTTP_OK) { body = response.body(); @@ -353,7 +366,7 @@ private static byte[] readBody( } catch (Exception ex) { - System.out.println("HBFSP readBody exception " + ex); // TODO: Ati + System.out.println("AHFSP readBody exception " + ex); // TODO: Ati rethrowUnchecked(ex); } return body; diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java index 6982ef2372..4caae7bb8d 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java @@ -23,27 +23,26 @@ import java.nio.file.FileSystem; import java.nio.file.Path; import java.nio.file.PathMatcher; -import java.nio.file.WatchService; import java.nio.file.attribute.UserPrincipalLookupService; import java.nio.file.spi.FileSystemProvider; import java.util.Set; -public class HttpFileSystem extends FileSystem +public final class HttpFileSystem extends FileSystem { public static final String SEPARATOR = "/"; - private final HttpBaseFileSystemProvider provider; - private final URI uri; + private final AbstractHttpFileSystemProvider provider; + private final URI root; - private byte[] body; + private byte[] body; // TODO: Ati - body should be moved from HFS to HttpPath HttpFileSystem( - HttpBaseFileSystemProvider provider, - URI uri) + AbstractHttpFileSystemProvider provider, + URI root) { this.provider = provider; - this.uri = uri; - this.body = null; + this.root = root; + this.body = null; // TODO: Ati - body should be moved from HFS to HttpPath } @Override @@ -55,7 +54,7 @@ public FileSystemProvider provider() @Override public void close() { - provider.closeFileSystem(uri); + provider.closeFileSystem(root); } @Override @@ -129,21 +128,27 @@ public UserPrincipalLookupService getUserPrincipalLookupService() } @Override - public WatchService newWatchService() + //public WatchService newWatchService() + public HttpWatchService newWatchService() { - return new HttpWatchService(this); + //return new HttpWatchService(this); + HttpWatchService service = new HttpWatchService(this); + service.start(); + return service; } URI baseUri() { - return this.uri; + return this.root; } + // TODO: Ati - body should be moved from HFS to HttpPath byte[] body() { return body; } + // TODO: Ati - body should be moved from HFS to HttpPath void body( byte[] body) { diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java index 36b28f3e50..cbefedb60d 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java @@ -14,12 +14,12 @@ */ package io.aklivity.zilla.runtime.filesystem.http; -public class HttpFileSystemProvider extends HttpBaseFileSystemProvider +public final class HttpFileSystemProvider extends AbstractHttpFileSystemProvider { - public static final String SCHEME = "http"; + private static final String SCHEME = "http"; @Override - public final String getScheme() + public String getScheme() { return SCHEME; } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index f626882c49..0498dfebe2 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -14,7 +14,6 @@ */ package io.aklivity.zilla.runtime.filesystem.http; -import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; import static java.util.Objects.requireNonNull; import static org.agrona.LangUtil.rethrowUnchecked; @@ -23,7 +22,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.nio.file.FileSystem; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.ProviderMismatchException; @@ -37,10 +35,12 @@ public class HttpPath implements Path { private final HttpFileSystem fs; private final URL url; + //private final URI location; HttpPath( HttpFileSystem fs, URL url) + //URI location) { if (!fs.provider().getScheme().equals(url.getProtocol())) { @@ -50,8 +50,15 @@ public class HttpPath implements Path this.url = url; } + HttpPath() + { + this.fs = null; + this.url = null; + } + @Override - public FileSystem getFileSystem() + //public FileSystem getFileSystem() + public HttpFileSystem getFileSystem() { return fs; } @@ -154,6 +161,7 @@ public Path resolveSibling( Path other) { return resolveSibling(other.toString()); + //return fs.resolveSibling(this, other); } @Override @@ -164,6 +172,8 @@ public Path resolveSibling( try { path = new HttpPath(fs, new URL(fs.baseUri().toURL(), other)); + //path = new HttpPath(fs, fs.baseUri().resolve(URI.create(other))); // TODO: Ati - test + // return fs.resolveSibling(this, other); // TODO: Ati - this should be here } catch (Exception ex) { @@ -225,17 +235,6 @@ public WatchKey register( { throw new ProviderMismatchException(); } - for (WatchEvent.Kind event : events) - { - if (!event.equals(ENTRY_MODIFY)) - { - throw new IllegalArgumentException("Only ENTRY_MODIFY event kind is supported"); - } - } - if (modifiers.length > 0) - { - throw new IllegalArgumentException("Modifiers are not supported"); - } return ((HttpWatchService) watcher).register(this, events, modifiers); } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java index 52321275e2..cabc908538 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -19,7 +19,6 @@ import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; import static org.agrona.LangUtil.rethrowUnchecked; -import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; @@ -28,7 +27,6 @@ import java.nio.file.WatchEvent; import java.nio.file.WatchKey; import java.nio.file.WatchService; -import java.nio.file.Watchable; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Duration; @@ -55,18 +53,21 @@ public class HttpWatchService implements WatchService, Callable .version(HTTP_2) .followRedirects(NORMAL) .build(); - private static final Path CLOSE_PATH = Path.of(URI.create("http://localhost:12345")); + //private static final Path CLOSE_PATH = Path.of(URI.create("http://localhost:12345")); + private static final HttpPath CLOSE_PATH = new HttpPath(); private static final byte[] EMPTY_BODY = new byte[0]; private final WatchKey closeKey = new HttpWatchKey(CLOSE_PATH); - private final HttpFileSystem fileSystem; + private final HttpFileSystem fileSystem; // TODO: Ati - remove this private final ScheduledExecutorService executor; private final LinkedBlockingQueue pendingKeys; - private final BlockingQueue pathQueue; - private final Map watchKeys; - private final Map etags; - private final Map hashes; + //private final BlockingQueue pathQueue; + private final BlockingQueue pathQueue; + //private final Map watchKeys; + private final Map watchKeys; + private final Map etags; // TODO: Ati + private final Map hashes; // TODO: Ati private final Map> futures; private final MessageDigest md5; @@ -74,9 +75,9 @@ public class HttpWatchService implements WatchService, Callable private volatile boolean closed; public HttpWatchService( - HttpFileSystem fileSystem) + HttpFileSystem fileSystem) // TODO: Ati - remove this { - this.fileSystem = fileSystem; + this.fileSystem = fileSystem; // TODO: Ati - remove this this.executor = Executors.newScheduledThreadPool(2); this.pendingKeys = new LinkedBlockingQueue<>(); this.watchKeys = new ConcurrentHashMap<>(); @@ -85,8 +86,15 @@ public HttpWatchService( this.hashes = new ConcurrentHashMap<>(); this.futures = new ConcurrentHashMap<>(); this.md5 = initMessageDigest("MD5"); - this.pollSeconds = 30; + //this.pollSeconds = 30; + // TODO: Ati - do Configuration for this project, too + this.pollSeconds = 2; this.closed = false; + //executor.submit(this); + } + + public void start() + { executor.submit(this); } @@ -95,7 +103,7 @@ public Void call() throws Exception { while (true) { - Path path = pathQueue.take(); + HttpPath path = pathQueue.take(); if (path == CLOSE_PATH) { break; @@ -109,17 +117,34 @@ public Void call() throws Exception } else { - handleChange(response); + handleChange(response); // TODO: Ati - this should receive path -> body should be stored in path } } return null; } + //@Override + /*public Void call_NEW() throws Exception + { + for (HttpWatchKey watchKey : watchKeys) // TODO: Ati watchKeys should be a List + { + HttpPath path = watchKey.watchable(); + if (path.isDone()) + { + HttpResponse response = path.poll(); + handleChange(response); // this throws Exception if error + path.watchBody(); // scheduleRequest can be deleted + // how to deal with poll interval + } + } + }*/ + @Override public void close() { + System.out.println("HWS close"); // TODO: Ati closed = true; - fileSystem.body(null); + fileSystem.body(null); // TODO: Ati - body should be moved from HFS to HttpPath pendingKeys.clear(); pendingKeys.offer(closeKey); futures.values().forEach(future -> future.cancel(true)); @@ -186,14 +211,29 @@ private void enqueueKey( pendingKeys.offer(key); } - WatchKey register( + //WatchKey register( + HttpWatchKey register( final HttpPath path, WatchEvent.Kind[] events, WatchEvent.Modifier... modifiers) { + for (WatchEvent.Kind event : events) + { + if (!event.equals(ENTRY_MODIFY)) + { + throw new IllegalArgumentException("Only ENTRY_MODIFY event kind is supported"); + } + } + if (modifiers.length > 0) + { + throw new IllegalArgumentException("Modifiers are not supported"); + } System.out.printf("HWS register path: %s\n", path); // TODO: Ati - WatchKey watchKey = watchKeys.computeIfAbsent(path, i -> new HttpWatchKey(path)); - pathQueue.offer(path); + //WatchKey watchKey = watchKeys.computeIfAbsent(path, i -> new HttpWatchKey(path)); + HttpWatchKey watchKey = watchKeys.computeIfAbsent(path, i -> new HttpWatchKey(path)); + // watchKeys.add(new HttpWatchKey(path)) // TODO: Ati - watchKeys should be a List + //pathQueue.offer(path); + pathQueue.add(path); return watchKey; } @@ -220,7 +260,7 @@ private HttpResponse send( catch (Exception e) { response = null; - //System.out.println("HWS send exception " + e); // TODO: Ati + System.out.println("HWS send exception " + e); // TODO: Ati } return response; } @@ -231,24 +271,24 @@ private void handleChange( System.out.println("HWS handleChange response: " + response); // TODO: Ati System.out.println("HWS handleChange response.headers: " + response.headers()); // TODO: Ati System.out.println("HWS handleChange response.body: " + new String(response.body())); // TODO: Ati - Path path = Path.of(response.request().uri()); + HttpPath path = (HttpPath) Path.of(response.request().uri()); int statusCode = response.statusCode(); int pollSeconds = 0; if (statusCode == 404) { - fileSystem.body(EMPTY_BODY); + fileSystem.body(EMPTY_BODY); // TODO: Ati - body should be moved from HFS to HttpPath addEvent(path); pollSeconds = this.pollSeconds; } else if (statusCode >= 500 && statusCode <= 599) { - fileSystem.body(EMPTY_BODY); + fileSystem.body(null); // TODO: Ati - body should be moved from HFS to HttpPath pollSeconds = this.pollSeconds; } else { byte[] body = response.body(); - fileSystem.body(body); + fileSystem.body(body); // TODO: Ati - body should be moved from HFS to HttpPath Optional etagOptional = response.headers().firstValue("Etag"); if (etagOptional.isPresent()) { @@ -285,7 +325,8 @@ private void addEvent( Path path) { System.out.println("HWS addEvent path " + path); // TODO: Ati - HttpWatchKey key = (HttpWatchKey) watchKeys.get(path); + //HttpWatchKey key = (HttpWatchKey) watchKeys.get(path); + HttpWatchKey key = watchKeys.get(path); if (key != null) { key.addEvent(ENTRY_MODIFY, path); @@ -294,7 +335,8 @@ private void addEvent( } private void scheduleRequest( - Path path, + //Path path, + HttpPath path, int pollSeconds) { if (pollSeconds == 0) @@ -309,6 +351,14 @@ private void scheduleRequest( } } + /*private void scheduleRequest___NEW( + //Path path, + HttpPath path, + int pollSeconds) + { + + }*/ + private byte[] computeHash( byte[] body) { @@ -332,14 +382,16 @@ private MessageDigest initMessageDigest( private final class HttpWatchKey implements WatchKey { - private final Path path; + //private final Path path; + private final HttpPath path; private List> events = Collections.synchronizedList(new LinkedList<>()); private volatile boolean valid; private HttpWatchKey( - Path path) + HttpPath path) + //Path path) { this.path = path; this.valid = true; @@ -373,7 +425,8 @@ public void cancel() } @Override - public Watchable watchable() + //public Watchable watchable() + public HttpPath watchable() { return path; } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpsFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpsFileSystemProvider.java index 8500c96638..67c8419a98 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpsFileSystemProvider.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpsFileSystemProvider.java @@ -14,12 +14,12 @@ */ package io.aklivity.zilla.runtime.filesystem.http; -public class HttpsFileSystemProvider extends HttpBaseFileSystemProvider +public final class HttpsFileSystemProvider extends AbstractHttpFileSystemProvider { - public static final String SCHEME = "https"; + private static final String SCHEME = "https"; @Override - public final String getScheme() + public String getScheme() { return SCHEME; } From 2a217f835fa3e2d4ba9d8cc407d03318f07e1718 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 11:34:00 +0200 Subject: [PATCH 23/43] WIP hfs uri --- .../filesystem/http/HttpFileSystem.java | 3 +- .../runtime/filesystem/http/HttpPath.java | 39 ++++++------------- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java index 4caae7bb8d..e7cf1dd2c7 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java @@ -18,7 +18,6 @@ import static org.agrona.LangUtil.rethrowUnchecked; import java.net.URI; -import java.net.URL; import java.nio.file.FileStore; import java.nio.file.FileSystem; import java.nio.file.Path; @@ -105,7 +104,7 @@ public Path getPath( Path result = null; try { - result = new HttpPath(this, new URL(path)); + result = new HttpPath(this, URI.create(path)); } catch (Exception ex) { diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index 0498dfebe2..6dbbc3ac3b 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -20,8 +20,6 @@ import java.io.File; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.ProviderMismatchException; @@ -34,30 +32,27 @@ public class HttpPath implements Path { private final HttpFileSystem fs; - private final URL url; - //private final URI location; + private final URI location; HttpPath( HttpFileSystem fs, - URL url) - //URI location) + URI location) { - if (!fs.provider().getScheme().equals(url.getProtocol())) + if (!fs.provider().getScheme().equals(location.getScheme())) { - throw new IllegalArgumentException(String.format("invalid protocol: %s", url.getProtocol())); + throw new IllegalArgumentException(String.format("invalid protocol: %s", location.getScheme())); } this.fs = fs; - this.url = url; + this.location = location; } HttpPath() { this.fs = null; - this.url = null; + this.location = null; } @Override - //public FileSystem getFileSystem() public HttpFileSystem getFileSystem() { return fs; @@ -161,7 +156,7 @@ public Path resolveSibling( Path other) { return resolveSibling(other.toString()); - //return fs.resolveSibling(this, other); + //return fs.resolveSibling(this, other); // TODO: Ati - this should be here } @Override @@ -171,8 +166,7 @@ public Path resolveSibling( Path path = null; try { - path = new HttpPath(fs, new URL(fs.baseUri().toURL(), other)); - //path = new HttpPath(fs, fs.baseUri().resolve(URI.create(other))); // TODO: Ati - test + path = new HttpPath(fs, fs.baseUri().resolve(URI.create(other))); // return fs.resolveSibling(this, other); // TODO: Ati - this should be here } catch (Exception ex) @@ -192,16 +186,7 @@ public Path relativize( @Override public URI toUri() { - URI result = null; - try - { - result = url.toURI(); - } - catch (URISyntaxException ex) - { - rethrowUnchecked(ex); - } - return result; + return location; } @Override @@ -262,7 +247,7 @@ public int compareTo( @Override public String toString() { - return url.toString(); + return location.toString(); } @Override @@ -279,12 +264,12 @@ public boolean equals( } HttpPath path = (HttpPath) o; - return Objects.equals(url, path.url); + return Objects.equals(location, path.location); } @Override public int hashCode() { - return Objects.hashCode(url); + return Objects.hashCode(location); } } From 4f5737ab1e1a1b66fcb9d84931881d611e9f4a1d Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 12:04:32 +0200 Subject: [PATCH 24/43] WIP hfs resolveSibling --- .../http/AbstractHttpFileSystemProvider.java | 2 +- .../filesystem/http/HttpFileSystem.java | 7 ++-- .../runtime/filesystem/http/HttpPath.java | 16 ++------- .../filesystem/http/HttpWatchService.java | 2 +- .../filesystem/http/TestHttpFileSystem.java | 33 ------------------- 5 files changed, 7 insertions(+), 53 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java index 9e674b1310..fc99c5e86b 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java @@ -358,7 +358,7 @@ private static byte[] readBody( .build(); System.out.println("AHFSP readBody path " + path + " request " + request); // TODO: Ati HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofByteArray()); - System.out.println("AHFSP readBody response.body " + new String(response.body())); // TODO: Ati + //System.out.println("AHFSP readBody response.body " + new String(response.body())); // TODO: Ati if (response.statusCode() == HTTP_OK) { body = response.body(); diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java index e7cf1dd2c7..0ded535a8f 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java @@ -127,18 +127,17 @@ public UserPrincipalLookupService getUserPrincipalLookupService() } @Override - //public WatchService newWatchService() public HttpWatchService newWatchService() { - //return new HttpWatchService(this); HttpWatchService service = new HttpWatchService(this); service.start(); return service; } - URI baseUri() + HttpPath resolveSibling( + String other) { - return this.root; + return new HttpPath(this, root.resolve(URI.create(other))); } // TODO: Ati - body should be moved from HFS to HttpPath diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index 6dbbc3ac3b..051b960259 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -15,7 +15,6 @@ package io.aklivity.zilla.runtime.filesystem.http; import static java.util.Objects.requireNonNull; -import static org.agrona.LangUtil.rethrowUnchecked; import java.io.File; import java.io.IOException; @@ -155,25 +154,14 @@ public Path resolve( public Path resolveSibling( Path other) { - return resolveSibling(other.toString()); - //return fs.resolveSibling(this, other); // TODO: Ati - this should be here + throw new UnsupportedOperationException("not implemented"); } @Override public Path resolveSibling( String other) { - Path path = null; - try - { - path = new HttpPath(fs, fs.baseUri().resolve(URI.create(other))); - // return fs.resolveSibling(this, other); // TODO: Ati - this should be here - } - catch (Exception ex) - { - rethrowUnchecked(ex); - } - return path; + return fs.resolveSibling(other); } @Override diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java index cabc908538..456c669cd4 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -270,7 +270,7 @@ private void handleChange( { System.out.println("HWS handleChange response: " + response); // TODO: Ati System.out.println("HWS handleChange response.headers: " + response.headers()); // TODO: Ati - System.out.println("HWS handleChange response.body: " + new String(response.body())); // TODO: Ati + //System.out.println("HWS handleChange response.body: " + new String(response.body())); // TODO: Ati HttpPath path = (HttpPath) Path.of(response.request().uri()); int statusCode = response.statusCode(); int pollSeconds = 0; diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/TestHttpFileSystem.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/TestHttpFileSystem.java index 127c1acc2c..1b0076c097 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/TestHttpFileSystem.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/TestHttpFileSystem.java @@ -54,39 +54,6 @@ public void testHttpsPath() throws Exception assertThat(path.getFileSystem().provider().getScheme(), equalTo("https")); } - @Test - public void testHttpSiblingPath() throws Exception - { - // GIVEN - String httpUrl = "http://localhost:4242/greeting/hello.txt"; - Path path = Path.of(new URI(httpUrl)); - - // WHEN - Path sibling = path.resolveSibling(Path.of("bye.txt")); - - // THEN - assertThat(sibling.getFileSystem().getClass().getSimpleName(), equalTo("HttpFileSystem")); - assertThat(sibling.getFileSystem().provider().getClass().getSimpleName(), equalTo("HttpFileSystemProvider")); - assertThat(sibling.toString(), equalTo("http://localhost:4242/greeting/bye.txt")); - } - - - @Test - public void testHttpsSiblingPath() throws Exception - { - // GIVEN - String httpUrl = "https://localhost:4242/greeting/hello.txt"; - Path path = Path.of(new URI(httpUrl)); - - // WHEN - Path sibling = path.resolveSibling(Path.of("bye.txt")); - - // THEN - assertThat(sibling.getFileSystem().getClass().getSimpleName(), equalTo("HttpFileSystem")); - assertThat(sibling.getFileSystem().provider().getClass().getSimpleName(), equalTo("HttpsFileSystemProvider")); - assertThat(sibling.toString(), equalTo("https://localhost:4242/greeting/bye.txt")); - } - @Test public void testHttpSiblingString() throws Exception { From 6395780ec7fc22c572f62c83ced4e3dc346cc0a3 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 12:10:08 +0200 Subject: [PATCH 25/43] WIP hfs rm timeout --- .../filesystem/http/AbstractHttpFileSystemProvider.java | 3 --- .../zilla/runtime/filesystem/http/HttpWatchService.java | 5 +---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java index fc99c5e86b..76bace02e5 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java @@ -43,7 +43,6 @@ import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.FileAttributeView; import java.nio.file.spi.FileSystemProvider; -import java.time.Duration; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -51,7 +50,6 @@ public abstract class AbstractHttpFileSystemProvider extends FileSystemProvider { - private static final Duration TIMEOUT = Duration.ofSeconds(5); private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() .version(HTTP_2) .followRedirects(NORMAL) @@ -354,7 +352,6 @@ private static byte[] readBody( HttpRequest request = HttpRequest.newBuilder() .GET() .uri(path.toUri()) - .timeout(TIMEOUT) .build(); System.out.println("AHFSP readBody path " + path + " request " + request); // TODO: Ati HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofByteArray()); diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java index 456c669cd4..8edf66c088 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -29,7 +29,6 @@ import java.nio.file.WatchService; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; @@ -48,7 +47,6 @@ public class HttpWatchService implements WatchService, Callable { - private static final Duration TIMEOUT = Duration.ofSeconds(5); private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() .version(HTTP_2) .followRedirects(NORMAL) @@ -243,8 +241,7 @@ private HttpResponse send( { HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() .GET() - .uri(path.toUri()) - .timeout(TIMEOUT); + .uri(path.toUri()); if (etag != null && !etag.isEmpty()) { requestBuilder = requestBuilder.headers("If-None-Match", etag, "Prefer", "wait=86400"); From 54a2b87f13878ca3ce26e2256b8889badd75f8fe Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 13:23:58 +0200 Subject: [PATCH 26/43] WIP hfs revert send --- .../filesystem/http/HttpWatchService.java | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java index 8edf66c088..858f889bfc 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -108,7 +108,8 @@ public Void call() throws Exception } String etag = etags.getOrDefault(path, ""); System.out.println("HWS call take path " + path + " etag [" + etag + "]"); // TODO: Ati - HttpResponse response = send(path, etag); + sendAsync(path, etag); + /*HttpResponse response = send(path, etag); if (response == null) { scheduleRequest(path, pollSeconds); @@ -116,7 +117,7 @@ public Void call() throws Exception else { handleChange(response); // TODO: Ati - this should receive path -> body should be stored in path - } + }*/ } return null; } @@ -235,8 +236,8 @@ HttpWatchKey register( return watchKey; } - private HttpResponse send( - Path path, + private void sendAsync( + HttpPath path, String etag) { HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() @@ -247,19 +248,20 @@ private HttpResponse send( requestBuilder = requestBuilder.headers("If-None-Match", etag, "Prefer", "wait=86400"); } - HttpResponse response; - try - { - response = HTTP_CLIENT.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()); - //System.out.println("HWS send response " + response); // TODO: Ati - //System.out.println("HWS send response.body " + new String(response.body())); // TODO: Ati - } - catch (Exception e) - { - response = null; - System.out.println("HWS send exception " + e); // TODO: Ati - } - return response; + System.out.println("HWS sendAsync path " + path + " etag " + etag); // TODO: Ati + CompletableFuture future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + .thenAccept(this::handleChange) + .exceptionally(ex -> handleException(ex, path)); + futures.put(path, future); + } + + private Void handleException( + Throwable throwable, + HttpPath path) + { + System.out.println("HWS handleException " + throwable.getMessage()); // TODO: Ati + scheduleRequest(path, pollSeconds); + return null; } private void handleChange( From a5d0a394ec4b3feffdda00707f78863335d4e5ca Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 14:13:06 +0200 Subject: [PATCH 27/43] WIP hfs HP readBody resolveBody --- .../http/AbstractHttpFileSystemProvider.java | 73 +++---------------- .../filesystem/http/HttpFileSystem.java | 10 +-- .../runtime/filesystem/http/HttpPath.java | 52 +++++++++++++ .../filesystem/http/HttpFileSystemIT.java | 4 + 4 files changed, 71 insertions(+), 68 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java index 76bace02e5..e8db435415 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java @@ -14,18 +14,10 @@ */ package io.aklivity.zilla.runtime.filesystem.http; -import static java.net.HttpURLConnection.HTTP_OK; -import static java.net.http.HttpClient.Redirect.NORMAL; -import static java.net.http.HttpClient.Version.HTTP_2; -import static org.agrona.LangUtil.rethrowUnchecked; - import java.io.ByteArrayInputStream; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; import java.nio.channels.AsynchronousFileChannel; import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; @@ -50,11 +42,6 @@ public abstract class AbstractHttpFileSystemProvider extends FileSystemProvider { - private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() - .version(HTTP_2) - .followRedirects(NORMAL) - .build(); - private final Map fileSystems = new ConcurrentHashMap<>(); @Override @@ -117,8 +104,8 @@ public InputStream newInputStream( Path path, OpenOption... options) { - // TODO: Ati - chk if path is http/s -> cast it to HttpPath - return new ByteArrayInputStream(resolveBody(path)); + checkPath(path); + return new ByteArrayInputStream(resolveBody((HttpPath) path)); } @Override @@ -154,8 +141,8 @@ public SeekableByteChannel newByteChannel( Set options, FileAttribute... attrs) { - // TODO: Ati - chk if path is http/s -> cast it to HttpPath - return new ReadOnlyByteArrayChannel(resolveBody(path)); + checkPath(path); + return new ReadOnlyByteArrayChannel(resolveBody((HttpPath) path)); } @Override @@ -314,58 +301,18 @@ private void checkUri( } } - // TODO: Ati - this should be the new resolveBody - /*private byte[] resolveBody( - Path path) - //HttpPath path) - { - // TODO: Ati - this should delegate the call to HttpPath path.resolveBody() - return readBody(path); - }*/ - - // TODO: Ati - remove this - private byte[] resolveBody( + private void checkPath( Path path) { - byte[] body; - //HttpFileSystem fileSystem = fileSystems.get(path.toUri()); - HttpFileSystem fileSystem = (HttpFileSystem) path.getFileSystem(); - if (fileSystem != null && fileSystem.body() != null) + if (!path.getFileSystem().provider().getScheme().equalsIgnoreCase(getScheme())) { - System.out.println("AHFSP resolveBody fs.body"); // TODO: Ati - body = fileSystem.body(); + throw new IllegalArgumentException("Scheme does not match this provider"); } - else - { - System.out.println("AHFSP resolveBody readBody"); // TODO: Ati - body = readBody(path); - } - return body; } - private static byte[] readBody( - Path path) + private byte[] resolveBody( + HttpPath path) { - byte[] body = new byte[0]; - try - { - HttpRequest request = HttpRequest.newBuilder() - .GET() - .uri(path.toUri()) - .build(); - System.out.println("AHFSP readBody path " + path + " request " + request); // TODO: Ati - HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofByteArray()); - //System.out.println("AHFSP readBody response.body " + new String(response.body())); // TODO: Ati - if (response.statusCode() == HTTP_OK) - { - body = response.body(); - } - } - catch (Exception ex) - { - System.out.println("AHFSP readBody exception " + ex); // TODO: Ati - rethrowUnchecked(ex); - } - return body; + return path.resolveBody(); } } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java index 0ded535a8f..4acca8a0fd 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java @@ -33,7 +33,7 @@ public final class HttpFileSystem extends FileSystem private final AbstractHttpFileSystemProvider provider; private final URI root; - private byte[] body; // TODO: Ati - body should be moved from HFS to HttpPath + //private byte[] body; // TODO: Ati - body should be moved from HFS to HttpPath HttpFileSystem( AbstractHttpFileSystemProvider provider, @@ -41,7 +41,7 @@ public final class HttpFileSystem extends FileSystem { this.provider = provider; this.root = root; - this.body = null; // TODO: Ati - body should be moved from HFS to HttpPath + //this.body = null; // TODO: Ati - body should be moved from HFS to HttpPath } @Override @@ -141,15 +141,15 @@ HttpPath resolveSibling( } // TODO: Ati - body should be moved from HFS to HttpPath - byte[] body() + /*byte[] body() { return body; - } + }*/ // TODO: Ati - body should be moved from HFS to HttpPath void body( byte[] body) { - this.body = body; + //this.body = body; } } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index 051b960259..305bd7d778 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -14,11 +14,18 @@ */ package io.aklivity.zilla.runtime.filesystem.http; +import static java.net.HttpURLConnection.HTTP_OK; +import static java.net.http.HttpClient.Redirect.NORMAL; +import static java.net.http.HttpClient.Version.HTTP_2; import static java.util.Objects.requireNonNull; +import static org.agrona.LangUtil.rethrowUnchecked; import java.io.File; import java.io.IOException; import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.ProviderMismatchException; @@ -30,9 +37,16 @@ public class HttpPath implements Path { + private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() + .version(HTTP_2) + .followRedirects(NORMAL) + .build(); + private final HttpFileSystem fs; private final URI location; + private byte[] body; + HttpPath( HttpFileSystem fs, URI location) @@ -43,12 +57,14 @@ public class HttpPath implements Path } this.fs = fs; this.location = location; + this.body = null; } HttpPath() { this.fs = null; this.location = null; + this.body = null; } @Override @@ -260,4 +276,40 @@ public int hashCode() { return Objects.hashCode(location); } + + byte[] resolveBody() + { + if (body == null) + { + body = readBody(); + } + return body; + } + + private byte[] readBody() + { + byte[] body = new byte[0]; + try + { + // TODO: Ati - add etag to the header + // TODO: Ati - check+store etag/hash + HttpRequest request = HttpRequest.newBuilder() + .GET() + .uri(location) + .build(); + System.out.println("HP readBody path " + location + " request " + request); // TODO: Ati + HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofByteArray()); + //System.out.println("HP readBody response.body " + new String(response.body())); // TODO: Ati + if (response.statusCode() == HTTP_OK) + { + body = response.body(); + } + } + catch (Exception ex) + { + System.out.println("AHFSP readBody exception " + ex); // TODO: Ati + rethrowUnchecked(ex); + } + return body; + } } diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java index e87edce52f..53008270ac 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java @@ -29,6 +29,7 @@ import java.nio.file.spi.FileSystemProvider; import java.util.List; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.DisableOnDebug; @@ -165,6 +166,7 @@ public void shouldWatch() throws Exception assertThat(events2.get(0).context(), equalTo(path)); } + @Ignore // TODO: Ati @Test @Specification({ "${app}/watch/server", @@ -193,6 +195,7 @@ public void shouldWatchReadString() throws Exception assertThat(body2, equalTo("Hello Universe!")); } + @Ignore // TODO: Ati @Test @Specification({ "${app}/watch.error.success/server", @@ -225,6 +228,7 @@ public void shouldWatchErrorSuccess() throws Exception assertThat(events2.get(0).context(), equalTo(path)); } + @Ignore // TODO: Ati @Test @Specification({ "${app}/watch.error.success/server", From f19e62356063eea54b362b16314f3f8ef8c9ad6d Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 15:12:15 +0200 Subject: [PATCH 28/43] WIP hfs HP readBody etag --- .../filesystem/http/HttpFileSystem.java | 2 +- .../runtime/filesystem/http/HttpPath.java | 42 ++++++++-- .../filesystem/http/HttpFileSystemIT.java | 77 +++++++++++++++++-- .../application/notfound.success/client.rpt | 34 ++++++++ .../application/notfound.success/server.rpt | 37 +++++++++ .../success.etag.modified/client.rpt | 37 +++++++++ .../success.etag.modified/server.rpt | 40 ++++++++++ .../success.etag.not.modified/client.rpt | 36 +++++++++ .../success.etag.not.modified/server.rpt | 39 ++++++++++ .../specs/filesystem/http/ApplicationIT.java | 28 +++++++ 10 files changed, 356 insertions(+), 16 deletions(-) create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/client.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/server.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/client.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/server.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/client.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/server.rpt diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java index 4acca8a0fd..76bbe18ce0 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java @@ -28,7 +28,7 @@ public final class HttpFileSystem extends FileSystem { - public static final String SEPARATOR = "/"; + private static final String SEPARATOR = "/"; private final AbstractHttpFileSystemProvider provider; private final URI root; diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index 305bd7d778..2aea28d737 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -14,6 +14,8 @@ */ package io.aklivity.zilla.runtime.filesystem.http; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import static java.net.HttpURLConnection.HTTP_NOT_MODIFIED; import static java.net.HttpURLConnection.HTTP_OK; import static java.net.http.HttpClient.Redirect.NORMAL; import static java.net.http.HttpClient.Version.HTTP_2; @@ -34,6 +36,7 @@ import java.nio.file.WatchService; import java.util.Iterator; import java.util.Objects; +import java.util.Optional; public class HttpPath implements Path { @@ -46,6 +49,7 @@ public class HttpPath implements Path private final URI location; private byte[] body; + private String etag; HttpPath( HttpFileSystem fs, @@ -58,6 +62,7 @@ public class HttpPath implements Path this.fs = fs; this.location = location; this.body = null; + this.etag = null; } HttpPath() @@ -279,30 +284,53 @@ public int hashCode() byte[] resolveBody() { - if (body == null) + // TODO: Ati - if we always call readBody, can this be removed? + System.out.println("HP resolveBody"); + body = readBody(); + /*if (body == null) { body = readBody(); - } + }*/ return body; } private byte[] readBody() { - byte[] body = new byte[0]; + //byte[] body = new byte[0]; try { // TODO: Ati - add etag to the header // TODO: Ati - check+store etag/hash - HttpRequest request = HttpRequest.newBuilder() + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() .GET() - .uri(location) - .build(); - System.out.println("HP readBody path " + location + " request " + request); // TODO: Ati + .uri(location); + if (etag != null && !etag.isEmpty()) + { + //requestBuilder = requestBuilder.headers("If-None-Match", etag, "Prefer", "wait=86400"); + // TODO: Ati - this is a sync call, I guess we don't need Prefer wait + requestBuilder = requestBuilder.headers("If-None-Match", etag); + } + HttpRequest request = requestBuilder.build(); + System.out.println("HP readBody path " + location + " request " + request + " etag " + etag); // TODO: Ati HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofByteArray()); //System.out.println("HP readBody response.body " + new String(response.body())); // TODO: Ati if (response.statusCode() == HTTP_OK) { body = response.body(); + Optional etagOptional = response.headers().firstValue("Etag"); + if (etagOptional.isPresent()) + { + etag = etagOptional.get(); + } + } + else if (response.statusCode() == HTTP_NOT_FOUND) + { + body = new byte[0]; + etag = null; + } + else if (response.statusCode() == HTTP_NOT_MODIFIED) + { + // no op } } catch (Exception ex) diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java index 53008270ac..02ade5afa3 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java @@ -41,16 +41,14 @@ public class HttpFileSystemIT { - private static final String HELLO_BODY = "Hello World!"; - private static final String EMPTY_BODY = ""; - private final K3poRule k3po = new K3poRule() .addScriptRoot("app", "io/aklivity/zilla/specs/filesystem/http/application"); private final TestRule timeout = new DisableOnDebug(new Timeout(10, SECONDS)); @Rule - public final TestRule chain = outerRule(timeout).around(k3po); + //public final TestRule chain = outerRule(timeout).around(k3po); // TODO: Ati + public final TestRule chain = outerRule(k3po).around(timeout); @Test @Specification({ @@ -68,7 +66,49 @@ public void shouldReadString() throws Exception k3po.finish(); // THEN - assertThat(helloBody, equalTo(HELLO_BODY)); + assertThat(helloBody, equalTo("Hello World!")); + } + + @Test + @Specification({ + "${app}/success.etag.not.modified/server", + }) + public void shouldReadStringEtagNotModified() throws Exception + { + // GIVEN + String helloUrl = "http://localhost:8080/hello.txt"; + Path helloPath = Path.of(new URI(helloUrl)); + + // WHEN + k3po.start(); + String helloBody1 = Files.readString(helloPath); + String helloBody2 = Files.readString(helloPath); + k3po.finish(); + + // THEN + assertThat(helloBody1, equalTo("Hello World!")); + assertThat(helloBody2, equalTo("Hello World!")); + } + + @Test + @Specification({ + "${app}/success.etag.modified/server", + }) + public void shouldReadStringEtagModified() throws Exception + { + // GIVEN + String helloUrl = "http://localhost:8080/hello.txt"; + Path helloPath = Path.of(new URI(helloUrl)); + + // WHEN + k3po.start(); + String helloBody1 = Files.readString(helloPath); + String helloBody2 = Files.readString(helloPath); + k3po.finish(); + + // THEN + assertThat(helloBody1, equalTo("Hello World!")); + assertThat(helloBody2, equalTo("Hello Universe!")); } @Test @@ -87,7 +127,28 @@ public void shouldReadStringNotFound() throws Exception k3po.finish(); // THEN - assertThat(notFoundBody, equalTo(EMPTY_BODY)); + assertThat(notFoundBody, equalTo("")); + } + + @Test + @Specification({ + "${app}/notfound.success/server", + }) + public void shouldReadStringNotFoundSuccess() throws Exception + { + // GIVEN + String helloUrl = "http://localhost:8080/hello.txt"; + Path helloPath = Path.of(new URI(helloUrl)); + + // WHEN + k3po.start(); + String helloBody1 = Files.readString(helloPath); + String helloBody2 = Files.readString(helloPath); + k3po.finish(); + + // THEN + assertThat(helloBody1, equalTo("")); + assertThat(helloBody2, equalTo("Hello World!")); } @Test @@ -109,7 +170,7 @@ public void shouldReadInputStream() throws Exception k3po.finish(); // THEN - assertThat(helloBody, equalTo(HELLO_BODY)); + assertThat(helloBody, equalTo("Hello World!")); } @Test @@ -131,7 +192,7 @@ public void shouldReadInputStreamNotFound() throws Exception k3po.finish(); // THEN - assertThat(notFoundBody, equalTo(EMPTY_BODY)); + assertThat(notFoundBody, equalTo("")); } @Test diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/client.rpt new file mode 100644 index 0000000000..c44e105eab --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/client.rpt @@ -0,0 +1,34 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write close + +read http:status "404" "Not Found" +read closed + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write close + +read http:status "200" "OK" +read http:header "Etag" "AAAAAAA" +read "Hello World!" +read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/server.rpt new file mode 100644 index 0000000000..74dfcfc1a7 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/server.rpt @@ -0,0 +1,37 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "http://localhost:8080/hello.txt" +accepted +connected + +read http:method "GET" +read closed + +write http:status "404" "Not Found" +write http:content-length +write close + +accepted +connected + +read http:method "GET" +read closed + +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "AAAAAAA" +write "Hello World!" +write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/client.rpt new file mode 100644 index 0000000000..9ebace3825 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/client.rpt @@ -0,0 +1,37 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write close + +read http:status "200" "OK" +read http:header "Etag" "AAAAAAA" +read "Hello World!" +read closed + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write http:header "If-None-Match" "AAAAAAA" +write close + +read http:status "200" "OK" +read http:header "Etag" "BBBBBBB" +read "Hello Universe!" +read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/server.rpt new file mode 100644 index 0000000000..0052eb0bf2 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/server.rpt @@ -0,0 +1,40 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "http://localhost:8080/hello.txt" +accepted +connected + +read http:method "GET" +read closed + +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "AAAAAAA" +write "Hello World!" +write close + +accepted +connected + +read http:method "GET" +read http:header "If-None-Match" "AAAAAAA" +read closed + +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "BBBBBBB" +write "Hello Universe!" +write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/client.rpt new file mode 100644 index 0000000000..7b386786a5 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/client.rpt @@ -0,0 +1,36 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write close + +read http:status "200" "OK" +read http:header "Etag" "AAAAAAA" +read "Hello World!" +read closed + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write http:header "If-None-Match" "AAAAAAA" +write close + +read http:status "304" "Not Modified" +read http:header "Etag" "AAAAAAA" +read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/server.rpt new file mode 100644 index 0000000000..1c7c544b51 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/server.rpt @@ -0,0 +1,39 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "http://localhost:8080/hello.txt" +accepted +connected + +read http:method "GET" +read closed + +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "AAAAAAA" +write "Hello World!" +write close + +accepted +connected + +read http:method "GET" +read http:header "If-None-Match" "AAAAAAA" +read closed + +write http:status "304" "Not Modified" +write http:content-length +write http:header "Etag" "AAAAAAA" +write close diff --git a/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java index 5e75275f4f..90bf03fc34 100644 --- a/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java +++ b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java @@ -45,6 +45,24 @@ public void shouldReadString() throws Exception k3po.finish(); } + @Test + @Specification({ + "${app}/success.etag.not.modified/client", + "${app}/success.etag.not.modified/server" }) + public void shouldReadStringEtagNotModified() throws Exception + { + k3po.finish(); + } + + @Test + @Specification({ + "${app}/success.etag.modified/client", + "${app}/success.etag.modified/server" }) + public void shouldReadStringEtagModified() throws Exception + { + k3po.finish(); + } + @Test @Specification({ "${app}/notfound/client", @@ -54,6 +72,16 @@ public void shouldReadStringNotFound() throws Exception k3po.finish(); } + @Test + @Specification({ + "${app}/notfound.success/client", + "${app}/notfound.success/server" }) + public void shouldReadStringNotFoundSuccess() throws Exception + { + k3po.finish(); + } + + @Test @Specification({ "${app}/watch/client", From a8a4015a5e70ab5e013aa68b98db9a8c841de6a2 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 15:37:32 +0200 Subject: [PATCH 29/43] WIP hfs HP --- .../io/aklivity/zilla/runtime/filesystem/http/HttpPath.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index 2aea28d737..f0807b36e4 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -299,8 +299,6 @@ private byte[] readBody() //byte[] body = new byte[0]; try { - // TODO: Ati - add etag to the header - // TODO: Ati - check+store etag/hash HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() .GET() .uri(location); @@ -318,6 +316,7 @@ private byte[] readBody() { body = response.body(); Optional etagOptional = response.headers().firstValue("Etag"); + // TODO: Ati - calculate and store hash if there is no etag if (etagOptional.isPresent()) { etag = etagOptional.get(); @@ -335,7 +334,7 @@ else if (response.statusCode() == HTTP_NOT_MODIFIED) } catch (Exception ex) { - System.out.println("AHFSP readBody exception " + ex); // TODO: Ati + System.out.println("HP readBody exception " + ex); // TODO: Ati rethrowUnchecked(ex); } return body; From c2efe2126a96e8aaa8ef959fa972244490fec966 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 17:36:45 +0200 Subject: [PATCH 30/43] WIP hfs ignore AppIT --- .../aklivity/zilla/specs/filesystem/http/ApplicationIT.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java index 90bf03fc34..c6f26fd714 100644 --- a/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java +++ b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java @@ -17,6 +17,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.rules.RuleChain.outerRule; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.DisableOnDebug; @@ -45,6 +46,7 @@ public void shouldReadString() throws Exception k3po.finish(); } + @Ignore // TODO: Ati @Test @Specification({ "${app}/success.etag.not.modified/client", @@ -54,6 +56,7 @@ public void shouldReadStringEtagNotModified() throws Exception k3po.finish(); } + @Ignore // TODO: Ati @Test @Specification({ "${app}/success.etag.modified/client", @@ -72,6 +75,7 @@ public void shouldReadStringNotFound() throws Exception k3po.finish(); } + @Ignore // TODO: Ati @Test @Specification({ "${app}/notfound.success/client", @@ -82,6 +86,7 @@ public void shouldReadStringNotFoundSuccess() throws Exception } + @Ignore // TODO: Ati @Test @Specification({ "${app}/watch/client", @@ -91,6 +96,7 @@ public void shouldWatch() throws Exception k3po.finish(); } + @Ignore // TODO: Ati @Test @Specification({ "${app}/watch.error.success/client", From 873b5e1c927d8a7e705ea579e73dbdd83df79eeb Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 17:44:27 +0200 Subject: [PATCH 31/43] WIP hfs fix AppIT --- .../http/application/notfound.success/client.rpt | 2 ++ .../http/application/success.etag.modified/client.rpt | 2 ++ .../http/application/success.etag.not.modified/client.rpt | 2 ++ .../http/application/watch.error.success/client.rpt | 2 ++ .../specs/filesystem/http/application/watch/client.rpt | 2 ++ .../zilla/specs/filesystem/http/ApplicationIT.java | 7 ------- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/client.rpt index c44e105eab..7709573bb1 100644 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/client.rpt +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/client.rpt @@ -20,11 +20,13 @@ write http:method "GET" write close read http:status "404" "Not Found" +read notify FIRST_READ read closed connect "http://localhost:8080/hello.txt" connected +write await FIRST_READ write http:method "GET" write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/client.rpt index 9ebace3825..27e498af0c 100644 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/client.rpt +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/client.rpt @@ -22,11 +22,13 @@ write close read http:status "200" "OK" read http:header "Etag" "AAAAAAA" read "Hello World!" +read notify FIRST_READ read closed connect "http://localhost:8080/hello.txt" connected +write await FIRST_READ write http:method "GET" write http:header "If-None-Match" "AAAAAAA" write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/client.rpt index 7b386786a5..68a283c0c5 100644 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/client.rpt +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/client.rpt @@ -22,11 +22,13 @@ write close read http:status "200" "OK" read http:header "Etag" "AAAAAAA" read "Hello World!" +read notify FIRST_READ read closed connect "http://localhost:8080/hello.txt" connected +write await FIRST_READ write http:method "GET" write http:header "If-None-Match" "AAAAAAA" write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/client.rpt index 8d530015d2..79cf7c7d1f 100644 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/client.rpt +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/client.rpt @@ -20,11 +20,13 @@ write http:method "GET" write close read http:status "404" "Not Found" +read notify FIRST_READ read closed connect "http://localhost:8080/hello.txt" connected +write await FIRST_READ write http:method "GET" write http:header "If-None-Match" "AAAAAAA" write http:header "Prefer" "wait=86400" diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt index 53048d7d00..cd91a9ea08 100644 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt @@ -22,11 +22,13 @@ write close read http:status "200" "OK" read http:header "Etag" "AAAAAAA" read "Hello World!" +read notify FIRST_READ read closed connect "http://localhost:8080/hello.txt" connected +write await FIRST_READ write http:method "GET" write http:header "If-None-Match" "AAAAAAA" write http:header "Prefer" "wait=86400" diff --git a/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java index c6f26fd714..576ac7ff15 100644 --- a/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java +++ b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java @@ -17,7 +17,6 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.rules.RuleChain.outerRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.DisableOnDebug; @@ -46,7 +45,6 @@ public void shouldReadString() throws Exception k3po.finish(); } - @Ignore // TODO: Ati @Test @Specification({ "${app}/success.etag.not.modified/client", @@ -56,7 +54,6 @@ public void shouldReadStringEtagNotModified() throws Exception k3po.finish(); } - @Ignore // TODO: Ati @Test @Specification({ "${app}/success.etag.modified/client", @@ -75,7 +72,6 @@ public void shouldReadStringNotFound() throws Exception k3po.finish(); } - @Ignore // TODO: Ati @Test @Specification({ "${app}/notfound.success/client", @@ -85,8 +81,6 @@ public void shouldReadStringNotFoundSuccess() throws Exception k3po.finish(); } - - @Ignore // TODO: Ati @Test @Specification({ "${app}/watch/client", @@ -96,7 +90,6 @@ public void shouldWatch() throws Exception k3po.finish(); } - @Ignore // TODO: Ati @Test @Specification({ "${app}/watch.error.success/client", From c1664313e56176528a379192952a1a19a227ff83 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 20:52:34 +0200 Subject: [PATCH 32/43] WIP hfs watchBody 1 --- runtime/filesystem-http/pom.xml | 2 +- .../runtime/filesystem/http/HttpPath.java | 124 +++++++++- .../filesystem/http/HttpWatchService.java | 228 +++++++++++------- .../filesystem/http/HttpFileSystemIT.java | 14 +- 4 files changed, 274 insertions(+), 94 deletions(-) diff --git a/runtime/filesystem-http/pom.xml b/runtime/filesystem-http/pom.xml index 10ab1d2590..f944c54a49 100644 --- a/runtime/filesystem-http/pom.xml +++ b/runtime/filesystem-http/pom.xml @@ -26,7 +26,7 @@ 11 11 - 0.60 + 0.50 0 diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index f0807b36e4..e9d64ba8b7 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -19,6 +19,7 @@ import static java.net.HttpURLConnection.HTTP_OK; import static java.net.http.HttpClient.Redirect.NORMAL; import static java.net.http.HttpClient.Version.HTTP_2; +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; import static java.util.Objects.requireNonNull; import static org.agrona.LangUtil.rethrowUnchecked; @@ -37,6 +38,7 @@ import java.util.Iterator; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.CompletableFuture; public class HttpPath implements Path { @@ -44,12 +46,17 @@ public class HttpPath implements Path .version(HTTP_2) .followRedirects(NORMAL) .build(); + private static final byte[] EMPTY_BODY = new byte[0]; private final HttpFileSystem fs; private final URI location; private byte[] body; private String etag; + private CompletableFuture future; + //private HttpResponse response; + private HttpWatchService.HttpWatchKey watchKey; + private int pollSeconds; HttpPath( HttpFileSystem fs, @@ -63,6 +70,9 @@ public class HttpPath implements Path this.location = location; this.body = null; this.etag = null; + this.future = null; + // TODO: Ati - do Configuration for this project, too + this.pollSeconds = 2; } HttpPath() @@ -70,6 +80,9 @@ public class HttpPath implements Path this.fs = null; this.location = null; this.body = null; + this.etag = null; + this.future = null; + this.pollSeconds = 0; } @Override @@ -229,7 +242,9 @@ public WatchKey register( { throw new ProviderMismatchException(); } - return ((HttpWatchService) watcher).register(this, events, modifiers); + watchKey = ((HttpWatchService) watcher).register(this, events, modifiers); + watchKey.watchBody(); + return watchKey; } @Override @@ -339,4 +354,111 @@ else if (response.statusCode() == HTTP_NOT_MODIFIED) } return body; } + + void watchBody() + { + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .GET() + .uri(location); + if (etag != null && !etag.isEmpty()) + { + requestBuilder = requestBuilder.headers("If-None-Match", etag, "Prefer", "wait=86400"); + } + // TODO: Ati - handle poll interval for both long polling and normal polling + HttpRequest request = requestBuilder.build(); + System.out.println("HP watchBody path " + location + " request " + request + " etag " + etag); // TODO: Ati + //CompletableFuture future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + .thenAccept(this::acceptResponse) + .exceptionally(this::handleException); + } + + boolean isDone() + { + return future.isDone(); + } + + /*HttpResponse poll() throws Exception + { + future.get(); + return response; + }*/ + + private void acceptResponse( + HttpResponse response) + { + //this.response = response; + System.out.println("HP acceptResponse response: " + response); // TODO: Ati + System.out.println("HP acceptResponse response.headers: " + response.headers()); // TODO: Ati + //System.out.println("HWS handleChange response.body: " + new String(response.body())); // TODO: Ati + HttpPath path = (HttpPath) Path.of(response.request().uri()); + int statusCode = response.statusCode(); + int pollSeconds = 0; + if (statusCode == 404) + { + body = EMPTY_BODY; + watchKey.addEvent(ENTRY_MODIFY, this); + pollSeconds = this.pollSeconds; + } + else if (statusCode >= 500 && statusCode <= 599) + { + body = null; + pollSeconds = this.pollSeconds; + } + else + { + //byte[] body = response.body(); + this.body = response.body(); + Optional etagOptional = response.headers().firstValue("Etag"); + if (etagOptional.isPresent()) + { + //String oldEtag = etags.getOrDefault(path, ""); // TODO: Ati + String newEtag = etagOptional.get(); + if (!newEtag.equals(etag)) + { + //etags.put(path, newEtag); // TODO: Ati + etag = newEtag; + watchKey.addEvent(ENTRY_MODIFY, this); + + } + else if (response.statusCode() != 304) + { + pollSeconds = this.pollSeconds; + } + } + else + { + // TODO: Ati - hash + //byte[] hash = hashes.get(path); + //byte[] newHash = computeHash(body); + //if (!Arrays.equals(hash, newHash)) + { + //hashes.put(path, newHash); // TODO: Ati + //addEvent(ENTRY_MODIFY, path); + watchKey.addEvent(ENTRY_MODIFY, this); + //addEvent(path); + } + pollSeconds = this.pollSeconds; + } + } + //futures.remove(path); + //scheduleRequest(path, pollSeconds); // ??? + try + { + Thread.sleep(pollSeconds * 1000); // TODO: Ati + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + watchBody(); + } + + private Void handleException( + Throwable throwable) + { + System.out.println("HP handleException " + throwable.getMessage()); // TODO: Ati + //scheduleRequest(path, pollSeconds); + return null; + } } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java index 858f889bfc..1b1c623e26 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -14,14 +14,9 @@ */ package io.aklivity.zilla.runtime.filesystem.http; -import static java.net.http.HttpClient.Redirect.NORMAL; -import static java.net.http.HttpClient.Version.HTTP_2; import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; import static org.agrona.LangUtil.rethrowUnchecked; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; import java.nio.file.ClosedWatchServiceException; import java.nio.file.Path; import java.nio.file.WatchEvent; @@ -29,17 +24,11 @@ import java.nio.file.WatchService; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; @@ -47,46 +36,47 @@ public class HttpWatchService implements WatchService, Callable { - private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() + /*private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() .version(HTTP_2) .followRedirects(NORMAL) - .build(); + .build();*/ //private static final Path CLOSE_PATH = Path.of(URI.create("http://localhost:12345")); private static final HttpPath CLOSE_PATH = new HttpPath(); private static final byte[] EMPTY_BODY = new byte[0]; private final WatchKey closeKey = new HttpWatchKey(CLOSE_PATH); - private final HttpFileSystem fileSystem; // TODO: Ati - remove this + //private final HttpFileSystem fileSystem; // TODO: Ati - remove this private final ScheduledExecutorService executor; private final LinkedBlockingQueue pendingKeys; //private final BlockingQueue pathQueue; - private final BlockingQueue pathQueue; + //private final BlockingQueue pathQueue; //private final Map watchKeys; - private final Map watchKeys; - private final Map etags; // TODO: Ati - private final Map hashes; // TODO: Ati - private final Map> futures; + //private final Map watchKeys; + private final List watchKeys; + //private final Map etags; // TODO: Ati + //private final Map hashes; // TODO: Ati + //private final Map> futures; private final MessageDigest md5; - private int pollSeconds; + //private int pollSeconds; private volatile boolean closed; public HttpWatchService( HttpFileSystem fileSystem) // TODO: Ati - remove this { - this.fileSystem = fileSystem; // TODO: Ati - remove this + //this.fileSystem = fileSystem; // TODO: Ati - remove this this.executor = Executors.newScheduledThreadPool(2); this.pendingKeys = new LinkedBlockingQueue<>(); - this.watchKeys = new ConcurrentHashMap<>(); - this.pathQueue = new LinkedBlockingQueue<>(); - this.etags = new ConcurrentHashMap<>(); - this.hashes = new ConcurrentHashMap<>(); - this.futures = new ConcurrentHashMap<>(); + //this.watchKeys = new ConcurrentHashMap<>(); + this.watchKeys = Collections.synchronizedList(new LinkedList<>()); + //this.pathQueue = new LinkedBlockingQueue<>(); + //this.etags = new ConcurrentHashMap<>(); + //this.hashes = new ConcurrentHashMap<>(); + //this.futures = new ConcurrentHashMap<>(); this.md5 = initMessageDigest("MD5"); //this.pollSeconds = 30; - // TODO: Ati - do Configuration for this project, too - this.pollSeconds = 2; + //this.pollSeconds = 2; this.closed = false; //executor.submit(this); } @@ -96,58 +86,61 @@ public void start() executor.submit(this); } + // @Override + // public Void call() throws Exception + // { + // while (true) + // { + // HttpPath path = pathQueue.take(); + // if (path == CLOSE_PATH) + // { + // break; + // } + // String etag = etags.getOrDefault(path, ""); + // System.out.println("HWS call take path " + path + " etag [" + etag + "]"); // TODO: Ati + // sendAsync(path, etag); + // /*HttpResponse response = send(path, etag); + // if (response == null) + // { + // scheduleRequest(path, pollSeconds); + // } + // else + // { + // handleChange(response); // TODO: Ati - this should receive path -> body should be stored in path + // }*/ + // } + // return null; + // } + @Override public Void call() throws Exception - { - while (true) - { - HttpPath path = pathQueue.take(); - if (path == CLOSE_PATH) - { - break; - } - String etag = etags.getOrDefault(path, ""); - System.out.println("HWS call take path " + path + " etag [" + etag + "]"); // TODO: Ati - sendAsync(path, etag); - /*HttpResponse response = send(path, etag); - if (response == null) - { - scheduleRequest(path, pollSeconds); - } - else - { - handleChange(response); // TODO: Ati - this should receive path -> body should be stored in path - }*/ - } - return null; - } - - //@Override - /*public Void call_NEW() throws Exception { for (HttpWatchKey watchKey : watchKeys) // TODO: Ati watchKeys should be a List { HttpPath path = watchKey.watchable(); if (path.isDone()) { - HttpResponse response = path.poll(); - handleChange(response); // this throws Exception if error - path.watchBody(); // scheduleRequest can be deleted + //HttpResponse response = path.poll(); + //watchKey.handleChange(response); + //handleChange(response); // this throws Exception if error // this should be in HttpWatchKey ?? + watchKey.watchBody(); + //path.watchBody(); // scheduleRequest can be deleted // actually watchKey.watchBody() that calls path.watchBody() // how to deal with poll interval } } - }*/ + return null; + } @Override public void close() { System.out.println("HWS close"); // TODO: Ati closed = true; - fileSystem.body(null); // TODO: Ati - body should be moved from HFS to HttpPath + //fileSystem.body(null); // TODO: Ati - body should be moved from HFS to HttpPath pendingKeys.clear(); pendingKeys.offer(closeKey); - futures.values().forEach(future -> future.cancel(true)); - pathQueue.add(CLOSE_PATH); + //futures.values().forEach(future -> future.cancel(true)); + //pathQueue.add(CLOSE_PATH); watchKeys.clear(); } @@ -180,11 +173,11 @@ public WatchKey take() throws InterruptedException return key; } - public void pollSeconds( + /*public void pollSeconds( int pollSeconds) { this.pollSeconds = pollSeconds; - } + }*/ private void checkOpen() { @@ -229,14 +222,15 @@ HttpWatchKey register( } System.out.printf("HWS register path: %s\n", path); // TODO: Ati //WatchKey watchKey = watchKeys.computeIfAbsent(path, i -> new HttpWatchKey(path)); - HttpWatchKey watchKey = watchKeys.computeIfAbsent(path, i -> new HttpWatchKey(path)); - // watchKeys.add(new HttpWatchKey(path)) // TODO: Ati - watchKeys should be a List + //HttpWatchKey watchKey = watchKeys.computeIfAbsent(path, i -> new HttpWatchKey(path)); + HttpWatchKey watchKey = new HttpWatchKey(path); // I moved this to HP + watchKeys.add(watchKey); // TODO: Ati - watchKeys should be a List //pathQueue.offer(path); - pathQueue.add(path); + //pathQueue.add(path); return watchKey; } - private void sendAsync( + /*private void sendAsync( HttpPath path, String etag) { @@ -253,18 +247,20 @@ private void sendAsync( .thenAccept(this::handleChange) .exceptionally(ex -> handleException(ex, path)); futures.put(path, future); - } + }*/ - private Void handleException( + // TODO: Ati - remove this + /*private Void handleException( Throwable throwable, HttpPath path) { System.out.println("HWS handleException " + throwable.getMessage()); // TODO: Ati - scheduleRequest(path, pollSeconds); + //scheduleRequest(path, pollSeconds); return null; - } + }*/ - private void handleChange( + // TODO: Ati - this should be in HWK + /*private void handleChange( HttpResponse response) { System.out.println("HWS handleChange response: " + response); // TODO: Ati @@ -318,9 +314,10 @@ else if (response.statusCode() != 304) } futures.remove(path); scheduleRequest(path, pollSeconds); - } + }*/ - private void addEvent( + // TODO: Ati - this should be in HWK + /*private void addEvent( Path path) { System.out.println("HWS addEvent path " + path); // TODO: Ati @@ -331,9 +328,9 @@ private void addEvent( key.addEvent(ENTRY_MODIFY, path); enqueueKey(key); } - } + }*/ - private void scheduleRequest( + /*private void scheduleRequest( //Path path, HttpPath path, int pollSeconds) @@ -341,14 +338,14 @@ private void scheduleRequest( if (pollSeconds == 0) { System.out.println("HWS scheduleRequest 0"); // TODO: Ati - pathQueue.add(path); + //pathQueue.add(path); } else { System.out.println("HWS scheduleRequest " + pollSeconds); // TODO: Ati - executor.schedule(() -> pathQueue.add(path), pollSeconds, TimeUnit.SECONDS); + //executor.schedule(() -> pathQueue.add(path), pollSeconds, TimeUnit.SECONDS); } - } + }*/ /*private void scheduleRequest___NEW( //Path path, @@ -379,9 +376,8 @@ private MessageDigest initMessageDigest( return md5; } - private final class HttpWatchKey implements WatchKey + public final class HttpWatchKey implements WatchKey { - //private final Path path; private final HttpPath path; private List> events = Collections.synchronizedList(new LinkedList<>()); @@ -390,7 +386,6 @@ private final class HttpWatchKey implements WatchKey private HttpWatchKey( HttpPath path) - //Path path) { this.path = path; this.valid = true; @@ -424,7 +419,6 @@ public void cancel() } @Override - //public Watchable watchable() public HttpPath watchable() { return path; @@ -434,8 +428,8 @@ void addEvent( WatchEvent.Kind kind, Path context) { - Event ev = new Event<>(kind, context); - events.add(ev); + events.add(new Event<>(kind, context)); + enqueueKey(this); } @Override @@ -460,6 +454,70 @@ public int hashCode() return Objects.hashCode(path); } + void watchBody() + { + path.watchBody(); + } + + /*private void handleChange( + HttpResponse response) + { + System.out.println("HWS handleChange response: " + response); // TODO: Ati + System.out.println("HWS handleChange response.headers: " + response.headers()); // TODO: Ati + //System.out.println("HWS handleChange response.body: " + new String(response.body())); // TODO: Ati + HttpPath path = (HttpPath) Path.of(response.request().uri()); + int statusCode = response.statusCode(); + int pollSeconds = 0; + if (statusCode == 404) + { + fileSystem.body(EMPTY_BODY); // TODO: Ati - body should be moved from HFS to HttpPath + addEvent(ENTRY_MODIFY, path); + //addEvent(path); + //pollSeconds = this.pollSeconds; + } + else if (statusCode >= 500 && statusCode <= 599) + { + fileSystem.body(null); // TODO: Ati - body should be moved from HFS to HttpPath + //pollSeconds = this.pollSeconds; + } + else + { + byte[] body = response.body(); + fileSystem.body(body); // TODO: Ati - body should be moved from HFS to HttpPath + Optional etagOptional = response.headers().firstValue("Etag"); + if (etagOptional.isPresent()) + { + String oldEtag = etags.getOrDefault(path, ""); // TODO: Ati + String newEtag = etagOptional.get(); + if (!oldEtag.equals(newEtag)) + { + etags.put(path, newEtag); // TODO: Ati + addEvent(ENTRY_MODIFY, path); + //addEvent(path); + + } + else if (response.statusCode() != 304) + { + //pollSeconds = this.pollSeconds; + } + } + else + { + byte[] hash = hashes.get(path); + byte[] newHash = computeHash(body); + if (!Arrays.equals(hash, newHash)) + { + hashes.put(path, newHash); // TODO: Ati + addEvent(ENTRY_MODIFY, path); + //addEvent(path); + } + //pollSeconds = this.pollSeconds; + } + } + //futures.remove(path); + //scheduleRequest(path, pollSeconds); + }*/ + private static class Event implements WatchEvent { private final WatchEvent.Kind kind; diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java index 02ade5afa3..6d57ef51c7 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java @@ -205,7 +205,7 @@ public void shouldWatch() throws Exception String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - watchService.pollSeconds(1); + //watchService.pollSeconds(1); // TODO: Ati path.register(watchService); // WHEN @@ -227,10 +227,10 @@ public void shouldWatch() throws Exception assertThat(events2.get(0).context(), equalTo(path)); } - @Ignore // TODO: Ati + @Ignore @Test @Specification({ - "${app}/watch/server", + "${app}/watch.read/server", }) public void shouldWatchReadString() throws Exception { @@ -238,7 +238,7 @@ public void shouldWatchReadString() throws Exception String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - watchService.pollSeconds(1); + //watchService.pollSeconds(1); // TODO: Ati path.register(watchService); // WHEN @@ -256,7 +256,7 @@ public void shouldWatchReadString() throws Exception assertThat(body2, equalTo("Hello Universe!")); } - @Ignore // TODO: Ati + @Ignore // TODO: Ati - flaky @Test @Specification({ "${app}/watch.error.success/server", @@ -267,7 +267,7 @@ public void shouldWatchErrorSuccess() throws Exception String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - watchService.pollSeconds(1); + //watchService.pollSeconds(1); // TODO: Ati path.register(watchService); // WHEN @@ -300,7 +300,7 @@ public void shouldWatchErrorSuccessReadString() throws Exception String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - watchService.pollSeconds(1); + //watchService.pollSeconds(1); // TODO: Ati path.register(watchService); // WHEN From 0c0f7f492bd6d8c5e4b416825371480f4647bbd8 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Thu, 20 Jun 2024 22:16:45 +0200 Subject: [PATCH 33/43] WIP hfs watchBody 2 --- .../filesystem/http/HttpFileSystem.java | 5 +- .../runtime/filesystem/http/HttpPath.java | 39 +++++++------ .../filesystem/http/HttpWatchService.java | 57 +++++++++++-------- .../filesystem/http/HttpFileSystemIT.java | 11 ++-- 4 files changed, 65 insertions(+), 47 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java index 76bbe18ce0..6e1a9f86de 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java @@ -129,9 +129,10 @@ public UserPrincipalLookupService getUserPrincipalLookupService() @Override public HttpWatchService newWatchService() { - HttpWatchService service = new HttpWatchService(this); + /*HttpWatchService service = new HttpWatchService(this); service.start(); - return service; + return service;*/ + return new HttpWatchService(this); } HttpPath resolveSibling( diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index e9d64ba8b7..fbcd7ca4af 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -56,7 +56,7 @@ public class HttpPath implements Path private CompletableFuture future; //private HttpResponse response; private HttpWatchService.HttpWatchKey watchKey; - private int pollSeconds; + private boolean longPolling; HttpPath( HttpFileSystem fs, @@ -71,8 +71,7 @@ public class HttpPath implements Path this.body = null; this.etag = null; this.future = null; - // TODO: Ati - do Configuration for this project, too - this.pollSeconds = 2; + this.longPolling = true; } HttpPath() @@ -82,7 +81,7 @@ public class HttpPath implements Path this.body = null; this.etag = null; this.future = null; - this.pollSeconds = 0; + this.longPolling = true; } @Override @@ -364,18 +363,23 @@ void watchBody() { requestBuilder = requestBuilder.headers("If-None-Match", etag, "Prefer", "wait=86400"); } - // TODO: Ati - handle poll interval for both long polling and normal polling HttpRequest request = requestBuilder.build(); System.out.println("HP watchBody path " + location + " request " + request + " etag " + etag); // TODO: Ati //CompletableFuture future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + //future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) .thenAccept(this::acceptResponse) .exceptionally(this::handleException); } - boolean isDone() + /*boolean isDone() { - return future.isDone(); + return future == null ? false : future.isDone(); + }*/ + + boolean longPolling() + { + return longPolling; } /*HttpResponse poll() throws Exception @@ -391,19 +395,21 @@ private void acceptResponse( System.out.println("HP acceptResponse response: " + response); // TODO: Ati System.out.println("HP acceptResponse response.headers: " + response.headers()); // TODO: Ati //System.out.println("HWS handleChange response.body: " + new String(response.body())); // TODO: Ati - HttpPath path = (HttpPath) Path.of(response.request().uri()); + //HttpPath path = (HttpPath) Path.of(response.request().uri()); int statusCode = response.statusCode(); - int pollSeconds = 0; + //int pollSeconds = 0; + //boolean longPolling = this.longPolling; if (statusCode == 404) { body = EMPTY_BODY; watchKey.addEvent(ENTRY_MODIFY, this); - pollSeconds = this.pollSeconds; + //longPolling = false; // TODO: Ati - ? + //pollSeconds = this.pollSeconds; } else if (statusCode >= 500 && statusCode <= 599) { body = null; - pollSeconds = this.pollSeconds; + //pollSeconds = this.pollSeconds; } else { @@ -419,11 +425,11 @@ else if (statusCode >= 500 && statusCode <= 599) //etags.put(path, newEtag); // TODO: Ati etag = newEtag; watchKey.addEvent(ENTRY_MODIFY, this); - } else if (response.statusCode() != 304) { - pollSeconds = this.pollSeconds; + longPolling = false; // TODO: Ati + //pollSeconds = this.pollSeconds; } } else @@ -438,12 +444,12 @@ else if (response.statusCode() != 304) watchKey.addEvent(ENTRY_MODIFY, this); //addEvent(path); } - pollSeconds = this.pollSeconds; + //pollSeconds = this.pollSeconds; } } //futures.remove(path); //scheduleRequest(path, pollSeconds); // ??? - try + /*try { Thread.sleep(pollSeconds * 1000); // TODO: Ati } @@ -451,7 +457,8 @@ else if (response.statusCode() != 304) { throw new RuntimeException(e); } - watchBody(); + watchBody();*/ + watchKey.watchBody(); } private Void handleException( diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java index 1b1c623e26..d631269f85 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -28,13 +28,13 @@ import java.util.LinkedList; import java.util.List; import java.util.Objects; -import java.util.concurrent.Callable; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -public class HttpWatchService implements WatchService, Callable +//public class HttpWatchService implements WatchService, Callable +public class HttpWatchService implements WatchService { /*private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() .version(HTTP_2) @@ -59,7 +59,7 @@ public class HttpWatchService implements WatchService, Callable //private final Map> futures; private final MessageDigest md5; - //private int pollSeconds; + private int pollSeconds; private volatile boolean closed; public HttpWatchService( @@ -75,16 +75,16 @@ public HttpWatchService( //this.hashes = new ConcurrentHashMap<>(); //this.futures = new ConcurrentHashMap<>(); this.md5 = initMessageDigest("MD5"); - //this.pollSeconds = 30; - //this.pollSeconds = 2; + this.pollSeconds = 30; + //this.pollSeconds = 2; // TODO: Ati this.closed = false; //executor.submit(this); } - public void start() + /*public void start() { executor.submit(this); - } + }*/ // @Override // public Void call() throws Exception @@ -112,10 +112,10 @@ public void start() // return null; // } - @Override + /*@Override public Void call() throws Exception { - for (HttpWatchKey watchKey : watchKeys) // TODO: Ati watchKeys should be a List + for (HttpWatchKey watchKey : watchKeys) { HttpPath path = watchKey.watchable(); if (path.isDone()) @@ -123,13 +123,15 @@ public Void call() throws Exception //HttpResponse response = path.poll(); //watchKey.handleChange(response); //handleChange(response); // this throws Exception if error // this should be in HttpWatchKey ?? - watchKey.watchBody(); + //scheduleRequest(watchKey); + System.out.println("HWS path is done " + path); + //watchKey.watchBody(); //path.watchBody(); // scheduleRequest can be deleted // actually watchKey.watchBody() that calls path.watchBody() // how to deal with poll interval } } return null; - } + }*/ @Override public void close() @@ -139,6 +141,7 @@ public void close() //fileSystem.body(null); // TODO: Ati - body should be moved from HFS to HttpPath pendingKeys.clear(); pendingKeys.offer(closeKey); + watchKeys.forEach(w -> w.cancel()); //futures.values().forEach(future -> future.cancel(true)); //pathQueue.add(CLOSE_PATH); watchKeys.clear(); @@ -173,11 +176,12 @@ public WatchKey take() throws InterruptedException return key; } - /*public void pollSeconds( + // TODO: Ati + public void pollSeconds( int pollSeconds) { this.pollSeconds = pollSeconds; - }*/ + } private void checkOpen() { @@ -347,14 +351,6 @@ else if (response.statusCode() != 304) } }*/ - /*private void scheduleRequest___NEW( - //Path path, - HttpPath path, - int pollSeconds) - { - - }*/ - private byte[] computeHash( byte[] body) { @@ -414,7 +410,7 @@ public boolean reset() @Override public void cancel() { - watchKeys.remove(path); + watchKeys.remove(this); valid = false; } @@ -454,9 +450,24 @@ public int hashCode() return Objects.hashCode(path); } - void watchBody() + /*void watchBody() { path.watchBody(); + }*/ + + void watchBody() + { + if (valid) + { + if (path.longPolling()) + { + path.watchBody(); + } + else + { + executor.schedule(path::watchBody, pollSeconds, TimeUnit.SECONDS); + } + } } /*private void handleChange( diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java index 6d57ef51c7..f2e4a49df7 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java @@ -205,7 +205,7 @@ public void shouldWatch() throws Exception String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - //watchService.pollSeconds(1); // TODO: Ati + watchService.pollSeconds(1); // TODO: Ati path.register(watchService); // WHEN @@ -238,14 +238,14 @@ public void shouldWatchReadString() throws Exception String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - //watchService.pollSeconds(1); // TODO: Ati + watchService.pollSeconds(1); // TODO: Ati path.register(watchService); // WHEN k3po.start(); watchService.take(); - String body1 = Files.readString(path); k3po.notifyBarrier("CHANGED"); + String body1 = Files.readString(path); watchService.take(); String body2 = Files.readString(path); watchService.close(); @@ -256,7 +256,6 @@ public void shouldWatchReadString() throws Exception assertThat(body2, equalTo("Hello Universe!")); } - @Ignore // TODO: Ati - flaky @Test @Specification({ "${app}/watch.error.success/server", @@ -267,7 +266,7 @@ public void shouldWatchErrorSuccess() throws Exception String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - //watchService.pollSeconds(1); // TODO: Ati + watchService.pollSeconds(1); // TODO: Ati path.register(watchService); // WHEN @@ -300,7 +299,7 @@ public void shouldWatchErrorSuccessReadString() throws Exception String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - //watchService.pollSeconds(1); // TODO: Ati + watchService.pollSeconds(1); // TODO: Ati path.register(watchService); // WHEN From 31b420cba06dc3b0bdd76e2b0a83735ce5589064 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Fri, 21 Jun 2024 09:28:32 +0200 Subject: [PATCH 34/43] WIP hfs watchBody 3 --- .../runtime/filesystem/http/HttpPath.java | 10 ++- .../filesystem/http/HttpFileSystemIT.java | 85 +++++++++++-------- .../client.rpt | 0 .../server.rpt | 0 .../{notfound => read.notfound}/client.rpt | 0 .../{notfound => read.notfound}/server.rpt | 0 .../client.rpt | 0 .../server.rpt | 0 .../client.rpt | 0 .../server.rpt | 0 .../{success => read.success}/client.rpt | 0 .../{success => read.success}/server.rpt | 0 .../client.rpt | 0 .../server.rpt | 1 + .../watch.read.notfound.success/client.rpt | 41 +++++++++ .../watch.read.notfound.success/server.rpt | 79 +++++++++++++++++ .../http/application/watch.read/client.rpt | 41 +++++++++ .../http/application/watch.read/server.rpt | 83 ++++++++++++++++++ .../http/application/watch/server.rpt | 3 +- .../specs/filesystem/http/ApplicationIT.java | 50 ++++++++--- 20 files changed, 340 insertions(+), 53 deletions(-) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{notfound.success => read.notfound.success}/client.rpt (100%) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{notfound.success => read.notfound.success}/server.rpt (100%) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{notfound => read.notfound}/client.rpt (100%) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{notfound => read.notfound}/server.rpt (100%) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{success.etag.modified => read.success.etag.modified}/client.rpt (100%) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{success.etag.modified => read.success.etag.modified}/server.rpt (100%) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{success.etag.not.modified => read.success.etag.not.modified}/client.rpt (100%) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{success.etag.not.modified => read.success.etag.not.modified}/server.rpt (100%) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{success => read.success}/client.rpt (100%) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{success => read.success}/server.rpt (100%) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{watch.error.success => watch.notfound.success}/client.rpt (100%) rename specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/{watch.error.success => watch.notfound.success}/server.rpt (97%) create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/client.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/server.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt create mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/server.rpt diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index fbcd7ca4af..7595f6e846 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -325,6 +325,7 @@ private byte[] readBody() HttpRequest request = requestBuilder.build(); System.out.println("HP readBody path " + location + " request " + request + " etag " + etag); // TODO: Ati HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofByteArray()); + System.out.println("HP readBody response " + response); // TODO: Ati //System.out.println("HP readBody response.body " + new String(response.body())); // TODO: Ati if (response.statusCode() == HTTP_OK) { @@ -343,6 +344,7 @@ else if (response.statusCode() == HTTP_NOT_FOUND) } else if (response.statusCode() == HTTP_NOT_MODIFIED) { + System.out.println("HP readBody response 304 body " + new String(body)); // no op } } @@ -403,18 +405,19 @@ private void acceptResponse( { body = EMPTY_BODY; watchKey.addEvent(ENTRY_MODIFY, this); - //longPolling = false; // TODO: Ati - ? + longPolling = false; // TODO: Ati - ? //pollSeconds = this.pollSeconds; } else if (statusCode >= 500 && statusCode <= 599) { body = null; + longPolling = false; // TODO: Ati - ? //pollSeconds = this.pollSeconds; } else { //byte[] body = response.body(); - this.body = response.body(); + System.out.println("HP acceptResponse body " + new String(response.body())); Optional etagOptional = response.headers().firstValue("Etag"); if (etagOptional.isPresent()) { @@ -424,10 +427,13 @@ else if (statusCode >= 500 && statusCode <= 599) { //etags.put(path, newEtag); // TODO: Ati etag = newEtag; + this.body = response.body(); + //longPolling = true; // TODO: Ati watchKey.addEvent(ENTRY_MODIFY, this); } else if (response.statusCode() != 304) { + this.body = response.body(); longPolling = false; // TODO: Ati //pollSeconds = this.pollSeconds; } diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java index f2e4a49df7..3b99d3315a 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java @@ -52,7 +52,7 @@ public class HttpFileSystemIT @Test @Specification({ - "${app}/success/server", + "${app}/read.success/server", }) public void shouldReadString() throws Exception { @@ -71,7 +71,7 @@ public void shouldReadString() throws Exception @Test @Specification({ - "${app}/success.etag.not.modified/server", + "${app}/read.success.etag.not.modified/server", }) public void shouldReadStringEtagNotModified() throws Exception { @@ -92,7 +92,7 @@ public void shouldReadStringEtagNotModified() throws Exception @Test @Specification({ - "${app}/success.etag.modified/server", + "${app}/read.success.etag.modified/server", }) public void shouldReadStringEtagModified() throws Exception { @@ -113,7 +113,7 @@ public void shouldReadStringEtagModified() throws Exception @Test @Specification({ - "${app}/notfound/server", + "${app}/read.notfound/server", }) public void shouldReadStringNotFound() throws Exception { @@ -132,7 +132,7 @@ public void shouldReadStringNotFound() throws Exception @Test @Specification({ - "${app}/notfound.success/server", + "${app}/read.notfound.success/server", }) public void shouldReadStringNotFoundSuccess() throws Exception { @@ -153,7 +153,7 @@ public void shouldReadStringNotFoundSuccess() throws Exception @Test @Specification({ - "${app}/success/server", + "${app}/read.success/server", }) public void shouldReadInputStream() throws Exception { @@ -175,7 +175,7 @@ public void shouldReadInputStream() throws Exception @Test @Specification({ - "${app}/notfound/server", + "${app}/read.notfound/server", }) public void shouldReadInputStreamNotFound() throws Exception { @@ -206,13 +206,14 @@ public void shouldWatch() throws Exception Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); watchService.pollSeconds(1); // TODO: Ati - path.register(watchService); // WHEN k3po.start(); + k3po.notifyBarrier("REGISTERED"); + path.register(watchService); WatchKey key1 = watchService.take(); List> events1 = key1.pollEvents(); - k3po.notifyBarrier("CHANGED"); + k3po.notifyBarrier("MODIFIED"); WatchKey key2 = watchService.take(); List> events2 = key2.pollEvents(); watchService.close(); @@ -227,73 +228,78 @@ public void shouldWatch() throws Exception assertThat(events2.get(0).context(), equalTo(path)); } - @Ignore @Test @Specification({ - "${app}/watch.read/server", + "${app}/watch.notfound.success/server", }) - public void shouldWatchReadString() throws Exception + public void shouldWatchNotfoundSuccess() throws Exception { // GIVEN String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); watchService.pollSeconds(1); // TODO: Ati - path.register(watchService); // WHEN k3po.start(); - watchService.take(); - k3po.notifyBarrier("CHANGED"); - String body1 = Files.readString(path); - watchService.take(); - String body2 = Files.readString(path); + k3po.notifyBarrier("REGISTERED"); + path.register(watchService); + WatchKey key1 = watchService.take(); + List> events1 = key1.pollEvents(); + k3po.notifyBarrier("FOUND"); + WatchKey key2 = watchService.take(); + List> events2 = key2.pollEvents(); watchService.close(); k3po.finish(); // THEN - assertThat(body1, equalTo("Hello World!")); - assertThat(body2, equalTo("Hello Universe!")); + assertThat(events1.size(), equalTo(1)); + assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); + assertThat(events1.get(0).context(), equalTo(path)); + assertThat(events2.size(), equalTo(1)); + assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); + assertThat(events2.get(0).context(), equalTo(path)); } + @Ignore // TODO: Ati @Test @Specification({ - "${app}/watch.error.success/server", + "${app}/watch.read/server", }) - public void shouldWatchErrorSuccess() throws Exception + public void shouldWatchRead() throws Exception { // GIVEN String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); watchService.pollSeconds(1); // TODO: Ati - path.register(watchService); // WHEN k3po.start(); - WatchKey key1 = watchService.take(); - List> events1 = key1.pollEvents(); - k3po.notifyBarrier("FOUND"); - WatchKey key2 = watchService.take(); - List> events2 = key2.pollEvents(); + k3po.notifyBarrier("FIRST_READ"); + String body1 = Files.readString(path); + k3po.notifyBarrier("REGISTERED"); + path.register(watchService); + k3po.notifyBarrier("MODIFIED"); + watchService.take(); + k3po.notifyBarrier("NOT_MODIFIED"); + watchService.take(); + k3po.notifyBarrier("SECOND_READ"); + String body2 = Files.readString(path); watchService.close(); k3po.finish(); // THEN - assertThat(events1.size(), equalTo(1)); - assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); - assertThat(events1.get(0).context(), equalTo(path)); - assertThat(events2.size(), equalTo(1)); - assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); - assertThat(events2.get(0).context(), equalTo(path)); + assertThat(body1, equalTo("Hello World!")); + assertThat(body2, equalTo("Hello Universe!")); } @Ignore // TODO: Ati @Test @Specification({ - "${app}/watch.error.success/server", + "${app}/watch.read.notfound.success/server", }) - public void shouldWatchErrorSuccessReadString() throws Exception + public void shouldWatchReadNotFoundSuccess() throws Exception { // GIVEN String url = "http://localhost:8080/hello.txt"; @@ -304,10 +310,15 @@ public void shouldWatchErrorSuccessReadString() throws Exception // WHEN k3po.start(); - watchService.take(); + k3po.notifyBarrier("FIRST_READ"); String body1 = Files.readString(path); + k3po.notifyBarrier("REGISTERED"); + path.register(watchService); k3po.notifyBarrier("FOUND"); watchService.take(); + k3po.notifyBarrier("NOT_MODIFIED"); + watchService.take(); + k3po.notifyBarrier("SECOND_READ"); String body2 = Files.readString(path); watchService.close(); k3po.finish(); diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.notfound.success/client.rpt similarity index 100% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/client.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.notfound.success/client.rpt diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.notfound.success/server.rpt similarity index 100% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound.success/server.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.notfound.success/server.rpt diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.notfound/client.rpt similarity index 100% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/client.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.notfound/client.rpt diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.notfound/server.rpt similarity index 100% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/notfound/server.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.notfound/server.rpt diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success.etag.modified/client.rpt similarity index 100% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/client.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success.etag.modified/client.rpt diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success.etag.modified/server.rpt similarity index 100% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.modified/server.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success.etag.modified/server.rpt diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success.etag.not.modified/client.rpt similarity index 100% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/client.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success.etag.not.modified/client.rpt diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success.etag.not.modified/server.rpt similarity index 100% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success.etag.not.modified/server.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success.etag.not.modified/server.rpt diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success/client.rpt similarity index 100% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/client.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success/client.rpt diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success/server.rpt similarity index 100% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/success/server.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/read.success/server.rpt diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/client.rpt similarity index 100% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/client.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/client.rpt diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/server.rpt similarity index 97% rename from specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/server.rpt rename to specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/server.rpt index aa5072487c..bc5d4bca63 100644 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.error.success/server.rpt +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/server.rpt @@ -20,6 +20,7 @@ connected read http:method "GET" read closed +write await REGISTERED write http:status "404" "Not Found" write http:content-length write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/client.rpt new file mode 100644 index 0000000000..cd91a9ea08 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/client.rpt @@ -0,0 +1,41 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write close + +read http:status "200" "OK" +read http:header "Etag" "AAAAAAA" +read "Hello World!" +read notify FIRST_READ +read closed + +connect "http://localhost:8080/hello.txt" +connected + +write await FIRST_READ +write http:method "GET" +write http:header "If-None-Match" "AAAAAAA" +write http:header "Prefer" "wait=86400" +write close + +read notify CHANGED +read http:status "200" "OK" +read http:header "Etag" "BBBBBBB" +read "Hello Universe!" +read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/server.rpt new file mode 100644 index 0000000000..a6d4a736c8 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/server.rpt @@ -0,0 +1,79 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "http://localhost:8080/hello.txt" +accepted +connected + +read http:method "GET" +read closed + +write await FIRST_READ +write http:status "404" "Not Found" +write http:content-length +write close + +accepted +connected + +read http:method "GET" +#read http:header "If-None-Match" "AAAAAAA" +#read http:header "Prefer" "wait=86400" +read closed + +write await REGISTERED +write http:status "404" "Not Found" +write http:content-length +write close + +accepted +connected + +read http:method "GET" +#read http:header "If-None-Match" "AAAAAAA" +#read http:header "Prefer" "wait=86400" +read closed + +write await FOUND +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "AAAAAAA" +write "Hello World!" +write close + +accepted +connected + +read http:method "GET" +read http:header "If-None-Match" "AAAAAAA" +read http:header "Prefer" "wait=86400" +read closed + +write await NOT_MODIFIED +write http:status "304" "Not changed" +write http:content-length +write close + +accepted +connected + +read http:method "GET" +read http:header "If-None-Match" "AAAAAAA" +read closed + +write await SECOND_READ +write http:status "304" "Not changed" +write http:content-length +write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt new file mode 100644 index 0000000000..cd91a9ea08 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt @@ -0,0 +1,41 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write close + +read http:status "200" "OK" +read http:header "Etag" "AAAAAAA" +read "Hello World!" +read notify FIRST_READ +read closed + +connect "http://localhost:8080/hello.txt" +connected + +write await FIRST_READ +write http:method "GET" +write http:header "If-None-Match" "AAAAAAA" +write http:header "Prefer" "wait=86400" +write close + +read notify CHANGED +read http:status "200" "OK" +read http:header "Etag" "BBBBBBB" +read "Hello Universe!" +read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/server.rpt new file mode 100644 index 0000000000..4ca89ac7f3 --- /dev/null +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/server.rpt @@ -0,0 +1,83 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "http://localhost:8080/hello.txt" +accepted +connected + +read http:method "GET" +read closed + +write await FIRST_READ +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "AAAAAAA" +write "Hello World!" +write close + +accepted +connected + +read http:method "GET" +read http:header "If-None-Match" "AAAAAAA" +read http:header "Prefer" "wait=86400" +read closed + +write await REGISTERED +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "AAAAAAA" +write "Hello World!" +write close + +accepted +connected + +read http:method "GET" +read http:header "If-None-Match" "AAAAAAA" +read http:header "Prefer" "wait=86400" +read closed + +write await MODIFIED +write http:status "200" "OK" +write http:content-length +write http:header "Etag" "BBBBBBB" +write "Hello Universe!" +write close + +accepted +connected + +read http:method "GET" +read http:header "If-None-Match" "BBBBBBB" +read http:header "Prefer" "wait=86400" +read closed + +write await NOT_MODIFIED +write http:status "304" "Not changed" +write http:content-length +write close + +accepted +connected + +read http:method "GET" +read http:header "If-None-Match" "BBBBBBB" +read closed + +write await SECOND_READ +write http:status "304" "Not changed" +write http:content-length +write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/server.rpt index 19eed58ea1..5b330c1ffc 100644 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/server.rpt +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/server.rpt @@ -20,6 +20,7 @@ connected read http:method "GET" read closed +write await REGISTERED write http:status "200" "OK" write http:content-length write http:header "Etag" "AAAAAAA" @@ -34,7 +35,7 @@ read http:header "If-None-Match" "AAAAAAA" read http:header "Prefer" "wait=86400" read closed -write await CHANGED +write await MODIFIED write http:status "200" "OK" write http:content-length write http:header "Etag" "BBBBBBB" diff --git a/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java index 576ac7ff15..e2dd468fc5 100644 --- a/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java +++ b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java @@ -17,6 +17,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.rules.RuleChain.outerRule; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.DisableOnDebug; @@ -38,8 +39,8 @@ public class ApplicationIT @Test @Specification({ - "${app}/success/client", - "${app}/success/server" }) + "${app}/read.success/client", + "${app}/read.success/server" }) public void shouldReadString() throws Exception { k3po.finish(); @@ -47,8 +48,8 @@ public void shouldReadString() throws Exception @Test @Specification({ - "${app}/success.etag.not.modified/client", - "${app}/success.etag.not.modified/server" }) + "${app}/read.success.etag.not.modified/client", + "${app}/read.success.etag.not.modified/server" }) public void shouldReadStringEtagNotModified() throws Exception { k3po.finish(); @@ -56,8 +57,8 @@ public void shouldReadStringEtagNotModified() throws Exception @Test @Specification({ - "${app}/success.etag.modified/client", - "${app}/success.etag.modified/server" }) + "${app}/read.success.etag.modified/client", + "${app}/read.success.etag.modified/server" }) public void shouldReadStringEtagModified() throws Exception { k3po.finish(); @@ -65,8 +66,8 @@ public void shouldReadStringEtagModified() throws Exception @Test @Specification({ - "${app}/notfound/client", - "${app}/notfound/server" }) + "${app}/read.notfound/client", + "${app}/read.notfound/server" }) public void shouldReadStringNotFound() throws Exception { k3po.finish(); @@ -74,13 +75,14 @@ public void shouldReadStringNotFound() throws Exception @Test @Specification({ - "${app}/notfound.success/client", - "${app}/notfound.success/server" }) + "${app}/read.notfound.success/client", + "${app}/read.notfound.success/server" }) public void shouldReadStringNotFoundSuccess() throws Exception { k3po.finish(); } + @Ignore // TODO: Ati @Test @Specification({ "${app}/watch/client", @@ -90,12 +92,34 @@ public void shouldWatch() throws Exception k3po.finish(); } + @Ignore // TODO: Ati @Test @Specification({ - "${app}/watch.error.success/client", - "${app}/watch.error.success/server" }) - public void shouldWatchErrorSuccess() throws Exception + "${app}/watch.notfound.success/client", + "${app}/watch.notfound.success/server" }) + public void shouldWatchNotFoundSuccess() throws Exception { k3po.finish(); } + + @Ignore // TODO: Ati + @Test + @Specification({ + "${app}/watch.read/client", + "${app}/watch.read/server" }) + public void shouldWatchRead() throws Exception + { + k3po.finish(); + } + + @Ignore // TODO: Ati + @Test + @Specification({ + "${app}/watch.read.notfound.success/client", + "${app}/watch.read.notfound.success/server" }) + public void shouldWatchReadNotFoundSuccess() throws Exception + { + k3po.finish(); + } + } From d3109f7005350ee603128d2e25d4c79bc6f63145 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Fri, 21 Jun 2024 10:38:53 +0200 Subject: [PATCH 35/43] WIP hfs watchBody 4 --- .../zilla/runtime/filesystem/http/HttpPath.java | 1 - .../runtime/filesystem/http/HttpWatchService.java | 12 ++++-------- .../runtime/filesystem/http/HttpFileSystemIT.java | 4 ++-- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index 7595f6e846..a06299a1b3 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -242,7 +242,6 @@ public WatchKey register( throw new ProviderMismatchException(); } watchKey = ((HttpWatchService) watcher).register(this, events, modifiers); - watchKey.watchBody(); return watchKey; } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java index d631269f85..8e90c61418 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -141,7 +141,7 @@ public void close() //fileSystem.body(null); // TODO: Ati - body should be moved from HFS to HttpPath pendingKeys.clear(); pendingKeys.offer(closeKey); - watchKeys.forEach(w -> w.cancel()); + watchKeys.forEach(HttpWatchKey::cancel); //futures.values().forEach(future -> future.cancel(true)); //pathQueue.add(CLOSE_PATH); watchKeys.clear(); @@ -207,7 +207,6 @@ private void enqueueKey( pendingKeys.offer(key); } - //WatchKey register( HttpWatchKey register( final HttpPath path, WatchEvent.Kind[] events, @@ -225,12 +224,9 @@ HttpWatchKey register( throw new IllegalArgumentException("Modifiers are not supported"); } System.out.printf("HWS register path: %s\n", path); // TODO: Ati - //WatchKey watchKey = watchKeys.computeIfAbsent(path, i -> new HttpWatchKey(path)); - //HttpWatchKey watchKey = watchKeys.computeIfAbsent(path, i -> new HttpWatchKey(path)); - HttpWatchKey watchKey = new HttpWatchKey(path); // I moved this to HP - watchKeys.add(watchKey); // TODO: Ati - watchKeys should be a List - //pathQueue.offer(path); - //pathQueue.add(path); + HttpWatchKey watchKey = new HttpWatchKey(path); + watchKey.watchBody(); + watchKeys.add(watchKey); return watchKey; } diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java index 3b99d3315a..317b24d95a 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java @@ -228,11 +228,12 @@ public void shouldWatch() throws Exception assertThat(events2.get(0).context(), equalTo(path)); } + @Ignore // TODO: Ati @Test @Specification({ "${app}/watch.notfound.success/server", }) - public void shouldWatchNotfoundSuccess() throws Exception + public void shouldWatchNotFoundSuccess() throws Exception { // GIVEN String url = "http://localhost:8080/hello.txt"; @@ -261,7 +262,6 @@ public void shouldWatchNotfoundSuccess() throws Exception assertThat(events2.get(0).context(), equalTo(path)); } - @Ignore // TODO: Ati @Test @Specification({ "${app}/watch.read/server", From 3198b109196ac22cbc6d53132b48cf349dadf225 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Fri, 21 Jun 2024 11:20:59 +0200 Subject: [PATCH 36/43] WIP hfs watchBody 5 --- .../runtime/filesystem/http/HttpPath.java | 6 ++ .../filesystem/http/HttpWatchService.java | 1 + .../filesystem/http/HttpFileSystemIT.java | 69 ---------------- .../watch.notfound.success/client.rpt | 39 --------- .../watch.notfound.success/server.rpt | 39 --------- .../watch.read.notfound.success/client.rpt | 41 ---------- .../watch.read.notfound.success/server.rpt | 79 ------------------- .../http/application/watch.read/client.rpt | 42 +++++++++- .../http/application/watch/client.rpt | 5 +- .../specs/filesystem/http/ApplicationIT.java | 24 ------ 10 files changed, 48 insertions(+), 297 deletions(-) delete mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/client.rpt delete mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/server.rpt delete mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/client.rpt delete mode 100644 specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/server.rpt diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index a06299a1b3..8da3af7385 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -383,6 +383,12 @@ boolean longPolling() return longPolling; } + void cancel() + { + System.out.println("HP cancel"); // TODO: Ati + future.cancel(true); + } + /*HttpResponse poll() throws Exception { future.get(); diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java index 8e90c61418..ebb103eff0 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -407,6 +407,7 @@ public boolean reset() public void cancel() { watchKeys.remove(this); + path.cancel(); valid = false; } diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java index 317b24d95a..579eaf98f3 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java @@ -29,7 +29,6 @@ import java.nio.file.spi.FileSystemProvider; import java.util.List; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.DisableOnDebug; @@ -228,40 +227,6 @@ public void shouldWatch() throws Exception assertThat(events2.get(0).context(), equalTo(path)); } - @Ignore // TODO: Ati - @Test - @Specification({ - "${app}/watch.notfound.success/server", - }) - public void shouldWatchNotFoundSuccess() throws Exception - { - // GIVEN - String url = "http://localhost:8080/hello.txt"; - Path path = Path.of(new URI(url)); - HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - watchService.pollSeconds(1); // TODO: Ati - - // WHEN - k3po.start(); - k3po.notifyBarrier("REGISTERED"); - path.register(watchService); - WatchKey key1 = watchService.take(); - List> events1 = key1.pollEvents(); - k3po.notifyBarrier("FOUND"); - WatchKey key2 = watchService.take(); - List> events2 = key2.pollEvents(); - watchService.close(); - k3po.finish(); - - // THEN - assertThat(events1.size(), equalTo(1)); - assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); - assertThat(events1.get(0).context(), equalTo(path)); - assertThat(events2.size(), equalTo(1)); - assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); - assertThat(events2.get(0).context(), equalTo(path)); - } - @Test @Specification({ "${app}/watch.read/server", @@ -293,38 +258,4 @@ public void shouldWatchRead() throws Exception assertThat(body1, equalTo("Hello World!")); assertThat(body2, equalTo("Hello Universe!")); } - - @Ignore // TODO: Ati - @Test - @Specification({ - "${app}/watch.read.notfound.success/server", - }) - public void shouldWatchReadNotFoundSuccess() throws Exception - { - // GIVEN - String url = "http://localhost:8080/hello.txt"; - Path path = Path.of(new URI(url)); - HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - watchService.pollSeconds(1); // TODO: Ati - path.register(watchService); - - // WHEN - k3po.start(); - k3po.notifyBarrier("FIRST_READ"); - String body1 = Files.readString(path); - k3po.notifyBarrier("REGISTERED"); - path.register(watchService); - k3po.notifyBarrier("FOUND"); - watchService.take(); - k3po.notifyBarrier("NOT_MODIFIED"); - watchService.take(); - k3po.notifyBarrier("SECOND_READ"); - String body2 = Files.readString(path); - watchService.close(); - k3po.finish(); - - // THEN - assertThat(body1, equalTo("")); - assertThat(body2, equalTo("Hello World!")); - } } diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/client.rpt deleted file mode 100644 index 79cf7c7d1f..0000000000 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/client.rpt +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2021-2023 Aklivity Inc -# -# Licensed under the Aklivity Community License (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy of the -# License at -# -# https://www.aklivity.io/aklivity-community-license/ -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# - -connect "http://localhost:8080/hello.txt" -connected - -write http:method "GET" -write close - -read http:status "404" "Not Found" -read notify FIRST_READ -read closed - -connect "http://localhost:8080/hello.txt" -connected - -write await FIRST_READ -write http:method "GET" -write http:header "If-None-Match" "AAAAAAA" -write http:header "Prefer" "wait=86400" -write close - -read notify FOUND -read http:status "200" "OK" -read http:header "Etag" "AAAAAAA" -read "Hello World!" -read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/server.rpt deleted file mode 100644 index bc5d4bca63..0000000000 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.notfound.success/server.rpt +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2021-2023 Aklivity Inc -# -# Licensed under the Aklivity Community License (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy of the -# License at -# -# https://www.aklivity.io/aklivity-community-license/ -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# - -accept "http://localhost:8080/hello.txt" -accepted -connected - -read http:method "GET" -read closed - -write await REGISTERED -write http:status "404" "Not Found" -write http:content-length -write close - -accepted -connected - -read http:method "GET" -read closed - -write await FOUND -write http:status "200" "OK" -write http:content-length -write http:header "Etag" "AAAAAAA" -write "Hello World!" -write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/client.rpt deleted file mode 100644 index cd91a9ea08..0000000000 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/client.rpt +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright 2021-2023 Aklivity Inc -# -# Licensed under the Aklivity Community License (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy of the -# License at -# -# https://www.aklivity.io/aklivity-community-license/ -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# - -connect "http://localhost:8080/hello.txt" -connected - -write http:method "GET" -write close - -read http:status "200" "OK" -read http:header "Etag" "AAAAAAA" -read "Hello World!" -read notify FIRST_READ -read closed - -connect "http://localhost:8080/hello.txt" -connected - -write await FIRST_READ -write http:method "GET" -write http:header "If-None-Match" "AAAAAAA" -write http:header "Prefer" "wait=86400" -write close - -read notify CHANGED -read http:status "200" "OK" -read http:header "Etag" "BBBBBBB" -read "Hello Universe!" -read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/server.rpt deleted file mode 100644 index a6d4a736c8..0000000000 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read.notfound.success/server.rpt +++ /dev/null @@ -1,79 +0,0 @@ -# -# Copyright 2021-2023 Aklivity Inc -# -# Licensed under the Aklivity Community License (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy of the -# License at -# -# https://www.aklivity.io/aklivity-community-license/ -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# - -accept "http://localhost:8080/hello.txt" -accepted -connected - -read http:method "GET" -read closed - -write await FIRST_READ -write http:status "404" "Not Found" -write http:content-length -write close - -accepted -connected - -read http:method "GET" -#read http:header "If-None-Match" "AAAAAAA" -#read http:header "Prefer" "wait=86400" -read closed - -write await REGISTERED -write http:status "404" "Not Found" -write http:content-length -write close - -accepted -connected - -read http:method "GET" -#read http:header "If-None-Match" "AAAAAAA" -#read http:header "Prefer" "wait=86400" -read closed - -write await FOUND -write http:status "200" "OK" -write http:content-length -write http:header "Etag" "AAAAAAA" -write "Hello World!" -write close - -accepted -connected - -read http:method "GET" -read http:header "If-None-Match" "AAAAAAA" -read http:header "Prefer" "wait=86400" -read closed - -write await NOT_MODIFIED -write http:status "304" "Not changed" -write http:content-length -write close - -accepted -connected - -read http:method "GET" -read http:header "If-None-Match" "AAAAAAA" -read closed - -write await SECOND_READ -write http:status "304" "Not changed" -write http:content-length -write close diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt index cd91a9ea08..e74045a90c 100644 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt @@ -19,23 +19,59 @@ connected write http:method "GET" write close +read notify FIRST_READ +read http:status "200" "OK" +read http:header "Etag" "AAAAAAA" +read "Hello World!" +read closed + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write http:header "If-None-Match" "AAAAAAA" +write http:header "Prefer" "wait=86400" +write close + +read notify REGISTERED read http:status "200" "OK" read http:header "Etag" "AAAAAAA" read "Hello World!" -read notify FIRST_READ read closed connect "http://localhost:8080/hello.txt" connected -write await FIRST_READ write http:method "GET" write http:header "If-None-Match" "AAAAAAA" write http:header "Prefer" "wait=86400" write close -read notify CHANGED +read notify MODIFIED read http:status "200" "OK" read http:header "Etag" "BBBBBBB" read "Hello Universe!" read closed + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write http:header "If-None-Match" "BBBBBBB" +write http:header "Prefer" "wait=86400" +write close + +read notify NOT_MODIFIED +read http:status "304" "Not changed" +read closed + +connect "http://localhost:8080/hello.txt" +connected + +write http:method "GET" +write http:header "If-None-Match" "BBBBBBB" +write close + +read notify SECOND_READ +read http:status "304" "Not changed" +read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt index cd91a9ea08..28bac19a0b 100644 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch/client.rpt @@ -19,22 +19,21 @@ connected write http:method "GET" write close +read notify REGISTERED read http:status "200" "OK" read http:header "Etag" "AAAAAAA" read "Hello World!" -read notify FIRST_READ read closed connect "http://localhost:8080/hello.txt" connected -write await FIRST_READ write http:method "GET" write http:header "If-None-Match" "AAAAAAA" write http:header "Prefer" "wait=86400" write close -read notify CHANGED +read notify MODIFIED read http:status "200" "OK" read http:header "Etag" "BBBBBBB" read "Hello Universe!" diff --git a/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java index e2dd468fc5..54003f25fe 100644 --- a/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java +++ b/specs/filesystem-http.spec/src/test/java/io/aklivity/zilla/specs/filesystem/http/ApplicationIT.java @@ -17,7 +17,6 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.rules.RuleChain.outerRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.DisableOnDebug; @@ -82,7 +81,6 @@ public void shouldReadStringNotFoundSuccess() throws Exception k3po.finish(); } - @Ignore // TODO: Ati @Test @Specification({ "${app}/watch/client", @@ -92,17 +90,6 @@ public void shouldWatch() throws Exception k3po.finish(); } - @Ignore // TODO: Ati - @Test - @Specification({ - "${app}/watch.notfound.success/client", - "${app}/watch.notfound.success/server" }) - public void shouldWatchNotFoundSuccess() throws Exception - { - k3po.finish(); - } - - @Ignore // TODO: Ati @Test @Specification({ "${app}/watch.read/client", @@ -111,15 +98,4 @@ public void shouldWatchRead() throws Exception { k3po.finish(); } - - @Ignore // TODO: Ati - @Test - @Specification({ - "${app}/watch.read.notfound.success/client", - "${app}/watch.read.notfound.success/server" }) - public void shouldWatchReadNotFoundSuccess() throws Exception - { - k3po.finish(); - } - } From ef2d377a71d325c536bdd63b298f5232082c4311 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Fri, 21 Jun 2024 12:01:59 +0200 Subject: [PATCH 37/43] WIP hfs watchBody 6 --- .../http/AbstractHttpFileSystemProvider.java | 8 +- .../filesystem/http/HttpFileSystem.java | 21 +- .../runtime/filesystem/http/HttpPath.java | 103 ++----- .../filesystem/http/HttpWatchService.java | 277 +----------------- 4 files changed, 39 insertions(+), 370 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java index e8db435415..8269c9ede6 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java @@ -105,7 +105,7 @@ public InputStream newInputStream( OpenOption... options) { checkPath(path); - return new ByteArrayInputStream(resolveBody((HttpPath) path)); + return new ByteArrayInputStream(readBody((HttpPath) path)); } @Override @@ -142,7 +142,7 @@ public SeekableByteChannel newByteChannel( FileAttribute... attrs) { checkPath(path); - return new ReadOnlyByteArrayChannel(resolveBody((HttpPath) path)); + return new ReadOnlyByteArrayChannel(readBody((HttpPath) path)); } @Override @@ -310,9 +310,9 @@ private void checkPath( } } - private byte[] resolveBody( + private byte[] readBody( HttpPath path) { - return path.resolveBody(); + return path.readBody(); } } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java index 6e1a9f86de..704e90ae2b 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java @@ -33,15 +33,12 @@ public final class HttpFileSystem extends FileSystem private final AbstractHttpFileSystemProvider provider; private final URI root; - //private byte[] body; // TODO: Ati - body should be moved from HFS to HttpPath - HttpFileSystem( AbstractHttpFileSystemProvider provider, URI root) { this.provider = provider; this.root = root; - //this.body = null; // TODO: Ati - body should be moved from HFS to HttpPath } @Override @@ -129,10 +126,7 @@ public UserPrincipalLookupService getUserPrincipalLookupService() @Override public HttpWatchService newWatchService() { - /*HttpWatchService service = new HttpWatchService(this); - service.start(); - return service;*/ - return new HttpWatchService(this); + return new HttpWatchService(); } HttpPath resolveSibling( @@ -140,17 +134,4 @@ HttpPath resolveSibling( { return new HttpPath(this, root.resolve(URI.create(other))); } - - // TODO: Ati - body should be moved from HFS to HttpPath - /*byte[] body() - { - return body; - }*/ - - // TODO: Ati - body should be moved from HFS to HttpPath - void body( - byte[] body) - { - //this.body = body; - } } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index 8da3af7385..78be148a34 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -52,11 +52,10 @@ public class HttpPath implements Path private final URI location; private byte[] body; + private boolean longPolling; private String etag; - private CompletableFuture future; - //private HttpResponse response; private HttpWatchService.HttpWatchKey watchKey; - private boolean longPolling; + private CompletableFuture future; HttpPath( HttpFileSystem fs, @@ -68,9 +67,6 @@ public class HttpPath implements Path } this.fs = fs; this.location = location; - this.body = null; - this.etag = null; - this.future = null; this.longPolling = true; } @@ -78,9 +74,6 @@ public class HttpPath implements Path { this.fs = null; this.location = null; - this.body = null; - this.etag = null; - this.future = null; this.longPolling = true; } @@ -295,21 +288,9 @@ public int hashCode() return Objects.hashCode(location); } - byte[] resolveBody() - { - // TODO: Ati - if we always call readBody, can this be removed? - System.out.println("HP resolveBody"); - body = readBody(); - /*if (body == null) - { - body = readBody(); - }*/ - return body; - } - - private byte[] readBody() + byte[] readBody() { - //byte[] body = new byte[0]; + System.out.println("HP readBody"); // TODO: Ati try { HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() @@ -317,8 +298,6 @@ private byte[] readBody() .uri(location); if (etag != null && !etag.isEmpty()) { - //requestBuilder = requestBuilder.headers("If-None-Match", etag, "Prefer", "wait=86400"); - // TODO: Ati - this is a sync call, I guess we don't need Prefer wait requestBuilder = requestBuilder.headers("If-None-Match", etag); } HttpRequest request = requestBuilder.build(); @@ -343,7 +322,6 @@ else if (response.statusCode() == HTTP_NOT_FOUND) } else if (response.statusCode() == HTTP_NOT_MODIFIED) { - System.out.println("HP readBody response 304 body " + new String(body)); // no op } } @@ -366,80 +344,48 @@ void watchBody() } HttpRequest request = requestBuilder.build(); System.out.println("HP watchBody path " + location + " request " + request + " etag " + etag); // TODO: Ati - //CompletableFuture future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) - //future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) - .thenAccept(this::acceptResponse) + .thenAccept(this::handleResponse) .exceptionally(this::handleException); } - /*boolean isDone() - { - return future == null ? false : future.isDone(); - }*/ - - boolean longPolling() - { - return longPolling; - } - - void cancel() - { - System.out.println("HP cancel"); // TODO: Ati - future.cancel(true); - } - - /*HttpResponse poll() throws Exception - { - future.get(); - return response; - }*/ - - private void acceptResponse( + private void handleResponse( HttpResponse response) { - //this.response = response; - System.out.println("HP acceptResponse response: " + response); // TODO: Ati - System.out.println("HP acceptResponse response.headers: " + response.headers()); // TODO: Ati - //System.out.println("HWS handleChange response.body: " + new String(response.body())); // TODO: Ati - //HttpPath path = (HttpPath) Path.of(response.request().uri()); + System.out.println("HP handleResponse response: " + response); // TODO: Ati + System.out.println("HP handleResponse response.headers: " + response.headers()); // TODO: Ati int statusCode = response.statusCode(); //int pollSeconds = 0; - //boolean longPolling = this.longPolling; if (statusCode == 404) { body = EMPTY_BODY; watchKey.addEvent(ENTRY_MODIFY, this); - longPolling = false; // TODO: Ati - ? + longPolling = false; //pollSeconds = this.pollSeconds; } else if (statusCode >= 500 && statusCode <= 599) { body = null; - longPolling = false; // TODO: Ati - ? + longPolling = false; //pollSeconds = this.pollSeconds; } else { - //byte[] body = response.body(); - System.out.println("HP acceptResponse body " + new String(response.body())); + System.out.println("HP handleResponse body " + new String(response.body())); Optional etagOptional = response.headers().firstValue("Etag"); if (etagOptional.isPresent()) { - //String oldEtag = etags.getOrDefault(path, ""); // TODO: Ati String newEtag = etagOptional.get(); if (!newEtag.equals(etag)) { - //etags.put(path, newEtag); // TODO: Ati etag = newEtag; - this.body = response.body(); - //longPolling = true; // TODO: Ati + body = response.body(); watchKey.addEvent(ENTRY_MODIFY, this); } else if (response.statusCode() != 304) { this.body = response.body(); - longPolling = false; // TODO: Ati + longPolling = false; //pollSeconds = this.pollSeconds; } } @@ -458,25 +404,24 @@ else if (response.statusCode() != 304) //pollSeconds = this.pollSeconds; } } - //futures.remove(path); - //scheduleRequest(path, pollSeconds); // ??? - /*try - { - Thread.sleep(pollSeconds * 1000); // TODO: Ati - } - catch (InterruptedException e) - { - throw new RuntimeException(e); - } - watchBody();*/ watchKey.watchBody(); } + boolean longPolling() + { + return longPolling; + } + + void cancel() + { + System.out.println("HP cancel"); // TODO: Ati + future.cancel(true); + } + private Void handleException( Throwable throwable) { System.out.println("HP handleException " + throwable.getMessage()); // TODO: Ati - //scheduleRequest(path, pollSeconds); return null; } } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java index ebb103eff0..bb13fcc331 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -33,118 +33,40 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -//public class HttpWatchService implements WatchService, Callable public class HttpWatchService implements WatchService { - /*private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() - .version(HTTP_2) - .followRedirects(NORMAL) - .build();*/ - //private static final Path CLOSE_PATH = Path.of(URI.create("http://localhost:12345")); private static final HttpPath CLOSE_PATH = new HttpPath(); - private static final byte[] EMPTY_BODY = new byte[0]; private final WatchKey closeKey = new HttpWatchKey(CLOSE_PATH); - //private final HttpFileSystem fileSystem; // TODO: Ati - remove this private final ScheduledExecutorService executor; - private final LinkedBlockingQueue pendingKeys; - //private final BlockingQueue pathQueue; - //private final BlockingQueue pathQueue; - //private final Map watchKeys; - //private final Map watchKeys; private final List watchKeys; - //private final Map etags; // TODO: Ati - //private final Map hashes; // TODO: Ati - //private final Map> futures; + private final LinkedBlockingQueue pendingKeys; private final MessageDigest md5; private int pollSeconds; private volatile boolean closed; - public HttpWatchService( - HttpFileSystem fileSystem) // TODO: Ati - remove this + public HttpWatchService() { - //this.fileSystem = fileSystem; // TODO: Ati - remove this this.executor = Executors.newScheduledThreadPool(2); - this.pendingKeys = new LinkedBlockingQueue<>(); - //this.watchKeys = new ConcurrentHashMap<>(); this.watchKeys = Collections.synchronizedList(new LinkedList<>()); - //this.pathQueue = new LinkedBlockingQueue<>(); - //this.etags = new ConcurrentHashMap<>(); - //this.hashes = new ConcurrentHashMap<>(); - //this.futures = new ConcurrentHashMap<>(); + this.pendingKeys = new LinkedBlockingQueue<>(); this.md5 = initMessageDigest("MD5"); + // TODO: Ati - HttpFileSystemConfiguration this.pollSeconds = 30; - //this.pollSeconds = 2; // TODO: Ati this.closed = false; - //executor.submit(this); } - /*public void start() - { - executor.submit(this); - }*/ - - // @Override - // public Void call() throws Exception - // { - // while (true) - // { - // HttpPath path = pathQueue.take(); - // if (path == CLOSE_PATH) - // { - // break; - // } - // String etag = etags.getOrDefault(path, ""); - // System.out.println("HWS call take path " + path + " etag [" + etag + "]"); // TODO: Ati - // sendAsync(path, etag); - // /*HttpResponse response = send(path, etag); - // if (response == null) - // { - // scheduleRequest(path, pollSeconds); - // } - // else - // { - // handleChange(response); // TODO: Ati - this should receive path -> body should be stored in path - // }*/ - // } - // return null; - // } - - /*@Override - public Void call() throws Exception - { - for (HttpWatchKey watchKey : watchKeys) - { - HttpPath path = watchKey.watchable(); - if (path.isDone()) - { - //HttpResponse response = path.poll(); - //watchKey.handleChange(response); - //handleChange(response); // this throws Exception if error // this should be in HttpWatchKey ?? - //scheduleRequest(watchKey); - System.out.println("HWS path is done " + path); - //watchKey.watchBody(); - //path.watchBody(); // scheduleRequest can be deleted // actually watchKey.watchBody() that calls path.watchBody() - // how to deal with poll interval - } - } - return null; - }*/ - @Override public void close() { System.out.println("HWS close"); // TODO: Ati - closed = true; - //fileSystem.body(null); // TODO: Ati - body should be moved from HFS to HttpPath - pendingKeys.clear(); - pendingKeys.offer(closeKey); watchKeys.forEach(HttpWatchKey::cancel); - //futures.values().forEach(future -> future.cancel(true)); - //pathQueue.add(CLOSE_PATH); watchKeys.clear(); + pendingKeys.clear(); + pendingKeys.offer(closeKey); + closed = true; } @Override @@ -176,7 +98,7 @@ public WatchKey take() throws InterruptedException return key; } - // TODO: Ati + // TODO: Ati - HttpFileSystemConfiguration public void pollSeconds( int pollSeconds) { @@ -230,129 +152,14 @@ HttpWatchKey register( return watchKey; } - /*private void sendAsync( - HttpPath path, - String etag) - { - HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() - .GET() - .uri(path.toUri()); - if (etag != null && !etag.isEmpty()) - { - requestBuilder = requestBuilder.headers("If-None-Match", etag, "Prefer", "wait=86400"); - } - - System.out.println("HWS sendAsync path " + path + " etag " + etag); // TODO: Ati - CompletableFuture future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) - .thenAccept(this::handleChange) - .exceptionally(ex -> handleException(ex, path)); - futures.put(path, future); - }*/ - - // TODO: Ati - remove this - /*private Void handleException( - Throwable throwable, - HttpPath path) - { - System.out.println("HWS handleException " + throwable.getMessage()); // TODO: Ati - //scheduleRequest(path, pollSeconds); - return null; - }*/ - - // TODO: Ati - this should be in HWK - /*private void handleChange( - HttpResponse response) - { - System.out.println("HWS handleChange response: " + response); // TODO: Ati - System.out.println("HWS handleChange response.headers: " + response.headers()); // TODO: Ati - //System.out.println("HWS handleChange response.body: " + new String(response.body())); // TODO: Ati - HttpPath path = (HttpPath) Path.of(response.request().uri()); - int statusCode = response.statusCode(); - int pollSeconds = 0; - if (statusCode == 404) - { - fileSystem.body(EMPTY_BODY); // TODO: Ati - body should be moved from HFS to HttpPath - addEvent(path); - pollSeconds = this.pollSeconds; - } - else if (statusCode >= 500 && statusCode <= 599) - { - fileSystem.body(null); // TODO: Ati - body should be moved from HFS to HttpPath - pollSeconds = this.pollSeconds; - } - else - { - byte[] body = response.body(); - fileSystem.body(body); // TODO: Ati - body should be moved from HFS to HttpPath - Optional etagOptional = response.headers().firstValue("Etag"); - if (etagOptional.isPresent()) - { - String oldEtag = etags.getOrDefault(path, ""); - String newEtag = etagOptional.get(); - if (!oldEtag.equals(newEtag)) - { - etags.put(path, newEtag); - addEvent(path); - - } - else if (response.statusCode() != 304) - { - pollSeconds = this.pollSeconds; - } - } - else - { - byte[] hash = hashes.get(path); - byte[] newHash = computeHash(body); - if (!Arrays.equals(hash, newHash)) - { - hashes.put(path, newHash); - addEvent(path); - } - pollSeconds = this.pollSeconds; - } - } - futures.remove(path); - scheduleRequest(path, pollSeconds); - }*/ - - // TODO: Ati - this should be in HWK - /*private void addEvent( - Path path) - { - System.out.println("HWS addEvent path " + path); // TODO: Ati - //HttpWatchKey key = (HttpWatchKey) watchKeys.get(path); - HttpWatchKey key = watchKeys.get(path); - if (key != null) - { - key.addEvent(ENTRY_MODIFY, path); - enqueueKey(key); - } - }*/ - - /*private void scheduleRequest( - //Path path, - HttpPath path, - int pollSeconds) - { - if (pollSeconds == 0) - { - System.out.println("HWS scheduleRequest 0"); // TODO: Ati - //pathQueue.add(path); - } - else - { - System.out.println("HWS scheduleRequest " + pollSeconds); // TODO: Ati - //executor.schedule(() -> pathQueue.add(path), pollSeconds, TimeUnit.SECONDS); - } - }*/ - + // TODO: Ati private byte[] computeHash( byte[] body) { return md5.digest(body); } + // TODO: Ati private MessageDigest initMessageDigest( String algorithm) { @@ -447,11 +254,6 @@ public int hashCode() return Objects.hashCode(path); } - /*void watchBody() - { - path.watchBody(); - }*/ - void watchBody() { if (valid) @@ -467,65 +269,6 @@ void watchBody() } } - /*private void handleChange( - HttpResponse response) - { - System.out.println("HWS handleChange response: " + response); // TODO: Ati - System.out.println("HWS handleChange response.headers: " + response.headers()); // TODO: Ati - //System.out.println("HWS handleChange response.body: " + new String(response.body())); // TODO: Ati - HttpPath path = (HttpPath) Path.of(response.request().uri()); - int statusCode = response.statusCode(); - int pollSeconds = 0; - if (statusCode == 404) - { - fileSystem.body(EMPTY_BODY); // TODO: Ati - body should be moved from HFS to HttpPath - addEvent(ENTRY_MODIFY, path); - //addEvent(path); - //pollSeconds = this.pollSeconds; - } - else if (statusCode >= 500 && statusCode <= 599) - { - fileSystem.body(null); // TODO: Ati - body should be moved from HFS to HttpPath - //pollSeconds = this.pollSeconds; - } - else - { - byte[] body = response.body(); - fileSystem.body(body); // TODO: Ati - body should be moved from HFS to HttpPath - Optional etagOptional = response.headers().firstValue("Etag"); - if (etagOptional.isPresent()) - { - String oldEtag = etags.getOrDefault(path, ""); // TODO: Ati - String newEtag = etagOptional.get(); - if (!oldEtag.equals(newEtag)) - { - etags.put(path, newEtag); // TODO: Ati - addEvent(ENTRY_MODIFY, path); - //addEvent(path); - - } - else if (response.statusCode() != 304) - { - //pollSeconds = this.pollSeconds; - } - } - else - { - byte[] hash = hashes.get(path); - byte[] newHash = computeHash(body); - if (!Arrays.equals(hash, newHash)) - { - hashes.put(path, newHash); // TODO: Ati - addEvent(ENTRY_MODIFY, path); - //addEvent(path); - } - //pollSeconds = this.pollSeconds; - } - } - //futures.remove(path); - //scheduleRequest(path, pollSeconds); - }*/ - private static class Event implements WatchEvent { private final WatchEvent.Kind kind; From 8f982fe580c9b797edf9de384b3e54ae67fd55c9 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Fri, 21 Jun 2024 12:18:54 +0200 Subject: [PATCH 38/43] WIP hfs getPath --- .../aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java index 704e90ae2b..f3b03899bd 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java @@ -96,8 +96,7 @@ public Path getPath( { requireNonNull(first); requireNonNull(more); - String second = String.join(SEPARATOR, more); - String path = second.isBlank() ? first : first + SEPARATOR + second; + String path = more.length > 0 ? first + SEPARATOR + String.join(SEPARATOR, more) : first; Path result = null; try { From 50c87920da823034998f920e4b663bcc671ba4ab Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Fri, 21 Jun 2024 21:39:28 +0200 Subject: [PATCH 39/43] WIP hfs watchBody 7 --- .../runtime/filesystem/http/HttpPath.java | 77 ++++++++++++++----- .../filesystem/http/HttpWatchService.java | 29 ++----- .../filesystem/http/HttpFileSystemIT.java | 6 +- 3 files changed, 65 insertions(+), 47 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java index 78be148a34..57ad2df059 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java @@ -21,7 +21,6 @@ import static java.net.http.HttpClient.Version.HTTP_2; import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; import static java.util.Objects.requireNonNull; -import static org.agrona.LangUtil.rethrowUnchecked; import java.io.File; import java.io.IOException; @@ -39,6 +38,9 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; public class HttpPath implements Path { @@ -50,9 +52,10 @@ public class HttpPath implements Path private final HttpFileSystem fs; private final URI location; + private final ScheduledExecutorService executor; + private int pollSeconds; // TODO: Ati - HttpFileSystemConfiguration -> final private byte[] body; - private boolean longPolling; private String etag; private HttpWatchService.HttpWatchKey watchKey; private CompletableFuture future; @@ -65,16 +68,21 @@ public class HttpPath implements Path { throw new IllegalArgumentException(String.format("invalid protocol: %s", location.getScheme())); } + System.out.println("HP constructor location " + location); // TODO: Ati this.fs = fs; this.location = location; - this.longPolling = true; + this.executor = Executors.newScheduledThreadPool(2); + // TODO: Ati - HttpFileSystemConfiguration + //this.pollSeconds = 30; + this.pollSeconds = 3; } HttpPath() { this.fs = null; this.location = null; - this.longPolling = true; + this.executor = null; + this.pollSeconds = 0; } @Override @@ -243,6 +251,7 @@ public WatchKey register( WatchService watcher, WatchEvent.Kind... events) throws IOException { + System.out.println("HP register"); // TODO: Ati return register(watcher, events, new WatchEvent.Modifier[0]); } @@ -288,6 +297,13 @@ public int hashCode() return Objects.hashCode(location); } + // TODO: Ati - HttpFileSystemConfiguration + public void pollSeconds( + int pollSeconds) + { + this.pollSeconds = pollSeconds; + } + byte[] readBody() { System.out.println("HP readBody"); // TODO: Ati @@ -328,12 +344,30 @@ else if (response.statusCode() == HTTP_NOT_MODIFIED) catch (Exception ex) { System.out.println("HP readBody exception " + ex); // TODO: Ati - rethrowUnchecked(ex); + body = new byte[0]; } return body; } - void watchBody() + void watch() + { + scheduleWatchBody(this.pollSeconds); + } + + private void scheduleWatchBody( + int pollSeconds) + { + if (pollSeconds == 0) + { + watchBody(); + } + else + { + executor.schedule(this::watchBody, pollSeconds, TimeUnit.SECONDS); + } + } + + private void watchBody() { HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() .GET() @@ -355,19 +389,17 @@ private void handleResponse( System.out.println("HP handleResponse response: " + response); // TODO: Ati System.out.println("HP handleResponse response.headers: " + response.headers()); // TODO: Ati int statusCode = response.statusCode(); - //int pollSeconds = 0; + int pollSeconds = 1; if (statusCode == 404) { body = EMPTY_BODY; watchKey.addEvent(ENTRY_MODIFY, this); - longPolling = false; - //pollSeconds = this.pollSeconds; + pollSeconds = this.pollSeconds; } else if (statusCode >= 500 && statusCode <= 599) { body = null; - longPolling = false; - //pollSeconds = this.pollSeconds; + pollSeconds = this.pollSeconds; } else { @@ -384,9 +416,8 @@ else if (statusCode >= 500 && statusCode <= 599) } else if (response.statusCode() != 304) { - this.body = response.body(); - longPolling = false; - //pollSeconds = this.pollSeconds; + body = response.body(); + pollSeconds = this.pollSeconds; } } else @@ -401,23 +432,27 @@ else if (response.statusCode() != 304) watchKey.addEvent(ENTRY_MODIFY, this); //addEvent(path); } - //pollSeconds = this.pollSeconds; + pollSeconds = this.pollSeconds; } } - watchKey.watchBody(); - } - - boolean longPolling() - { - return longPolling; + scheduleWatchBody(pollSeconds); } void cancel() { System.out.println("HP cancel"); // TODO: Ati + body = null; + etag = null; future.cancel(true); } + // required for testing + public void shutdown() + { + System.out.println("HP shutdown"); // TODO: Ati + executor.shutdownNow(); + } + private Void handleException( Throwable throwable) { diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java index bb13fcc331..b7abc3655a 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java @@ -28,9 +28,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Objects; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; public class HttpWatchService implements WatchService @@ -39,22 +37,18 @@ public class HttpWatchService implements WatchService private final WatchKey closeKey = new HttpWatchKey(CLOSE_PATH); - private final ScheduledExecutorService executor; private final List watchKeys; private final LinkedBlockingQueue pendingKeys; private final MessageDigest md5; - private int pollSeconds; private volatile boolean closed; public HttpWatchService() { - this.executor = Executors.newScheduledThreadPool(2); + this.watchKeys = Collections.synchronizedList(new LinkedList<>()); this.pendingKeys = new LinkedBlockingQueue<>(); this.md5 = initMessageDigest("MD5"); - // TODO: Ati - HttpFileSystemConfiguration - this.pollSeconds = 30; this.closed = false; } @@ -98,13 +92,6 @@ public WatchKey take() throws InterruptedException return key; } - // TODO: Ati - HttpFileSystemConfiguration - public void pollSeconds( - int pollSeconds) - { - this.pollSeconds = pollSeconds; - } - private void checkOpen() { if (closed) @@ -147,7 +134,7 @@ HttpWatchKey register( } System.out.printf("HWS register path: %s\n", path); // TODO: Ati HttpWatchKey watchKey = new HttpWatchKey(path); - watchKey.watchBody(); + watchKey.watch(); watchKeys.add(watchKey); return watchKey; } @@ -213,6 +200,7 @@ public boolean reset() @Override public void cancel() { + System.out.println("HWK cancel"); // TODO: Ati watchKeys.remove(this); path.cancel(); valid = false; @@ -254,18 +242,11 @@ public int hashCode() return Objects.hashCode(path); } - void watchBody() + void watch() { if (valid) { - if (path.longPolling()) - { - path.watchBody(); - } - else - { - executor.schedule(path::watchBody, pollSeconds, TimeUnit.SECONDS); - } + path.watch(); } } diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java index 579eaf98f3..e95d780bf5 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java @@ -203,8 +203,8 @@ public void shouldWatch() throws Exception // GIVEN String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); + ((HttpPath) path).pollSeconds(1); // TODO: Ati HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - watchService.pollSeconds(1); // TODO: Ati // WHEN k3po.start(); @@ -216,6 +216,7 @@ public void shouldWatch() throws Exception WatchKey key2 = watchService.take(); List> events2 = key2.pollEvents(); watchService.close(); + ((HttpPath) path).shutdown(); k3po.finish(); // THEN @@ -237,7 +238,7 @@ public void shouldWatchRead() throws Exception String url = "http://localhost:8080/hello.txt"; Path path = Path.of(new URI(url)); HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - watchService.pollSeconds(1); // TODO: Ati + ((HttpPath) path).pollSeconds(1); // TODO: Ati // WHEN k3po.start(); @@ -252,6 +253,7 @@ public void shouldWatchRead() throws Exception k3po.notifyBarrier("SECOND_READ"); String body2 = Files.readString(path); watchService.close(); + ((HttpPath) path).shutdown(); k3po.finish(); // THEN From 71231a9a8382bf6ff715f613c848d0d4198e42c6 Mon Sep 17 00:00:00 2001 From: Attila Kreiner Date: Fri, 21 Jun 2024 21:39:54 +0200 Subject: [PATCH 40/43] WIP fix --- .../main/java/io/aklivity/zilla/runtime/engine/Engine.java | 4 ++-- .../runtime/engine/internal/watcher/ConfigWatcherTask.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index 46aba6b3ac..82f6567325 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -291,9 +291,9 @@ private String readPath( String result; try { - System.out.println("ENG readPath path " + path); // TODO: Ati + //System.out.println("ENG readPath path " + path); // TODO: Ati result = Files.readString(path); - System.out.println("ENG readPath result [" + result + "]"); // TODO: Ati + //System.out.println("ENG readPath result [" + result + "]"); // TODO: Ati } catch (Exception ex) { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java index b9c7aac38b..ef592e6a4e 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java @@ -119,11 +119,11 @@ public void watchConfig( } else { + configText = readPath.apply(configPath); WatchedItem watchedItem = new WatchedItem(configPath, watchService); watchedItem.register(); watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); System.out.println("CWT watchConfig readPath " + configPath); // TODO: Ati - configText = readPath.apply(configPath); watchedItem.setHash(computeHash(configText)); } configChangeListener.apply(configText); From 7535666000f4e806372f80eab69f4ccd97b309a7 Mon Sep 17 00:00:00 2001 From: John Fallows Date: Tue, 25 Jun 2024 18:53:59 -0700 Subject: [PATCH 41/43] Remove BindingConfig.readLocation Remove GuardedConfig.readPath Rename ConfigAdapterContext.readLocation to ConfigAdapterContext.readResource Deprecate ConfigAdapterContext for removal Add EngineConfiguration.configURI() to resolve absolute file path Remove EngineConfiguration.configURL() but use to default EngineConfiguration.configURI() Simplify EngineManager Gather resources on NamespaceConfig from member configs Consolidate config and resource watcher as EngineConfigWatcher Resolve watched paths based on path filesystem provider scheme Configure HttpFileSystem poll internval duration via Map env Simplify ReconfigureHttpIT scripts and include /zilla.yaml in request path HttpFileSystem per origin (root path) not per individual path Track HttpPath change count vs read count to implement simple caching --- .../config/AsyncapiBindingConfig.java | 2 - .../config/GrpcOptionsConfigAdapter.java | 6 +- .../config/GrpcOptionsConfigAdapterTest.java | 2 +- .../config/OpenapiAsyncapiBindingConfig.java | 1 - .../internal/config/OpenapiBindingConfig.java | 1 - .../binding/tls/internal/bench/TlsWorker.java | 2 +- runtime/catalog-karapace/pom.xml | 2 +- .../internal/airline/ZillaStartCommand.java | 7 +- .../aklivity/zilla/runtime/engine/Engine.java | 40 +- .../runtime/engine/EngineConfiguration.java | 46 +- .../runtime/engine/config/BindingConfig.java | 1 - .../engine/config/ConfigAdapterContext.java | 4 +- .../engine/config/EngineConfigReader.java | 6 +- .../runtime/engine/config/GuardConfig.java | 4 - .../engine/config/NamespaceConfig.java | 60 ++- .../internal/registry/EngineManager.java | 149 +++--- .../internal/registry/EngineWorker.java | 10 +- .../internal/watcher/ConfigWatcherTask.java | 140 ------ .../watcher/EngineConfigWatchTask.java | 111 +++++ .../internal/watcher/EngineConfigWatcher.java | 293 +++++++---- .../internal/watcher/ResourceWatcherTask.java | 144 ------ .../engine/internal/watcher/WatchedItem.java | 155 ------ .../engine/internal/watcher/WatcherTask.java | 62 --- .../runtime/engine/internal/EngineTest.java | 1 - .../engine/internal/ReconfigureHttpIT.java | 56 +-- .../zilla/runtime/engine/test/EngineRule.java | 36 +- .../http/HttpFileSystemProvider.java | 26 - .../runtime/filesystem/http/HttpPath.java | 462 ------------------ .../filesystem/http/HttpWatchService.java | 287 ----------- .../http/{ => internal}/HttpFileSystem.java | 78 ++- .../internal/HttpFileSystemConfiguration.java | 40 ++ .../HttpFileSystemProvider.java} | 77 +-- .../filesystem/http/internal/HttpPath.java | 369 ++++++++++++++ .../http/internal/HttpWatchService.java | 384 +++++++++++++++ .../HttpsFileSystemProvider.java | 8 +- .../ReadOnlyByteArrayChannel.java | 6 +- .../src/main/moditect/module-info.java | 4 +- .../java.nio.file.spi.FileSystemProvider | 4 +- .../filesystem/http/HttpFileSystemIT.java | 307 +++++++----- ...ileSystem.java => HttpFileSystemTest.java} | 2 +- .../guard/jwt/internal/JwtGuardContext.java | 2 +- .../guard/jwt/internal/JwtGuardHandler.java | 27 +- .../config/JwtKeySetConfigAdapter.java | 26 +- .../jwt/internal/JwtGuardHandlerTest.java | 42 +- .../guard/jwt/internal/JwtGuardIT.java | 15 +- .../config/JwtKeySetConfigAdapterTest.java | 23 +- .../reconfigure.create.via.http/client.rpt | 16 +- .../reconfigure.create.via.http/server.rpt | 16 +- .../reconfigure.delete.via.http/client.rpt | 17 +- .../reconfigure.delete.via.http/server.rpt | 17 +- .../client.rpt | 8 +- .../server.rpt | 1 - .../reconfigure.modify.via.http/client.rpt | 25 +- .../reconfigure.modify.via.http/server.rpt | 12 +- .../client.rpt | 14 +- .../server.rpt | 6 +- .../reconfigure.create.via.http/client.rpt | 7 - .../reconfigure.create.via.http/server.rpt | 4 - .../reconfigure.delete.via.http/client.rpt | 9 - .../reconfigure.delete.via.http/server.rpt | 7 - .../client.rpt | 2 + .../server.rpt | 3 + .../reconfigure.modify.via.http/client.rpt | 2 + .../reconfigure.modify.via.http/server.rpt | 4 + .../client.rpt | 4 + .../server.rpt | 2 +- .../specs/engine/streams/ApplicationIT.java | 10 +- .../zilla/specs/engine/streams/NetworkIT.java | 18 +- .../guard/jwt/config/guard-keys-dynamic.yaml | 27 +- .../guard/jwt/config/guard-keys-implicit.yaml | 2 +- .../specs/guard/jwt/config/keys/issuer.rpt | 29 ++ 71 files changed, 1848 insertions(+), 1944 deletions(-) delete mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java create mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatchTask.java delete mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcherTask.java delete mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatchedItem.java delete mode 100644 runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatcherTask.java delete mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java delete mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java delete mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java rename runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/{ => internal}/HttpFileSystem.java (61%) create mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystemConfiguration.java rename runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/{AbstractHttpFileSystemProvider.java => internal/HttpFileSystemProvider.java} (83%) create mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java create mode 100644 runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpWatchService.java rename runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/{ => internal}/HttpsFileSystemProvider.java (75%) rename runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/{ => internal}/ReadOnlyByteArrayChannel.java (94%) rename runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/{TestHttpFileSystem.java => HttpFileSystemTest.java} (98%) create mode 100644 specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/keys/issuer.rpt diff --git a/runtime/binding-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/asyncapi/internal/config/AsyncapiBindingConfig.java b/runtime/binding-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/asyncapi/internal/config/AsyncapiBindingConfig.java index 4fa6213924..dd236f8f38 100644 --- a/runtime/binding-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/asyncapi/internal/config/AsyncapiBindingConfig.java +++ b/runtime/binding-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/asyncapi/internal/config/AsyncapiBindingConfig.java @@ -320,7 +320,6 @@ private void attachProxyBinding( namespaceGenerator.init(binding); final List labels = configs.stream().map(c -> c.apiLabel).collect(toList()); final NamespaceConfig composite = namespaceGenerator.generateProxy(binding, asyncapis, schemaIdsByApiId::get, labels); - composite.readLocation = binding.readLocation; attach.accept(composite); updateNamespace(configs, composite, new ArrayList<>(asyncapis.values())); } @@ -349,7 +348,6 @@ private void attachServerClientBinding( namespaceConfig.servers.forEach(s -> s.setAsyncapiProtocol( namespaceGenerator.resolveProtocol(s.protocol(), options, namespaceConfig.asyncapis, namespaceConfig.servers))); final NamespaceConfig composite = namespaceGenerator.generate(binding, namespaceConfig); - composite.readLocation = binding.readLocation; attach.accept(composite); updateNamespace(namespaceConfig.configs, composite, namespaceConfig.asyncapis); } diff --git a/runtime/binding-grpc/src/main/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapter.java b/runtime/binding-grpc/src/main/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapter.java index 93f38131b2..19e7e18acf 100644 --- a/runtime/binding-grpc/src/main/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapter.java +++ b/runtime/binding-grpc/src/main/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapter.java @@ -41,7 +41,7 @@ public final class GrpcOptionsConfigAdapter implements OptionsConfigAdapterSpi, private final GrpcProtobufParser parser = new GrpcProtobufParser(); - private Function readLocation; + private Function readResource; @Override public Kind kind() @@ -88,7 +88,7 @@ public OptionsConfig adaptFromJson( public void adaptContext( ConfigAdapterContext context) { - this.readLocation = context::readLocation; + this.readResource = context::readResource; } private List asListProtobufs( @@ -103,7 +103,7 @@ private GrpcProtobufConfig asProtobuf( JsonValue value) { final String location = ((JsonString) value).getString(); - final String protobuf = readLocation.apply(location); + final String protobuf = readResource.apply(location); return parser.parse(location, protobuf); } diff --git a/runtime/binding-grpc/src/test/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapterTest.java b/runtime/binding-grpc/src/test/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapterTest.java index 1e29b6c5a8..e33893f8ff 100644 --- a/runtime/binding-grpc/src/test/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapterTest.java +++ b/runtime/binding-grpc/src/test/java/io/aklivity/zilla/runtime/binding/grpc/internal/config/GrpcOptionsConfigAdapterTest.java @@ -66,7 +66,7 @@ public void initJson() throws IOException { content = new String(resource.readAllBytes(), UTF_8); } - Mockito.doReturn(content).when(context).readLocation("protobuf/echo.proto"); + Mockito.doReturn(content).when(context).readResource("protobuf/echo.proto"); adapter = new OptionsConfigAdapter(OptionsConfigAdapterSpi.Kind.BINDING, context); adapter.adaptType("grpc"); JsonbConfig config = new JsonbConfig() diff --git a/runtime/binding-openapi-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/asyncapi/internal/config/OpenapiAsyncapiBindingConfig.java b/runtime/binding-openapi-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/asyncapi/internal/config/OpenapiAsyncapiBindingConfig.java index e7e66289d9..c6cb86540b 100644 --- a/runtime/binding-openapi-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/asyncapi/internal/config/OpenapiAsyncapiBindingConfig.java +++ b/runtime/binding-openapi-asyncapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/asyncapi/internal/config/OpenapiAsyncapiBindingConfig.java @@ -144,7 +144,6 @@ public void attach( Object2ObjectHashMap::new)); this.composite = namespaceGenerator.generate(binding, openapis, asyncapis, openapiSchemaIdsByApiId::get); - this.composite.readLocation = binding.readLocation; attach.accept(this.composite); BindingConfig mappingBinding = composite.bindings.stream() diff --git a/runtime/binding-openapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/internal/config/OpenapiBindingConfig.java b/runtime/binding-openapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/internal/config/OpenapiBindingConfig.java index a7cc361413..a3a3d8e4aa 100644 --- a/runtime/binding-openapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/internal/config/OpenapiBindingConfig.java +++ b/runtime/binding-openapi/src/main/java/io/aklivity/zilla/runtime/binding/openapi/internal/config/OpenapiBindingConfig.java @@ -138,7 +138,6 @@ public void attach( for (OpenapiNamespaceConfig namespaceConfig : namespaceConfigs.values()) { final NamespaceConfig composite = namespaceGenerator.generate(binding, namespaceConfig); - composite.readLocation = binding.readLocation; attach.accept(composite); namespaceConfig.configs.forEach(c -> { diff --git a/runtime/binding-tls/src/test/java/io/aklivity/zilla/runtime/binding/tls/internal/bench/TlsWorker.java b/runtime/binding-tls/src/test/java/io/aklivity/zilla/runtime/binding/tls/internal/bench/TlsWorker.java index 9f90718b95..43cafdddf0 100644 --- a/runtime/binding-tls/src/test/java/io/aklivity/zilla/runtime/binding/tls/internal/bench/TlsWorker.java +++ b/runtime/binding-tls/src/test/java/io/aklivity/zilla/runtime/binding/tls/internal/bench/TlsWorker.java @@ -103,7 +103,7 @@ public TlsWorker( .readonly(false) .build() .bufferPool(); - this.configPath = config.configPath(); + this.configPath = Path.of(config.configURI()); this.signaler = new TlsSignaler(); diff --git a/runtime/catalog-karapace/pom.xml b/runtime/catalog-karapace/pom.xml index 2b954a62c7..a39e5877af 100644 --- a/runtime/catalog-karapace/pom.xml +++ b/runtime/catalog-karapace/pom.xml @@ -22,7 +22,7 @@ - 0.93 + 0.92 0 diff --git a/runtime/command-start/src/main/java/io/aklivity/zilla/runtime/command/start/internal/airline/ZillaStartCommand.java b/runtime/command-start/src/main/java/io/aklivity/zilla/runtime/command/start/internal/airline/ZillaStartCommand.java index cc983dbb43..e72804d72a 100644 --- a/runtime/command-start/src/main/java/io/aklivity/zilla/runtime/command/start/internal/airline/ZillaStartCommand.java +++ b/runtime/command-start/src/main/java/io/aklivity/zilla/runtime/command/start/internal/airline/ZillaStartCommand.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.net.URI; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -127,11 +126,9 @@ public void run() EngineConfiguration config = new EngineConfiguration(props); - URL configURL = config.configURL(); - if ("file".equals(configURL.getProtocol())) + Path configPath = Path.of(config.configURI()); + if ("file".equals(configPath.getFileSystem().provider().getScheme())) { - final Path configPath = Paths.get(configURL.getPath()); - if (configPath.endsWith("zilla.yaml") && Files.notExists(configPath)) { Path configJson = configPath.resolveSibling("zilla.json"); diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java index 82f6567325..78085f8e9a 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/Engine.java @@ -21,8 +21,6 @@ import static org.agrona.LangUtil.rethrowUnchecked; import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -79,7 +77,6 @@ public final class Engine implements Collector, AutoCloseable private final AtomicInteger nextTaskId; private final ThreadFactory factory; - private final Path configPath; private final List workers; private final boolean readonly; private final EngineConfiguration config; @@ -148,7 +145,7 @@ public final class Engine implements Collector, AutoCloseable for (int workerIndex = 0; workerIndex < workerCount; workerIndex++) { EngineWorker worker = - new EngineWorker(config, tasks, labels, errorHandler, tuning::affinity, this::resolvePath, bindings, exporters, + new EngineWorker(config, tasks, labels, errorHandler, tuning::affinity, bindings, exporters, guards, vaults, catalogs, models, metricGroups, this, this::supplyEventReader, eventFormatterFactory, workerIndex, readonly, this::process); workers.add(worker); @@ -177,7 +174,6 @@ public final class Engine implements Collector, AutoCloseable final Map guardsByType = guards.stream() .collect(Collectors.toMap(g -> g.name(), g -> g)); - this.configPath = config.configPath(); EngineManager manager = new EngineManager( schemaTypes, bindingsByType::get, @@ -190,10 +186,7 @@ public final class Engine implements Collector, AutoCloseable logger, context, config, - extensions, - this.configPath, - this::readPath, - this::readLocation); + extensions); this.bindings = bindings; this.tasks = tasks; @@ -285,35 +278,6 @@ public static EngineBuilder builder() return new EngineBuilder(); } - private String readPath( - Path path) - { - String result; - try - { - //System.out.println("ENG readPath path " + path); // TODO: Ati - result = Files.readString(path); - //System.out.println("ENG readPath result [" + result + "]"); // TODO: Ati - } - catch (Exception ex) - { - result = ""; - } - return result; - } - - public Path resolvePath( - String location) - { - return configPath.resolveSibling(location); - } - - private String readLocation( - String location) - { - return readPath(resolvePath(location)); - } - private Thread newTaskThread( Runnable r) { diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java index 60f772c1c5..61e17d3710 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/EngineConfiguration.java @@ -19,6 +19,7 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; +import java.io.File; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; @@ -44,7 +45,7 @@ public class EngineConfiguration extends Configuration public static final boolean DEBUG_BUDGETS = Boolean.getBoolean("zilla.engine.debug.budgets"); public static final PropertyDef ENGINE_CONFIG_URL; - public static final PropertyDef ENGINE_CONFIG_PATH; + public static final PropertyDef ENGINE_CONFIG_URI; public static final IntPropertyDef ENGINE_CONFIG_POLL_INTERVAL_SECONDS; public static final PropertyDef ENGINE_NAME; public static final PropertyDef ENGINE_DIRECTORY; @@ -82,8 +83,8 @@ public class EngineConfiguration extends Configuration { final ConfigurationDef config = new ConfigurationDef("zilla.engine"); ENGINE_CONFIG_URL = config.property(URL.class, "config.url", EngineConfiguration::configURL, "file:zilla.yaml"); - ENGINE_CONFIG_PATH = config.property(Path.class, "config.path", EngineConfiguration::decodeConfigPath, - EngineConfiguration::defaultConfigPath); + ENGINE_CONFIG_URI = config.property(URI.class, "config.uri", EngineConfiguration::decodeConfigURI, + EngineConfiguration::defaultConfigURI); ENGINE_CONFIG_POLL_INTERVAL_SECONDS = config.property("config.poll.interval.seconds", 60); ENGINE_NAME = config.property("name", EngineConfiguration::defaultName); ENGINE_DIRECTORY = config.property("directory", EngineConfiguration::defaultDirectory); @@ -144,14 +145,15 @@ public EngineConfiguration() super(ENGINE_CONFIG, new Configuration()); } + @Deprecated public URL configURL() { return ENGINE_CONFIG_URL.get(this); } - public Path configPath() + public URI configURI() { - return ENGINE_CONFIG_PATH.get(this); + return ENGINE_CONFIG_URI.get(this); } public int configPollIntervalSeconds() @@ -424,39 +426,19 @@ private static HostResolver defaultHostResolver( }; } - private static Path decodeConfigPath( + private static URI decodeConfigURI( Configuration config, String value) { - return resolveConfigPath(value); + return value.startsWith("file:") + ? new File(value.substring("file:".length())).toURI() + : URI.create(value); } - private static Path defaultConfigPath( + private static URI defaultConfigURI( Configuration config) { - URL url = ((EngineConfiguration) config).configURL(); - return resolveConfigPath(url.toString()); - } - - private static Path resolveConfigPath( - String config) - { - Path configPath; - URI configUri = URI.create(config); - if ("file".equals(configUri.getScheme()) && !Path.of(configUri.getSchemeSpecificPart()).isAbsolute()) - { - // this works for relative file e.g. file:zilla.yaml - Path basePath = Path.of("").toAbsolutePath(); - configPath = basePath.resolve(configUri.getSchemeSpecificPart()); - } - else - { - // this works for absolute file e.g. file:/path/dir/zilla.yaml - // this works for http e.g. http://localhost:7115/zilla.yaml - // this works for jar e.g. jar:file:/path/engine.jar!/package/zilla.yaml - // (the jar filesystem is opened and closed by EngineRule) - configPath = Path.of(configUri); - } - return configPath; + URL url = ENGINE_CONFIG_URL.get(config); + return decodeConfigURI(config, url.toString()); } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/BindingConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/BindingConfig.java index 1c799d2ecd..0eb2852e64 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/BindingConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/BindingConfig.java @@ -27,7 +27,6 @@ public class BindingConfig public transient long id; public transient long entryId; public transient ToLongFunction resolveId; - public transient Function readLocation; public transient long vaultId; public transient String qvault; diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ConfigAdapterContext.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ConfigAdapterContext.java index 91f8349981..bddd2c77f5 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ConfigAdapterContext.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/ConfigAdapterContext.java @@ -15,7 +15,9 @@ */ package io.aklivity.zilla.runtime.engine.config; +@Deprecated public interface ConfigAdapterContext { - String readLocation(String location); + String readResource( + String location); } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java index 6caa3a872a..b2be03ac84 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/EngineConfigReader.java @@ -59,22 +59,19 @@ public final class EngineConfigReader private final Resolver expressions; private final Collection schemaTypes; private final Consumer logger; - private final Consumer addResources; public EngineConfigReader( EngineConfiguration config, ConfigAdapterContext context, Resolver expressions, Collection schemaTypes, - Consumer logger, - Consumer addResources) + Consumer logger) { this.config = config; this.context = context; this.expressions = expressions; this.schemaTypes = schemaTypes; this.logger = logger; - this.addResources = addResources; } public EngineConfig read( @@ -164,7 +161,6 @@ public EngineConfig read( reader.reset(); reader.skip(configAt); NamespaceConfig namespace = jsonb.fromJson(reader, NamespaceConfig.class); - addResources.accept(namespace); builder.namespace(namespace); if (!errors.isEmpty()) diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/GuardConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/GuardConfig.java index e7d6d4669f..5363211bfd 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/GuardConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/GuardConfig.java @@ -18,13 +18,9 @@ import static java.util.Objects.requireNonNull; import static java.util.function.Function.identity; -import java.nio.file.Path; -import java.util.function.Function; - public class GuardConfig { public transient long id; - public transient Function readPath; public final String namespace; public final String name; diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java index 4e791b8104..16e6fe30f8 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/config/NamespaceConfig.java @@ -20,14 +20,12 @@ import java.util.LinkedList; import java.util.List; -import java.util.function.Function; public class NamespaceConfig { public static final String FILESYSTEM = "filesystem"; public transient int id; - public transient Function readLocation; public final String name; public final TelemetryConfig telemetry; @@ -56,26 +54,52 @@ public static NamespaceConfigBuilder builder() this.guards = requireNonNull(guards); this.vaults = requireNonNull(vaults); this.catalogs = requireNonNull(catalogs); - this.resources = resolveResources(); + this.resources = resolveResources(this, telemetry, bindings, guards, vaults, catalogs); } - private List resolveResources() + private static List resolveResources( + NamespaceConfig namespace, + TelemetryConfig telemetry, + List bindings, + List guards, + List vaults, + List catalogs) { - List resources = new LinkedList<>(); - for (CatalogConfig catalog : catalogs) - { - if (FILESYSTEM.equals(catalog.type)) - { - resources.addAll(catalog.options.resources); - } - } - for (VaultConfig vault : vaults) + List options = new LinkedList<>(); + + if (telemetry != null && telemetry.exporters != null) { - if (FILESYSTEM.equals(vault.type)) - { - resources.addAll(vault.options.resources); - } + telemetry.exporters.stream() + .filter(e -> e.options != null) + .map(e -> e.options) + .forEach(options::add); } - return resources; + + bindings.stream() + .filter(b -> b.options != null) + .map(b -> b.options) + .forEach(options::add); + + guards.stream() + .filter(g -> g.options != null) + .map(g -> g.options) + .forEach(options::add); + + vaults.stream() + .filter(v -> v.options != null) + .map(v -> v.options) + .forEach(options::add); + + catalogs.stream() + .filter(c -> c.options != null) + .map(c -> c.options) + .forEach(options::add); + + return options.stream() + .filter(o -> o.resources != null) + .flatMap(o -> o.resources.stream()) + .sorted() + .distinct() + .toList(); } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java index 60a4cda8b2..6666cd6c78 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineManager.java @@ -18,12 +18,15 @@ import static java.util.stream.Collectors.toList; import static org.agrona.LangUtil.rethrowUnchecked; +import java.io.IOException; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; @@ -62,7 +65,7 @@ import io.aklivity.zilla.runtime.engine.guard.Guard; import io.aklivity.zilla.runtime.engine.internal.Tuning; import io.aklivity.zilla.runtime.engine.internal.config.NamespaceAdapter; -import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatcher; +import io.aklivity.zilla.runtime.engine.internal.watcher.EngineConfigWatchTask; import io.aklivity.zilla.runtime.engine.namespace.NamespacedId; import io.aklivity.zilla.runtime.engine.resolver.Resolver; @@ -82,11 +85,11 @@ public class EngineManager private final EngineExtContext context; private final EngineConfiguration config; private final List extensions; - private final Function readPath; - private final Function readLocation; private final Resolver expressions; - private final EngineConfigWatcher watcher; + private final Path configPath; + private final EngineConfigWatchTask watchTask; + private String currentText; private EngineConfig current; public EngineManager( @@ -101,10 +104,7 @@ public EngineManager( Consumer logger, EngineExtContext context, EngineConfiguration config, - List extensions, - Path configPath, - Function readPath, - Function readLocation) + List extensions) { this.schemaTypes = schemaTypes; this.bindingByType = bindingByType; @@ -118,10 +118,19 @@ public EngineManager( this.context = context; this.config = config; this.extensions = extensions; - this.readPath = readPath; - this.readLocation = readLocation; this.expressions = Resolver.instantiate(config); - this.watcher = new EngineConfigWatcher(configPath, readPath, this::reconfigure, this::reconfigure); + this.configPath = Path.of(config.configURI()); + this.watchTask = new WatchTaskImpl(configPath); + } + + public void start() throws Exception + { + watchTask.submit(); + } + + public void close() + { + watchTask.close(); } public void process( @@ -135,39 +144,47 @@ public void process( process(guards, namespace); } - public void start() throws Exception - { - watcher.startWatchingConfig(); - } - - public void close() - { - watcher.close(); - } - - private EngineConfig reconfigure( - String configText) + private void onPathChanged( + Path watchedPath) { EngineConfig newConfig = null; + reconfigure: try { - newConfig = parse(configText); + if (!Files.exists(watchedPath)) + { + break reconfigure; + } + + String newConfigText = Files.readString(configPath); + if (Objects.equals(currentText, newConfigText)) + { + break reconfigure; + } + + newConfig = parse(newConfigText); if (newConfig != null) { + final String oldConfigText = currentText; final EngineConfig oldConfig = current; + unregister(oldConfig); try { + currentText = newConfigText; current = newConfig; + register(newConfig); } catch (Exception ex) { context.onError(ex); + currentText = oldConfigText; current = oldConfig; + register(oldConfig); rethrowUnchecked(ex); @@ -186,15 +203,6 @@ private EngineConfig reconfigure( throw new ConfigException("Engine configuration failed", ex); } } - - return newConfig; - } - - private void reconfigure( - Set namespaces) - { - unregister(current); - register(current); } private EngineConfig parse( @@ -213,11 +221,10 @@ private EngineConfig parse( { EngineConfigReader reader = new EngineConfigReader( config, - new NamespaceConfigAdapterContext(readLocation), + new NamespaceConfigAdapterContext(Path.of(config.configURI())), expressions, schemaTypes, - logger, - this::addResources); + logger); engine = reader.read(configText); @@ -228,7 +235,6 @@ private EngineConfig parse( for (NamespaceConfig namespace : engine.namespaces) { - namespace.readLocation = readLocation; process(guards, namespace); } } @@ -240,19 +246,10 @@ private EngineConfig parse( return engine; } - private void addResources( - NamespaceConfig namespace) - { - watcher.addResources(namespace); - } - private void process( List guards, NamespaceConfig namespace) { - assert readPath != null; - assert namespace.readLocation != null; - namespace.id = supplyId.applyAsInt(namespace.name); NameResolver resolver = new NameResolver(namespace.id); @@ -260,7 +257,6 @@ private void process( for (GuardConfig guard : namespace.guards) { guard.id = resolver.resolve(guard.name); - guard.readPath = readPath; } for (VaultConfig vault : namespace.vaults) @@ -292,7 +288,6 @@ private void process( binding.id = resolver.resolve(binding.name); binding.entryId = resolver.resolve(binding.entry); binding.resolveId = resolver::resolve; - binding.readLocation = namespace.readLocation; binding.typeId = supplyId.applyAsInt(binding.type); binding.kindId = supplyId.applyAsInt(binding.kind.name().toLowerCase()); @@ -404,9 +399,8 @@ private void register( { for (NamespaceConfig namespace : config.namespaces) { - System.out.println("register: " + namespace.name); // TODO: Ati - watcher.addResources(namespace); register(namespace); + watch(namespace); } } @@ -420,15 +414,28 @@ private void unregister( { for (NamespaceConfig namespace : config.namespaces) { - System.out.println("unregister: " + namespace.name); // TODO: Ati + unwatch(namespace); unregister(namespace); - watcher.removeNamespace(namespace.name); } } extensions.forEach(e -> e.onUnregistered(context)); } + private void watch( + NamespaceConfig namespace) + { + namespace.resources.stream() + .forEach(watchTask::watch); + } + + private void unwatch( + NamespaceConfig namespace) + { + namespace.resources.stream() + .forEach(watchTask::unwatch); + } + private void register( NamespaceConfig namespace) { @@ -491,19 +498,47 @@ private String format( private static final class NamespaceConfigAdapterContext implements ConfigAdapterContext { - private final Function readLocation; + private final Path configPath; NamespaceConfigAdapterContext( - Function readLocation) + Path configPath) { - this.readLocation = readLocation; + this.configPath = configPath; } @Override - public String readLocation( + public String readResource( String location) { - return readLocation.apply(location); + String content = null; + + try + { + Path path = configPath.resolveSibling(location); + content = Files.readString(path); + } + catch (IOException ex) + { + rethrowUnchecked(ex); + } + + return content; + } + } + + private final class WatchTaskImpl extends EngineConfigWatchTask + { + WatchTaskImpl( + Path configPath) + { + super(configPath); + } + + @Override + protected void onPathChanged( + Path watchedPath) + { + EngineManager.this.onPathChanged(watchedPath); } } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java index f29251f5e7..69262daf48 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/registry/EngineWorker.java @@ -40,6 +40,7 @@ import static org.agrona.concurrent.AgentRunner.startOnThread; import java.net.InetAddress; +import java.net.URI; import java.nio.channels.SelectableChannel; import java.nio.file.Path; import java.time.Clock; @@ -214,7 +215,7 @@ public class EngineWorker implements EngineContext, Agent private final EngineRegistry registry; private final Deque taskQueue; private final LongUnaryOperator affinityMask; - private final Function resolvePath; + private final Path configPath; private final AgentRunner runner; private final IdleStrategy idleStrategy; private final ErrorHandler errorHandler; @@ -242,7 +243,6 @@ public EngineWorker( LabelManager labels, ErrorHandler errorHandler, LongUnaryOperator affinityMask, - Function resolvePath, Collection bindings, Collection exporters, Collection guards, @@ -259,7 +259,7 @@ public EngineWorker( { this.localIndex = index; this.config = config; - this.resolvePath = resolvePath; + this.configPath = Path.of(config.configURI()); this.labels = labels; this.affinityMask = affinityMask; @@ -743,7 +743,9 @@ public ConverterHandler supplyWriteConverter( public Path resolvePath( String location) { - return resolvePath.apply(location); + return location.indexOf(':') == -1 + ? configPath.resolveSibling(location) + : Path.of(URI.create(location)); } @Override diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java deleted file mode 100644 index ef592e6a4e..0000000000 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ConfigWatcherTask.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright 2021-2023 Aklivity Inc. - * - * Aklivity licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package io.aklivity.zilla.runtime.engine.internal.watcher; - -import static org.agrona.LangUtil.rethrowUnchecked; - -import java.io.IOException; -import java.nio.file.ClosedWatchServiceException; -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.nio.file.WatchKey; -import java.nio.file.WatchService; -import java.util.IdentityHashMap; -import java.util.Map; -import java.util.concurrent.Future; -import java.util.function.Function; - -import io.aklivity.zilla.runtime.engine.config.EngineConfig; - -public class ConfigWatcherTask extends WatcherTask -{ - private final Map watchedItems; - private final WatchService watchService; - private final Function configChangeListener; - private final Function readPath; - - public ConfigWatcherTask( - FileSystem fileSystem, - Function configChangeListener, - Function readPath) - { - this.configChangeListener = configChangeListener; - this.readPath = readPath; - this.watchedItems = new IdentityHashMap<>(); - WatchService watchService = null; - if (!"jar".equals(fileSystem.provider().getScheme())) // we can't watch in jar fs - { - try - { - watchService = fileSystem.newWatchService(); - } - catch (Exception ex) - { - rethrowUnchecked(ex); - } - } - this.watchService = watchService; - } - - @Override - public Future submit() - { - return executor.submit(this); - } - - @Override - public Void call() - { - if (watchService != null) - { - while (true) - { - try - { - final WatchKey key = watchService.take(); - System.out.println("CWT call key " + key); // TODO: Ati - - WatchedItem watchedItem = watchedItems.get(key); - - if (watchedItem != null && watchedItem.isWatchedKey(key)) - { - // Even if no reconfigure needed, recalculation is necessary, since symlinks might have changed. - watchedItem.keys().forEach(watchedItems::remove); - watchedItem.unregister(); - watchedItem.register(); - watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); - String newText = readPath.apply(watchedItem.getPath()); - byte[] newHash = computeHash(newText); - if (watchedItem.isReconfigureNeeded(newHash)) - { - watchedItem.setHash(newHash); - if (configChangeListener != null) - { - configChangeListener.apply(newText); - } - } - } - } - catch (InterruptedException | ClosedWatchServiceException ex) - { - break; - } - } - } - - return null; - } - - public void watchConfig( - Path configPath) - { - String configText; - if ("jar".equals(configPath.getFileSystem().provider().getScheme())) - { - configText = readPath.apply(configPath); - } - else - { - configText = readPath.apply(configPath); - WatchedItem watchedItem = new WatchedItem(configPath, watchService); - watchedItem.register(); - watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); - System.out.println("CWT watchConfig readPath " + configPath); // TODO: Ati - watchedItem.setHash(computeHash(configText)); - } - configChangeListener.apply(configText); - } - - @Override - public void close() throws IOException - { - if (watchService != null) - { - watchService.close(); - } - } -} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatchTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatchTask.java new file mode 100644 index 0000000000..33fa665f5b --- /dev/null +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatchTask.java @@ -0,0 +1,111 @@ +/* + * Copyright 2021-2023 Aklivity Inc. + * + * Aklivity licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.aklivity.zilla.runtime.engine.internal.watcher; + +import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; +import static org.agrona.LangUtil.rethrowUnchecked; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.WatchKey; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public abstract class EngineConfigWatchTask implements AutoCloseable, Callable +{ + private final Path configPath; + private final EngineConfigWatcher watcher; + private final ExecutorService executor; + private Map resourceKeys; + + protected EngineConfigWatchTask( + Path configPath) + { + this.configPath = configPath; + this.watcher = new EngineConfigWatcher(configPath.getFileSystem()); + this.executor = Executors.newScheduledThreadPool(2); + this.resourceKeys = new HashMap<>(); + } + + public void submit() + { + onPathChanged(configPath); + executor.submit(this); + } + + public void watch( + String resource) + { + try + { + Path resourcePath = configPath.resolveSibling(resource); + WatchKey resourceKey = watcher.register(resourcePath, ENTRY_MODIFY, ENTRY_CREATE, ENTRY_DELETE); + resourceKeys.put(resource, resourceKey); + } + catch (IOException ex) + { + rethrowUnchecked(ex); + } + } + + public void unwatch( + String resource) + { + WatchKey resourceKey = resourceKeys.remove(resource); + + if (resourceKey != null) + { + resourceKey.cancel(); + } + } + + @Override + public final Void call() throws IOException + { + watcher.register(configPath, ENTRY_MODIFY, ENTRY_CREATE, ENTRY_DELETE); + + while (true) + { + try + { + final WatchKey key = watcher.take(); + final Path watchable = (Path) key.watchable(); + onPathChanged(watchable); + } + catch (InterruptedException ex) + { + watcher.close(); + break; + } + } + + return null; + } + + @Override + public final void close() + { + executor.shutdownNow(); + } + + protected abstract void onPathChanged( + Path watchedPath); +} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java index 1bc849a429..2a66ffee7f 100644 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java +++ b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/EngineConfigWatcher.java @@ -17,147 +17,256 @@ import static org.agrona.LangUtil.rethrowUnchecked; +import java.io.IOException; import java.nio.file.FileSystem; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.nio.file.WatchEvent.Kind; +import java.nio.file.WatchEvent.Modifier; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Consumer; import java.util.function.Function; -import io.aklivity.zilla.runtime.engine.config.EngineConfig; -import io.aklivity.zilla.runtime.engine.config.NamespaceConfig; +import org.agrona.LangUtil; -public class EngineConfigWatcher +public final class EngineConfigWatcher implements AutoCloseable { - private final Path configPath; - private final FileSystem fileSystem; - private final Function readPath; - private final Map> resources; - private final Map resourceTasks; - private final Consumer> resourceChangeListener; - private final ConfigWatcherTask configWatcherTask; + private static final Function>> LOOKUP_RESOLVER; + static + { + Map>> resolvers = new HashMap<>(); + resolvers.put("http", Set::of); + + Function> defaultResolver = EngineConfigWatcher::resolveWatchables; + LOOKUP_RESOLVER = scheme -> resolvers.getOrDefault(scheme, defaultResolver); + } + + private final Function> resolver; + private final WatchService watcher; + private final Map compoundKeys; public EngineConfigWatcher( - Path configPath, - Function readPath, - Function configChangeListener, - Consumer> resourceChangeListener) + FileSystem fileSystem) { - this.configPath = configPath; - this.fileSystem = configPath.getFileSystem(); - this.readPath = readPath; - this.resources = new ConcurrentHashMap<>(); - this.resourceTasks = new ConcurrentHashMap<>(); - this.resourceChangeListener = resourceChangeListener; - this.configWatcherTask = new ConfigWatcherTask(this.fileSystem, configChangeListener, readPath); + this.resolver = LOOKUP_RESOLVER.apply(fileSystem.provider().getScheme()); + this.watcher = newWatchService(fileSystem); + this.compoundKeys = new IdentityHashMap<>(); } - public void startWatchingConfig() throws Exception + public WatchKey register( + Path watchable, + WatchEvent.Kind... events) throws IOException { - configWatcherTask.watchConfig(configPath); - configWatcherTask.submit(); + return register(watchable, events, new WatchEvent.Modifier[0]); } - public void addResources( - NamespaceConfig namespace) + public WatchKey register( + Path watchable, + Kind[] events, + Modifier... modifiers) throws IOException { - namespace.resources.forEach(resource -> - { - resources.computeIfAbsent(resource, i -> - { - startWatchingResource(resource, namespace.name); - return ConcurrentHashMap.newKeySet(); - } - ).add(namespace.name); - resourceTasks.get(resource).addNamespace(namespace.name); - } - ); + WatchKey watchKey = null; + + if (watcher != null) + { + watchKey = registerImpl(watchable, events, modifiers); + } + + return watchKey; } - public void removeNamespace( - String namespace) + public WatchKey take() throws InterruptedException { - resources.entrySet().removeIf(e -> - { - String resource = e.getKey(); - Set namespaces = e.getValue(); - namespaces.remove(namespace); - boolean empty = namespaces.isEmpty(); - if (empty) - { - stopWatchingResource(resource); - } - else - { - removeNamespaceFromWatchedResource(resource, namespace); - } - return empty; - } - ); + return watcher != null ? takeImpl() : null; } - private void startWatchingResource( - String resource, - String namespace) + @Override + public void close() throws IOException { - try + if (watcher != null) { - ResourceWatcherTask watcherTask = new ResourceWatcherTask(fileSystem, resourceChangeListener, readPath); - watcherTask.addNamespace(namespace); - watcherTask.submit(); - Path resourcePath = configPath.resolveSibling(resource); - watcherTask.watchResource(resourcePath); - resourceTasks.put(resource, watcherTask); - System.out.printf("started watching resource: %s resourcePath: %s\n", resource, resourcePath); // TODO: Ati + watcher.close(); } - catch (Exception ex) + } + + private WatchKey registerImpl( + Path watchable, + Kind[] events, + Modifier... modifiers) throws IOException + { + Set watchPaths = resolver.apply(watchable); + Set watchKeys = new HashSet<>(); + + for (Path watchPath : watchPaths) { - rethrowUnchecked(ex); + WatchKey registeredKey = watchPath.register(watcher, events, modifiers); + watchKeys.add(registeredKey); } + + CompoundWatchKey compoundKey = new CompoundWatchKey(watchable, watchKeys); + watchKeys.forEach(k -> compoundKeys.put(k, compoundKey)); + + return compoundKey; } - private void stopWatchingResource( - String resource) + private WatchKey takeImpl() throws InterruptedException { - try + WatchKey watchKey = watcher.take(); + CompoundWatchKey compoundKey = compoundKeys.get(watchKey); + + return compoundKey; + } + + private final class CompoundWatchKey implements WatchKey + { + private final Path watchable; + private final Set keys; + private final List> events; + + CompoundWatchKey( + Path watchable, + Set keys) { - resourceTasks.remove(resource).close(); - System.out.println("stopped watching resource: " + resource); // TODO: Ati + this.watchable = watchable; + this.keys = keys; + this.events = new LinkedList<>(); } - catch (Exception ex) + + @Override + public boolean isValid() { - rethrowUnchecked(ex); + return keys.stream().allMatch(WatchKey::isValid); } - } - private void removeNamespaceFromWatchedResource( - String resource, - String namespace) - { - resourceTasks.get(resource).removeNamespace(namespace); + @Override + public List> pollEvents() + { + List> events = this.events; + + events.clear(); + for (WatchKey key : keys) + { + // TODO filter watch events + events.addAll(key.pollEvents()); + } + + return events; + } + + @Override + public boolean reset() + { + return keys.stream().allMatch(WatchKey::reset); + } + + @Override + public void cancel() + { + keys.stream().forEach(WatchKey::cancel); + keys.forEach(compoundKeys::remove); + } + + @Override + public Path watchable() + { + return watchable; + } } - public void close() + private static Set resolveWatchables( + Path watchable) { - resourceTasks.forEach((resource, watcherTask) -> + Set watchedPaths = new HashSet<>(); + + Deque observablePaths = new LinkedList<>(); + observablePaths.addLast(watchable); + + while (!observablePaths.isEmpty()) { - try + Path observablePath = observablePaths.removeFirst(); + + if (watchedPaths.add(observablePath)) { - watcherTask.close(); + if (Files.isSymbolicLink(observablePath)) + { + Path targetPath = readSymbolicLink(observablePath); + targetPath = watchable.resolveSibling(targetPath).normalize(); + observablePaths.addLast(targetPath); + } + + for (Path ancestorPath = observablePath.getParent(); + ancestorPath != null; + ancestorPath = ancestorPath.getParent()) + { + if (Files.isSymbolicLink(ancestorPath)) + { + if (watchedPaths.add(ancestorPath)) + { + Path targetPath = readSymbolicLink(ancestorPath); + observablePaths.addLast(ancestorPath.resolve(targetPath).normalize()); + } + } + } } - catch (Exception ex) + } + + Set watchables = new HashSet<>(); + for (Path watchedPath : watchedPaths) + { + Path parentPath = watchedPath.getParent(); + if (Files.exists(parentPath)) { - rethrowUnchecked(ex); + watchables.add(parentPath); } - }); + } + + return watchables; + } + + private static Path readSymbolicLink( + Path link) + { + Path target = null; + try { - configWatcherTask.close(); + target = Files.readSymbolicLink(link); + } + catch (IOException ex) + { + LangUtil.rethrowUnchecked(ex); + } + + return target; + } + + private static WatchService newWatchService( + FileSystem fileSystem) + { + WatchService watcher = null; + + try + { + watcher = fileSystem.newWatchService(); + } + catch (UnsupportedOperationException ex) + { + // no watcher } catch (Exception ex) { rethrowUnchecked(ex); } + + return watcher; } } diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcherTask.java deleted file mode 100644 index bbd4b1c8a3..0000000000 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/ResourceWatcherTask.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2021-2023 Aklivity Inc. - * - * Aklivity licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package io.aklivity.zilla.runtime.engine.internal.watcher; - -import static org.agrona.LangUtil.rethrowUnchecked; - -import java.io.IOException; -import java.nio.file.ClosedWatchServiceException; -import java.nio.file.FileSystem; -import java.nio.file.Path; -import java.nio.file.WatchKey; -import java.nio.file.WatchService; -import java.util.HashSet; -import java.util.IdentityHashMap; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Future; -import java.util.function.Consumer; -import java.util.function.Function; - -public class ResourceWatcherTask extends WatcherTask -{ - private final Map watchedItems; - private final WatchService watchService; - private final Consumer> resourceChangeListener; - private final Function readPath; - private final Set namespaces; - - public ResourceWatcherTask( - FileSystem fileSystem, - Consumer> resourceChangeListener, - Function readPath) - { - this.resourceChangeListener = resourceChangeListener; - this.readPath = readPath; - this.watchedItems = new IdentityHashMap<>(); - this.namespaces = new HashSet<>(); - WatchService watchService = null; - if (!"jar".equals(fileSystem.provider().getScheme())) // we can't watch in jar fs - { - try - { - watchService = fileSystem.newWatchService(); - } - catch (Exception ex) - { - rethrowUnchecked(ex); - } - } - this.watchService = watchService; - - } - - @Override - public Future submit() - { - return executor.submit(this); - } - - @Override - public Void call() - { - if (watchService != null) - { - while (true) - { - try - { - final WatchKey key = watchService.take(); - - WatchedItem watchedItem = watchedItems.get(key); - - if (watchedItem != null && watchedItem.isWatchedKey(key)) - { - // Even if no reconfigure needed, recalculation is necessary, since symlinks might have changed. - watchedItem.keys().forEach(watchedItems::remove); - watchedItem.unregister(); - watchedItem.register(); - watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); - String newText = readPath.apply(watchedItem.getPath()); - byte[] newHash = computeHash(newText); - if (watchedItem.isReconfigureNeeded(newHash)) - { - watchedItem.setHash(newHash); - if (resourceChangeListener != null) - { - resourceChangeListener.accept(namespaces); - } - } - } - } - catch (InterruptedException | ClosedWatchServiceException ex) - { - break; - } - } - } - return null; - } - - public void watchResource( - Path resourcePath) - { - WatchedItem watchedItem = new WatchedItem(resourcePath, watchService); - watchedItem.register(); - watchedItem.keys().forEach(k -> watchedItems.put(k, watchedItem)); - String resource = readPath.apply(resourcePath); - watchedItem.setHash(computeHash(resource)); - } - - public void addNamespace( - String namespace) - { - namespaces.add(namespace); - } - - public void removeNamespace( - String namespace) - { - namespaces.remove(namespace); - } - - @Override - public void close() throws IOException - { - if (watchService != null) - { - watchService.close(); - } - } -} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatchedItem.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatchedItem.java deleted file mode 100644 index b51077b13c..0000000000 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatchedItem.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright 2021-2023 Aklivity Inc. - * - * Aklivity licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package io.aklivity.zilla.runtime.engine.internal.watcher; - -import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; -import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; -import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; -import static org.agrona.LangUtil.rethrowUnchecked; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.WatchKey; -import java.nio.file.WatchService; -import java.util.Arrays; -import java.util.Deque; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.Set; - -public class WatchedItem -{ - private final WatchService watchService; - private final Set watchKeys; - private final Path watchedPath; - private byte[] hash; - - public WatchedItem( - Path watchedPath, - WatchService watchService) - { - this.watchService = watchService; - this.watchKeys = new HashSet<>(); - this.watchedPath = watchedPath; - } - - public Set keys() - { - return watchKeys; - } - - public void register() - { - try - { - String scheme = watchedPath.getFileSystem().provider().getScheme(); - if ("file".equals(scheme)) - { - registerFilePath(); - } - else if ("http".equals(scheme)) - { - watchKeys.add(watchedPath.register(watchService)); - } - } - catch (IOException ex) - { - rethrowUnchecked(ex); - } - } - - private void registerFilePath() throws IOException - { - Set watchedPaths = new HashSet<>(); - Deque observablePaths = new LinkedList<>(); - observablePaths.addLast(watchedPath); - - while (!observablePaths.isEmpty()) - { - Path observablePath = observablePaths.removeFirst(); - - if (watchedPaths.add(observablePath)) - { - if (Files.isSymbolicLink(observablePath)) - { - Path targetPath = Files.readSymbolicLink(observablePath); - targetPath = watchedPath.resolveSibling(targetPath).normalize(); - observablePaths.addLast(targetPath); - } - - for (Path ancestorPath = observablePath.getParent(); - ancestorPath != null; - ancestorPath = ancestorPath.getParent()) - { - if (Files.isSymbolicLink(ancestorPath)) - { - if (watchedPaths.add(ancestorPath)) - { - Path targetPath = Files.readSymbolicLink(ancestorPath); - observablePaths.addLast(ancestorPath.resolve(targetPath).normalize()); - } - } - } - } - } - for (Path watchedPath : watchedPaths) - { - if (Files.exists(watchedPath.getParent())) - { - WatchKey key = null; - try - { - key = watchedPath.getParent().register(watchService, ENTRY_MODIFY, ENTRY_CREATE, ENTRY_DELETE); - } - catch (IOException ex) - { - rethrowUnchecked(ex); - } - watchKeys.add(key); - } - } - } - - public void unregister() - { - watchKeys.forEach(WatchKey::cancel); - watchKeys.clear(); - } - - public boolean isWatchedKey( - WatchKey key) - { - return watchKeys.contains(key); - } - - public boolean isReconfigureNeeded( - byte[] newConfigHash) - { - return !Arrays.equals(hash, newConfigHash); - } - - public void setHash( - byte[] newHash) - { - hash = newHash; - } - - public Path getPath() - { - return watchedPath; - } -} diff --git a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatcherTask.java b/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatcherTask.java deleted file mode 100644 index 5dc5834fc4..0000000000 --- a/runtime/engine/src/main/java/io/aklivity/zilla/runtime/engine/internal/watcher/WatcherTask.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2021-2023 Aklivity Inc. - * - * Aklivity licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package io.aklivity.zilla.runtime.engine.internal.watcher; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.agrona.LangUtil.rethrowUnchecked; - -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.concurrent.Callable; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; - -public abstract class WatcherTask implements Callable, AutoCloseable -{ - private final MessageDigest md5; - - protected final ScheduledExecutorService executor; - - abstract Future submit(); - - protected WatcherTask() - { - this.md5 = initMessageDigest("MD5"); - this.executor = Executors.newScheduledThreadPool(2); - } - - protected byte[] computeHash( - String text) - { - return md5.digest(text.getBytes(UTF_8)); - } - - private MessageDigest initMessageDigest( - String algorithm) - { - MessageDigest md5 = null; - try - { - md5 = MessageDigest.getInstance(algorithm); - } - catch (NoSuchAlgorithmException ex) - { - rethrowUnchecked(ex); - } - return md5; - } -} diff --git a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/EngineTest.java b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/EngineTest.java index ba13dd0d77..4ae65aba4e 100644 --- a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/EngineTest.java +++ b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/EngineTest.java @@ -284,7 +284,6 @@ public static final class TestEngineExt implements EngineExtSpi public void onRegistered( EngineExtContext context) { - System.out.println("TestEngineExt onRegistered"); // TODO: Ati if (registerLatch != null) { registerLatch.countDown(); diff --git a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/ReconfigureHttpIT.java b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/ReconfigureHttpIT.java index efa7f4932d..5abc52c836 100644 --- a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/ReconfigureHttpIT.java +++ b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/internal/ReconfigureHttpIT.java @@ -23,7 +23,6 @@ import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.DisableOnDebug; @@ -36,7 +35,6 @@ import io.aklivity.zilla.runtime.engine.test.annotation.Configuration; import io.aklivity.zilla.runtime.engine.test.annotation.Configure; - public class ReconfigureHttpIT { public static final String ENGINE_CONFIG_POLL_INTERVAL_SECONDS = "zilla.engine.config.poll.interval.seconds"; @@ -68,63 +66,60 @@ public void setupRegisterLatch() throws Exception EngineTest.TestEngineExt.registerLatch = new CountDownLatch(1); } - @Ignore // TODO: Ati @Test - @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "1") - @Configuration("http://localhost:8080/") + @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "0") + @Configuration("http://localhost:8080/zilla.yaml") @Specification({ - "${app}/reconfigure.modify.via.http/server", - "${net}/reconfigure.modify.via.http/client" + "${app}/reconfigure.create.via.http/server", + "${net}/reconfigure.create.via.http/client" }) - public void shouldReconfigureWhenModifiedHttp() throws Exception + public void shouldReconfigureWhenCreatedViaHttp() throws Exception { k3po.start(); - k3po.awaitBarrier("CONNECTED"); EngineTest.TestEngineExt.registerLatch.await(); - k3po.notifyBarrier("CONFIG_CHANGED"); + k3po.notifyBarrier("CONFIG_CREATED"); k3po.finish(); } - @Ignore // TODO: Ati @Test - @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "1") - @Configuration("http://localhost:8080/") + @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "0") + @Configuration("http://localhost:8080/zilla.yaml") @Specification({ - "${app}/reconfigure.create.via.http/server", - "${net}/reconfigure.create.via.http/client" + "${app}/reconfigure.delete.via.http/server", + "${net}/reconfigure.delete.via.http/client" }) - public void shouldReconfigureWhenCreatedHttp() throws Exception + public void shouldReconfigureWhenDeletedViaHttp() throws Exception { k3po.start(); EngineTest.TestEngineExt.registerLatch.await(); - k3po.notifyBarrier("CONFIG_CREATED"); + k3po.notifyBarrier("CONFIG_DELETED"); k3po.finish(); } - @Ignore // TODO: Ati @Test - @Configuration("http://localhost:8080/") + @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "0") + @Configuration("http://localhost:8080/zilla.yaml") @Specification({ - "${app}/reconfigure.delete.via.http/server", - "${net}/reconfigure.delete.via.http/client" + "${app}/reconfigure.modify.via.http/server", + "${net}/reconfigure.modify.via.http/client" }) - public void shouldReconfigureWhenDeletedHttp() throws Exception + public void shouldReconfigureWhenModifiedViaHttp() throws Exception { k3po.start(); + k3po.awaitBarrier("CONNECTED"); EngineTest.TestEngineExt.registerLatch.await(); - k3po.notifyBarrier("CONFIG_DELETED"); + k3po.notifyBarrier("CONFIG_CHANGED"); k3po.finish(); } - @Ignore // TODO: Ati @Test - @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "1") - @Configuration("http://localhost:8080/") + @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "0") + @Configuration("http://localhost:8080/zilla.yaml") @Specification({ "${app}/reconfigure.modify.no.etag.via.http/server", "${net}/reconfigure.modify.no.etag.via.http/client" }) - public void shouldReconfigureWhenModifiedHttpEtagNotSupported() throws Exception + public void shouldReconfigureWhenModifiedViaHttpWithNoEtag() throws Exception { k3po.start(); k3po.awaitBarrier("CONNECTED"); @@ -133,15 +128,14 @@ public void shouldReconfigureWhenModifiedHttpEtagNotSupported() throws Exception k3po.finish(); } - @Ignore // TODO: Ati @Test - @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "1") - @Configuration("http://localhost:8080/") + @Configure(name = ENGINE_CONFIG_POLL_INTERVAL_SECONDS, value = "0") + @Configuration("http://localhost:8080/zilla.yaml") @Specification({ "${app}/reconfigure.server.error.via.http/server", "${net}/reconfigure.server.error.via.http/client" }) - public void shouldNotReconfigureWhenStatus500() throws Exception + public void shouldNotReconfigureViaHttpWhenServerError() throws Exception { k3po.start(); k3po.awaitBarrier("CHECK_RECONFIGURE"); diff --git a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/EngineRule.java b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/EngineRule.java index 05b1b4947b..3a455f474a 100644 --- a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/EngineRule.java +++ b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/EngineRule.java @@ -76,7 +76,7 @@ public final class EngineRule implements TestRule private Engine engine; private EngineConfiguration configuration; - private String configurationRoot; + private String configRoot; private Predicate exceptions; private boolean clean; @@ -125,7 +125,7 @@ public EngineRule configure( public EngineRule configurationRoot( String configurationRoot) { - this.configurationRoot = configurationRoot; + this.configRoot = configurationRoot; return this; } @@ -268,7 +268,6 @@ public Statement apply( { Class testClass = description.getTestClass(); final String testMethod = description.getMethodName().replaceAll("\\[.*\\]", ""); - URI jarUri = null; try { Configure[] configures = testClass @@ -285,18 +284,11 @@ public Statement apply( { configure(ENGINE_CONFIG_URL, configURI.toURL()); } - else if (configurationRoot != null) + else if (configRoot != null) { - String resourceName = String.format("%s/%s", configurationRoot, config.value()); + String resourceName = String.format("%s/%s", configRoot, config.value()); URL configURL = testClass.getClassLoader().getResource(resourceName); configure(ENGINE_CONFIG_URL, configURL); - if ("jar".equals(configURL.getProtocol())) - { - String jarLocation = Path.of( - configURL.toString().replace("jar:file:", "").split("!")[0] - ).toUri().toString(); - jarUri = new URI("jar", jarLocation, null); - } } else { @@ -314,14 +306,13 @@ else if (configurationRoot != null) } boolean allowErrors = exceptions.test(testMethod); - URI jarUri0 = jarUri; return new Statement() { @Override public void evaluate() throws Throwable { - EngineConfiguration config = configuration(); + final EngineConfiguration config = configuration(); final Thread baseThread = Thread.currentThread(); final List errors = new ArrayList<>(); final ErrorHandler errorHandler = ex -> @@ -332,9 +323,22 @@ public void evaluate() throws Throwable }; FileSystem fs = null; - if (jarUri0 != null) + + final URI configURI = config.configURI(); + + switch (configURI.getScheme()) { - fs = FileSystems.newFileSystem(jarUri0, Map.of()); + case "jar": + String jarLocation = Path.of( + configURI.toString().replace("jar:file:", "").split("!")[0] + ).toUri().toString(); + URI jarURI = new URI("jar", jarLocation, null); + fs = FileSystems.newFileSystem(jarURI, Map.of()); + break; + case "http": + final String pollInterval = String.format("PT%dS", config.configPollIntervalSeconds()); + fs = FileSystems.newFileSystem(configURI, Map.of("zilla.filesystem.http.poll.interval", pollInterval)); + break; } engine = builder.config(config) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java deleted file mode 100644 index cbefedb60d..0000000000 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemProvider.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2021-2023 Aklivity Inc - * - * Licensed under the Aklivity Community License (the "License"); you may not use - * this file except in compliance with the License. You may obtain a copy of the - * License at - * - * https://www.aklivity.io/aklivity-community-license/ - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package io.aklivity.zilla.runtime.filesystem.http; - -public final class HttpFileSystemProvider extends AbstractHttpFileSystemProvider -{ - private static final String SCHEME = "http"; - - @Override - public String getScheme() - { - return SCHEME; - } -} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java deleted file mode 100644 index 57ad2df059..0000000000 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpPath.java +++ /dev/null @@ -1,462 +0,0 @@ -/* - * Copyright 2021-2023 Aklivity Inc - * - * Licensed under the Aklivity Community License (the "License"); you may not use - * this file except in compliance with the License. You may obtain a copy of the - * License at - * - * https://www.aklivity.io/aklivity-community-license/ - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package io.aklivity.zilla.runtime.filesystem.http; - -import static java.net.HttpURLConnection.HTTP_NOT_FOUND; -import static java.net.HttpURLConnection.HTTP_NOT_MODIFIED; -import static java.net.HttpURLConnection.HTTP_OK; -import static java.net.http.HttpClient.Redirect.NORMAL; -import static java.net.http.HttpClient.Version.HTTP_2; -import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; -import static java.util.Objects.requireNonNull; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.ProviderMismatchException; -import java.nio.file.WatchEvent; -import java.nio.file.WatchKey; -import java.nio.file.WatchService; -import java.util.Iterator; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -public class HttpPath implements Path -{ - private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder() - .version(HTTP_2) - .followRedirects(NORMAL) - .build(); - private static final byte[] EMPTY_BODY = new byte[0]; - - private final HttpFileSystem fs; - private final URI location; - private final ScheduledExecutorService executor; - private int pollSeconds; // TODO: Ati - HttpFileSystemConfiguration -> final - - private byte[] body; - private String etag; - private HttpWatchService.HttpWatchKey watchKey; - private CompletableFuture future; - - HttpPath( - HttpFileSystem fs, - URI location) - { - if (!fs.provider().getScheme().equals(location.getScheme())) - { - throw new IllegalArgumentException(String.format("invalid protocol: %s", location.getScheme())); - } - System.out.println("HP constructor location " + location); // TODO: Ati - this.fs = fs; - this.location = location; - this.executor = Executors.newScheduledThreadPool(2); - // TODO: Ati - HttpFileSystemConfiguration - //this.pollSeconds = 30; - this.pollSeconds = 3; - } - - HttpPath() - { - this.fs = null; - this.location = null; - this.executor = null; - this.pollSeconds = 0; - } - - @Override - public HttpFileSystem getFileSystem() - { - return fs; - } - - @Override - public boolean isAbsolute() - { - return true; - } - - @Override - public Path getRoot() - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public Path getFileName() - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public Path getParent() - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public int getNameCount() - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public Path getName( - int index) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public Path subpath( - int beginIndex, - int endIndex) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public boolean startsWith( - Path other) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public boolean startsWith( - String other) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public boolean endsWith( - Path other) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public boolean endsWith( - String other) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public Path normalize() - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public Path resolve( - Path other) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public Path resolve( - String other) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public Path resolveSibling( - Path other) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public Path resolveSibling( - String other) - { - return fs.resolveSibling(other); - } - - @Override - public Path relativize( - Path other) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public URI toUri() - { - return location; - } - - @Override - public Path toAbsolutePath() - { - return this; - } - - @Override - public Path toRealPath( - LinkOption... options) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public File toFile() - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public WatchKey register( - WatchService watcher, - WatchEvent.Kind[] events, - WatchEvent.Modifier... modifiers) - throws IOException - { - requireNonNull(watcher); - if (!(watcher instanceof HttpWatchService)) - { - throw new ProviderMismatchException(); - } - watchKey = ((HttpWatchService) watcher).register(this, events, modifiers); - return watchKey; - } - - @Override - public WatchKey register( - WatchService watcher, - WatchEvent.Kind... events) throws IOException - { - System.out.println("HP register"); // TODO: Ati - return register(watcher, events, new WatchEvent.Modifier[0]); - } - - @Override - public Iterator iterator() - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public int compareTo( - Path other) - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public String toString() - { - return location.toString(); - } - - @Override - public boolean equals( - Object o) - { - if (this == o) - { - return true; - } - if (o == null || getClass() != o.getClass()) - { - return false; - } - - HttpPath path = (HttpPath) o; - return Objects.equals(location, path.location); - } - - @Override - public int hashCode() - { - return Objects.hashCode(location); - } - - // TODO: Ati - HttpFileSystemConfiguration - public void pollSeconds( - int pollSeconds) - { - this.pollSeconds = pollSeconds; - } - - byte[] readBody() - { - System.out.println("HP readBody"); // TODO: Ati - try - { - HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() - .GET() - .uri(location); - if (etag != null && !etag.isEmpty()) - { - requestBuilder = requestBuilder.headers("If-None-Match", etag); - } - HttpRequest request = requestBuilder.build(); - System.out.println("HP readBody path " + location + " request " + request + " etag " + etag); // TODO: Ati - HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofByteArray()); - System.out.println("HP readBody response " + response); // TODO: Ati - //System.out.println("HP readBody response.body " + new String(response.body())); // TODO: Ati - if (response.statusCode() == HTTP_OK) - { - body = response.body(); - Optional etagOptional = response.headers().firstValue("Etag"); - // TODO: Ati - calculate and store hash if there is no etag - if (etagOptional.isPresent()) - { - etag = etagOptional.get(); - } - } - else if (response.statusCode() == HTTP_NOT_FOUND) - { - body = new byte[0]; - etag = null; - } - else if (response.statusCode() == HTTP_NOT_MODIFIED) - { - // no op - } - } - catch (Exception ex) - { - System.out.println("HP readBody exception " + ex); // TODO: Ati - body = new byte[0]; - } - return body; - } - - void watch() - { - scheduleWatchBody(this.pollSeconds); - } - - private void scheduleWatchBody( - int pollSeconds) - { - if (pollSeconds == 0) - { - watchBody(); - } - else - { - executor.schedule(this::watchBody, pollSeconds, TimeUnit.SECONDS); - } - } - - private void watchBody() - { - HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() - .GET() - .uri(location); - if (etag != null && !etag.isEmpty()) - { - requestBuilder = requestBuilder.headers("If-None-Match", etag, "Prefer", "wait=86400"); - } - HttpRequest request = requestBuilder.build(); - System.out.println("HP watchBody path " + location + " request " + request + " etag " + etag); // TODO: Ati - future = HTTP_CLIENT.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) - .thenAccept(this::handleResponse) - .exceptionally(this::handleException); - } - - private void handleResponse( - HttpResponse response) - { - System.out.println("HP handleResponse response: " + response); // TODO: Ati - System.out.println("HP handleResponse response.headers: " + response.headers()); // TODO: Ati - int statusCode = response.statusCode(); - int pollSeconds = 1; - if (statusCode == 404) - { - body = EMPTY_BODY; - watchKey.addEvent(ENTRY_MODIFY, this); - pollSeconds = this.pollSeconds; - } - else if (statusCode >= 500 && statusCode <= 599) - { - body = null; - pollSeconds = this.pollSeconds; - } - else - { - System.out.println("HP handleResponse body " + new String(response.body())); - Optional etagOptional = response.headers().firstValue("Etag"); - if (etagOptional.isPresent()) - { - String newEtag = etagOptional.get(); - if (!newEtag.equals(etag)) - { - etag = newEtag; - body = response.body(); - watchKey.addEvent(ENTRY_MODIFY, this); - } - else if (response.statusCode() != 304) - { - body = response.body(); - pollSeconds = this.pollSeconds; - } - } - else - { - // TODO: Ati - hash - //byte[] hash = hashes.get(path); - //byte[] newHash = computeHash(body); - //if (!Arrays.equals(hash, newHash)) - { - //hashes.put(path, newHash); // TODO: Ati - //addEvent(ENTRY_MODIFY, path); - watchKey.addEvent(ENTRY_MODIFY, this); - //addEvent(path); - } - pollSeconds = this.pollSeconds; - } - } - scheduleWatchBody(pollSeconds); - } - - void cancel() - { - System.out.println("HP cancel"); // TODO: Ati - body = null; - etag = null; - future.cancel(true); - } - - // required for testing - public void shutdown() - { - System.out.println("HP shutdown"); // TODO: Ati - executor.shutdownNow(); - } - - private Void handleException( - Throwable throwable) - { - System.out.println("HP handleException " + throwable.getMessage()); // TODO: Ati - return null; - } -} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java deleted file mode 100644 index b7abc3655a..0000000000 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpWatchService.java +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Copyright 2021-2023 Aklivity Inc - * - * Licensed under the Aklivity Community License (the "License"); you may not use - * this file except in compliance with the License. You may obtain a copy of the - * License at - * - * https://www.aklivity.io/aklivity-community-license/ - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package io.aklivity.zilla.runtime.filesystem.http; - -import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; -import static org.agrona.LangUtil.rethrowUnchecked; - -import java.nio.file.ClosedWatchServiceException; -import java.nio.file.Path; -import java.nio.file.WatchEvent; -import java.nio.file.WatchKey; -import java.nio.file.WatchService; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -public class HttpWatchService implements WatchService -{ - private static final HttpPath CLOSE_PATH = new HttpPath(); - - private final WatchKey closeKey = new HttpWatchKey(CLOSE_PATH); - - private final List watchKeys; - private final LinkedBlockingQueue pendingKeys; - private final MessageDigest md5; - - private volatile boolean closed; - - public HttpWatchService() - { - - this.watchKeys = Collections.synchronizedList(new LinkedList<>()); - this.pendingKeys = new LinkedBlockingQueue<>(); - this.md5 = initMessageDigest("MD5"); - this.closed = false; - } - - @Override - public void close() - { - System.out.println("HWS close"); // TODO: Ati - watchKeys.forEach(HttpWatchKey::cancel); - watchKeys.clear(); - pendingKeys.clear(); - pendingKeys.offer(closeKey); - closed = true; - } - - @Override - public WatchKey poll() - { - checkOpen(); - WatchKey key = pendingKeys.poll(); - checkKey(key); - return key; - } - - @Override - public WatchKey poll( - long timeout, - TimeUnit unit) throws InterruptedException - { - checkOpen(); - WatchKey key = pendingKeys.poll(timeout, unit); - checkKey(key); - return key; - } - - @Override - public WatchKey take() throws InterruptedException - { - checkOpen(); - WatchKey key = pendingKeys.take(); - checkKey(key); - return key; - } - - private void checkOpen() - { - if (closed) - { - throw new ClosedWatchServiceException(); - } - } - - private void checkKey( - WatchKey key) - { - if (key == closeKey) - { - enqueueKey(closeKey); - } - checkOpen(); - } - - private void enqueueKey( - WatchKey key) - { - pendingKeys.offer(key); - } - - HttpWatchKey register( - final HttpPath path, - WatchEvent.Kind[] events, - WatchEvent.Modifier... modifiers) - { - for (WatchEvent.Kind event : events) - { - if (!event.equals(ENTRY_MODIFY)) - { - throw new IllegalArgumentException("Only ENTRY_MODIFY event kind is supported"); - } - } - if (modifiers.length > 0) - { - throw new IllegalArgumentException("Modifiers are not supported"); - } - System.out.printf("HWS register path: %s\n", path); // TODO: Ati - HttpWatchKey watchKey = new HttpWatchKey(path); - watchKey.watch(); - watchKeys.add(watchKey); - return watchKey; - } - - // TODO: Ati - private byte[] computeHash( - byte[] body) - { - return md5.digest(body); - } - - // TODO: Ati - private MessageDigest initMessageDigest( - String algorithm) - { - MessageDigest md5 = null; - try - { - md5 = MessageDigest.getInstance(algorithm); - } - catch (NoSuchAlgorithmException ex) - { - rethrowUnchecked(ex); - } - return md5; - } - - public final class HttpWatchKey implements WatchKey - { - private final HttpPath path; - - private List> events = Collections.synchronizedList(new LinkedList<>()); - - private volatile boolean valid; - - private HttpWatchKey( - HttpPath path) - { - this.path = path; - this.valid = true; - } - - @Override - public boolean isValid() - { - return valid; - } - - @Override - public List> pollEvents() - { - List> result = events; - events = Collections.synchronizedList(new LinkedList<>()); - return result; - } - - @Override - public boolean reset() - { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public void cancel() - { - System.out.println("HWK cancel"); // TODO: Ati - watchKeys.remove(this); - path.cancel(); - valid = false; - } - - @Override - public HttpPath watchable() - { - return path; - } - - void addEvent( - WatchEvent.Kind kind, - Path context) - { - events.add(new Event<>(kind, context)); - enqueueKey(this); - } - - @Override - public boolean equals(Object o) - { - if (this == o) - { - return true; - } - if (o == null || getClass() != o.getClass()) - { - return false; - } - - HttpWatchKey watchKey = (HttpWatchKey) o; - return Objects.equals(path, watchKey.path); - } - - @Override - public int hashCode() - { - return Objects.hashCode(path); - } - - void watch() - { - if (valid) - { - path.watch(); - } - } - - private static class Event implements WatchEvent - { - private final WatchEvent.Kind kind; - private final T context; - private final int count; - - Event( - WatchEvent.Kind type, - T context) - { - this.kind = type; - this.context = context; - this.count = 1; - } - - @Override - public WatchEvent.Kind kind() - { - return kind; - } - - @Override - public T context() - { - return context; - } - - @Override - public int count() - { - return count; - } - } - } -} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystem.java similarity index 61% rename from runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java rename to runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystem.java index f3b03899bd..4f84312cea 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystem.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystem.java @@ -12,37 +12,52 @@ * WARRANTIES OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package io.aklivity.zilla.runtime.filesystem.http; +package io.aklivity.zilla.runtime.filesystem.http.internal; +import static java.net.http.HttpClient.Redirect.NORMAL; +import static java.net.http.HttpClient.Version.HTTP_2; import static java.util.Objects.requireNonNull; import static org.agrona.LangUtil.rethrowUnchecked; import java.net.URI; +import java.net.http.HttpClient; import java.nio.file.FileStore; import java.nio.file.FileSystem; import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.attribute.UserPrincipalLookupService; -import java.nio.file.spi.FileSystemProvider; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Map; import java.util.Set; public final class HttpFileSystem extends FileSystem { - private static final String SEPARATOR = "/"; + private static final String HTTP_PATH_SEPARATOR = "/"; - private final AbstractHttpFileSystemProvider provider; + private final HttpFileSystemProvider provider; private final URI root; + private final HttpFileSystemConfiguration config; + private final HttpClient client; + private final MessageDigest md5; HttpFileSystem( - AbstractHttpFileSystemProvider provider, - URI root) + HttpFileSystemProvider provider, + URI root, + Map env) { this.provider = provider; this.root = root; + this.config = new HttpFileSystemConfiguration(env); + this.client = HttpClient.newBuilder() + .version(HTTP_2) + .followRedirects(NORMAL) + .build(); + this.md5 = initMessageDigest("MD5"); } @Override - public FileSystemProvider provider() + public HttpFileSystemProvider provider() { return provider; } @@ -68,7 +83,7 @@ public boolean isReadOnly() @Override public String getSeparator() { - return SEPARATOR; + return HTTP_PATH_SEPARATOR; } @Override @@ -96,17 +111,18 @@ public Path getPath( { requireNonNull(first); requireNonNull(more); - String path = more.length > 0 ? first + SEPARATOR + String.join(SEPARATOR, more) : first; - Path result = null; - try - { - result = new HttpPath(this, URI.create(path)); - } - catch (Exception ex) - { - rethrowUnchecked(ex); - } - return result; + + String path = more.length > 0 + ? first + HTTP_PATH_SEPARATOR + String.join(HTTP_PATH_SEPARATOR, more) + : first; + + return getPath(URI.create(path)); + } + + public HttpPath getPath( + URI uri) + { + return new HttpPath(this, uri); } @Override @@ -125,12 +141,28 @@ public UserPrincipalLookupService getUserPrincipalLookupService() @Override public HttpWatchService newWatchService() { - return new HttpWatchService(); + return new HttpWatchService(config); } - HttpPath resolveSibling( - String other) + HttpClient client() { - return new HttpPath(this, root.resolve(URI.create(other))); + return client; + } + + private static MessageDigest initMessageDigest( + String algorithm) + { + MessageDigest md5 = null; + + try + { + md5 = MessageDigest.getInstance(algorithm); + } + catch (NoSuchAlgorithmException ex) + { + rethrowUnchecked(ex); + } + + return md5; } } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystemConfiguration.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystemConfiguration.java new file mode 100644 index 0000000000..c9757554db --- /dev/null +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystemConfiguration.java @@ -0,0 +1,40 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http.internal; + +import java.time.Duration; +import java.util.Map; +import java.util.Objects; + +public final class HttpFileSystemConfiguration +{ + public static final String POLL_INTERVAL_PROPERTY_NAME = "zilla.filesystem.http.poll.interval"; + + private static final Duration POLL_INTERVAL_PROPERTY_DEFAULT = Duration.parse("PT30S"); + + private final Map env; + + HttpFileSystemConfiguration( + Map env) + { + this.env = Objects.requireNonNull(env); + } + + public Duration pollInterval() + { + String value = env != null ? (String) env.get(POLL_INTERVAL_PROPERTY_NAME) : null; + return value != null ? Duration.parse(value) : POLL_INTERVAL_PROPERTY_DEFAULT; + } +} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystemProvider.java similarity index 83% rename from runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java rename to runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystemProvider.java index 8269c9ede6..14587c81f9 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/AbstractHttpFileSystemProvider.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystemProvider.java @@ -12,7 +12,7 @@ * WARRANTIES OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package io.aklivity.zilla.runtime.filesystem.http; +package io.aklivity.zilla.runtime.filesystem.http.internal; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -36,42 +36,40 @@ import java.nio.file.attribute.FileAttributeView; import java.nio.file.spi.FileSystemProvider; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -public abstract class AbstractHttpFileSystemProvider extends FileSystemProvider +public class HttpFileSystemProvider extends FileSystemProvider { private final Map fileSystems = new ConcurrentHashMap<>(); @Override - public abstract String getScheme(); + public String getScheme() + { + return "http"; + } @Override + @SuppressWarnings("resource") public FileSystem newFileSystem( URI uri, Map env) { - checkUri(uri); - HttpFileSystem hfs = fileSystems.get(uri); - if (hfs == null) - { - hfs = new HttpFileSystem(this, uri); - fileSystems.put(uri, hfs); - } - else - { - throw new FileSystemAlreadyExistsException(); - } - return hfs; + checkURI(uri); + + URI rootURI = uri.resolve("/"); + return fileSystems.compute(rootURI, (u, fs) -> computeHttpFileSystem(u, fs, env)); } @Override public FileSystem getFileSystem( URI uri) { - checkUri(uri); - HttpFileSystem hfs = fileSystems.get(uri); + checkURI(uri); + URI rootURI = uri.resolve("/"); + HttpFileSystem hfs = fileSystems.get(rootURI); if (hfs == null) { throw new FileSystemNotFoundException(); @@ -83,12 +81,10 @@ public FileSystem getFileSystem( public Path getPath( URI uri) { - FileSystem hfs = fileSystems.get(uri); - if (hfs == null) - { - hfs = newFileSystem(uri, Map.of()); - } - return hfs.getPath(uri.toString()); + checkURI(uri); + URI rootURI = uri.resolve("/"); + HttpFileSystem hfs = fileSystems.computeIfAbsent(rootURI, this::newHttpFileSystem); + return hfs.getPath(uri); } @Override @@ -104,8 +100,8 @@ public InputStream newInputStream( Path path, OpenOption... options) { - checkPath(path); - return new ByteArrayInputStream(readBody((HttpPath) path)); + HttpPath httpPath = checkPath(path); + return new ByteArrayInputStream(httpPath.readBody()); } @Override @@ -141,8 +137,8 @@ public SeekableByteChannel newByteChannel( Set options, FileAttribute... attrs) { - checkPath(path); - return new ReadOnlyByteArrayChannel(readBody((HttpPath) path)); + HttpPath httpPath = checkPath(path); + return new ReadOnlyByteArrayChannel(httpPath.readBody()); } @Override @@ -243,7 +239,7 @@ public void checkAccess( Path path, AccessMode... modes) { - throw new UnsupportedOperationException("not implemented"); + Objects.requireNonNull(path); } @Override @@ -288,31 +284,46 @@ void closeFileSystem( fileSystems.remove(uri); } - private void checkUri( + private void checkURI( URI uri) { if (!uri.getScheme().equalsIgnoreCase(getScheme())) { throw new IllegalArgumentException("URI does not match this provider"); } + if (uri.getPath() == null) { throw new IllegalArgumentException("Path component is undefined"); } } - private void checkPath( + private HttpPath checkPath( Path path) { if (!path.getFileSystem().provider().getScheme().equalsIgnoreCase(getScheme())) { throw new IllegalArgumentException("Scheme does not match this provider"); } + return HttpPath.class.cast(path); + } + + private HttpFileSystem computeHttpFileSystem( + URI uri, + HttpFileSystem hfs, + Map env) + { + if (hfs != null) + { + throw new FileSystemAlreadyExistsException(); + } + + return new HttpFileSystem(this, uri, env); } - private byte[] readBody( - HttpPath path) + private HttpFileSystem newHttpFileSystem( + URI uri) { - return path.readBody(); + return new HttpFileSystem(this, uri, Map.of()); } } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java new file mode 100644 index 0000000000..39b0169df9 --- /dev/null +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java @@ -0,0 +1,369 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http.internal; + +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import static java.net.HttpURLConnection.HTTP_NOT_MODIFIED; +import static java.net.HttpURLConnection.HTTP_OK; +import static java.util.Objects.requireNonNull; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.ProviderMismatchException; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.Iterator; +import java.util.Objects; + +public final class HttpPath implements Path +{ + private static final byte[] EMPTY_BODY = new byte[0]; + + private final HttpFileSystem fs; + private final URI location; + + private volatile byte[] body; + private volatile String etag; + + private volatile int changeCount; + private volatile int readCount; + + HttpPath( + HttpFileSystem fs, + URI location) + { + this.fs = Objects.requireNonNull(fs); + this.location = Objects.requireNonNull(location); + } + + @Override + public HttpFileSystem getFileSystem() + { + return fs; + } + + @Override + public boolean isAbsolute() + { + return true; + } + + @Override + public Path getRoot() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path getFileName() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path getParent() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public int getNameCount() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path getName( + int index) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path subpath( + int beginIndex, + int endIndex) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean startsWith( + Path other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean startsWith( + String other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean endsWith( + Path other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public boolean endsWith( + String other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path normalize() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path resolve( + Path other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path resolve( + String other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path resolveSibling( + Path other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public Path resolveSibling( + String other) + { + return new HttpPath(fs, location.resolve(URI.create(other))); + } + + @Override + public Path relativize( + Path other) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public URI toUri() + { + return location; + } + + @Override + public Path toAbsolutePath() + { + return this; + } + + @Override + public Path toRealPath( + LinkOption... options) + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public File toFile() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public WatchKey register( + WatchService watcher, + WatchEvent.Kind... events) throws IOException + { + return register(watcher, events, new WatchEvent.Modifier[0]); + } + + @Override + public WatchKey register( + WatchService watcher, + WatchEvent.Kind[] events, + WatchEvent.Modifier... modifiers) + throws IOException + { + HttpWatchService httpWatcher = checkWatcher(watcher); + return httpWatcher.register(this, events, modifiers); + } + + @Override + public Iterator iterator() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public int compareTo( + Path other) + { + HttpPath that = (HttpPath) other; + + return location.compareTo(that.location); + } + + @Override + public String toString() + { + return location.toString(); + } + + @Override + public boolean equals( + Object o) + { + if (this == o) + { + return true; + } + if (o == null || getClass() != o.getClass()) + { + return false; + } + + HttpPath path = (HttpPath) o; + return Objects.equals(location, path.location); + } + + @Override + public int hashCode() + { + return Objects.hashCode(location); + } + + byte[] readBody() + { + if (readCount == changeCount) + { + try + { + HttpClient client = fs.client(); + HttpRequest request = newReadRequest(); + HttpResponse response = client.send(request, BodyHandlers.ofByteArray()); + success(response); + } + catch (Exception ex) + { + failure(ex); + } + } + + readCount = changeCount; + + return body; + } + + void success( + HttpResponse response) + { + switch (response.statusCode()) + { + case HTTP_OK: + body = response.body(); + // TODO: Ati - calculate and store hash if there is no etag + etag = response.headers().firstValue("Etag").orElse(null); + changeCount++; + break; + case HTTP_NOT_FOUND: + body = HttpPath.EMPTY_BODY; + etag = null; + changeCount++; + break; + case HTTP_NOT_MODIFIED: + break; + } + } + + Void failure( + Throwable ex) + { + body = HttpPath.EMPTY_BODY; + etag = null; + changeCount++; + return null; + } + + int changeCount() + { + return changeCount; + } + + boolean exists() + { + return body != null; + } + + private HttpRequest newReadRequest() + { + HttpRequest.Builder request = HttpRequest.newBuilder() + .GET() + .uri(location); + + if (etag != null && !etag.isEmpty()) + { + request = request.headers("If-None-Match", etag); + } + + return request.build(); + } + + HttpRequest newWatchRequest() + { + HttpRequest.Builder request = HttpRequest.newBuilder() + .GET() + .uri(location); + + if (etag != null) + { + request = request.headers("If-None-Match", etag, "Prefer", "wait=86400"); + } + + return request.build(); + } + + private HttpWatchService checkWatcher( + WatchService watcher) + { + requireNonNull(watcher); + + if (!(watcher instanceof HttpWatchService)) + { + throw new ProviderMismatchException(); + } + + return (HttpWatchService) watcher; + } +} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpWatchService.java new file mode 100644 index 0000000000..52d0b5e870 --- /dev/null +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpWatchService.java @@ -0,0 +1,384 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.filesystem.http.internal; + +import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; +import static org.agrona.LangUtil.rethrowUnchecked; + +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.nio.file.ClosedWatchServiceException; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.time.Duration; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public final class HttpWatchService implements WatchService +{ + private final WatchKey closeKey = new HttpWatchKey(); + + private final Duration pollInterval; + private final Collection watchKeys; + private final BlockingQueue pendingKeys; + private final ScheduledExecutorService executor; + + private volatile boolean closed; + + HttpWatchService( + HttpFileSystemConfiguration config) + { + this.pollInterval = config.pollInterval(); + this.watchKeys = new ConcurrentSkipListSet<>(); + this.pendingKeys = new LinkedBlockingQueue<>(); + this.executor = Executors.newScheduledThreadPool(2); + } + + @Override + public void close() + { + watchKeys.forEach(HttpWatchKey::cancel); + watchKeys.clear(); + + closed = true; + pendingKeys.clear(); + pendingKeys.offer(closeKey); + + executor.shutdownNow(); + + try + { + executor.awaitTermination(5, TimeUnit.SECONDS); + } + catch (InterruptedException ex) + { + rethrowUnchecked(ex); + } + } + + @Override + public WatchKey poll() + { + checkOpen(); + WatchKey key = pendingKeys.poll(); + return key != closeKey ? key : null; + } + + @Override + public WatchKey poll( + long timeout, + TimeUnit unit) throws InterruptedException + { + checkOpen(); + WatchKey key = pendingKeys.poll(timeout, unit); + return key != closeKey ? key : null; + } + + @Override + public WatchKey take() throws InterruptedException + { + checkOpen(); + WatchKey key = pendingKeys.take(); + return key != closeKey ? key : null; + } + + HttpWatchKey register( + HttpPath path, + WatchEvent.Kind[] events, + WatchEvent.Modifier... modifiers) + { + checkEvents(events); + checkModifiers(modifiers); + + HttpWatchKey watchKey = new HttpWatchKey(this, path); + watchKeys.add(watchKey); + watchKey.watch(); + + return watchKey; + } + + private void checkOpen() + { + if (closed) + { + throw new ClosedWatchServiceException(); + } + } + + private void checkEvents( + WatchEvent.Kind[] events) + { + for (WatchEvent.Kind event : events) + { + if (!event.equals(ENTRY_CREATE) && + !event.equals(ENTRY_MODIFY) && + !event.equals(ENTRY_DELETE)) + { + throw new IllegalArgumentException(String.format("%s event kind not supported", event)); + } + } + } + + private void checkModifiers( + WatchEvent.Modifier[] modifiers) + { + if (modifiers.length > 0) + { + throw new IllegalArgumentException("Modifiers are not supported"); + } + } + + private void watchBody( + HttpWatchKey watchKey) + { + final long pollSeconds = pollInterval.getSeconds(); + executor.schedule(watchKey::watchBody, pollSeconds, TimeUnit.SECONDS); + } + + private void signalKey( + HttpWatchKey watchKey) + { + pendingKeys.offer(watchKey); + } + + private void cancelKey( + HttpWatchKey watchKey) + { + watchKeys.remove(watchKey); + } + + private static final class HttpWatchKey implements WatchKey, Comparable + { + private final HttpWatchService watcher; + private final HttpPath path; + + private List> watchEvents = Collections.synchronizedList(new LinkedList<>()); + + private volatile boolean valid; + private volatile CompletableFuture future; + + private HttpWatchKey() + { + this.watcher = null; + this.path = null; + this.valid = false; + } + + private HttpWatchKey( + HttpWatchService watcher, + HttpPath path) + { + this.watcher = watcher; + this.path = path; + this.valid = true; + } + + @Override + public boolean isValid() + { + return valid; + } + + @Override + public List> pollEvents() + { + List> result = watchEvents; + watchEvents = Collections.synchronizedList(new LinkedList<>()); + return result; + } + + @Override + public boolean reset() + { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void cancel() + { + future.cancel(true); + watcher.cancelKey(this); + valid = false; + } + + @Override + public HttpPath watchable() + { + return path; + } + + @Override + public boolean equals( + Object o) + { + if (this == o) + { + return true; + } + if (o == null || getClass() != o.getClass()) + { + return false; + } + + HttpWatchKey that = (HttpWatchKey) o; + return Objects.equals(path, that.path); + } + + @Override + public int hashCode() + { + return Objects.hashCode(path); + } + + @Override + public int compareTo( + HttpWatchKey that) + { + return path.compareTo(that.path); + } + + private void watch() + { + if (valid) + { + watcher.watchBody(this); + } + } + + private void watchBody() + { + HttpClient client = path.getFileSystem().client(); + HttpRequest request = path.newWatchRequest(); + + this.future = client.sendAsync(request, BodyHandlers.ofByteArray()) + .thenAccept(this::success) + .exceptionally(this::failure); + } + + private void success( + HttpResponse response) + { + int changeCount = path.changeCount(); + boolean exists = path.exists(); + + path.success(response); + + if (path.changeCount() != changeCount) + { + if (exists == path.exists()) + { + signalEvent(ENTRY_MODIFY); + } + else if (exists) + { + signalEvent(ENTRY_DELETE); + } + else + { + signalEvent(ENTRY_CREATE); + } + } + + watcher.watchBody(this); + } + + private Void failure( + Throwable ex) + { + int changeCount = path.changeCount(); + boolean exists = path.exists(); + + path.failure(ex); + + if (path.changeCount() != changeCount) + { + if (exists == path.exists()) + { + signalEvent(ENTRY_MODIFY); + } + else if (exists) + { + signalEvent(ENTRY_DELETE); + } + else + { + signalEvent(ENTRY_CREATE); + } + } + + // (back off)? + watcher.watchBody(this); + + return null; + } + + private void signalEvent( + WatchEvent.Kind kind) + { + watchEvents.add(new HttpWatchEvent(kind, path)); + watcher.signalKey(this); + } + + private static class HttpWatchEvent implements WatchEvent + { + private final WatchEvent.Kind kind; + private final Path context; + private final int count; + + HttpWatchEvent( + WatchEvent.Kind type, + Path context) + { + this.kind = type; + this.context = context; + this.count = 1; + } + + @Override + public WatchEvent.Kind kind() + { + return kind; + } + + @Override + public Path context() + { + return context; + } + + @Override + public int count() + { + return count; + } + } + } +} diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpsFileSystemProvider.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpsFileSystemProvider.java similarity index 75% rename from runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpsFileSystemProvider.java rename to runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpsFileSystemProvider.java index 67c8419a98..156d88ef35 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/HttpsFileSystemProvider.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpsFileSystemProvider.java @@ -12,15 +12,13 @@ * WARRANTIES OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package io.aklivity.zilla.runtime.filesystem.http; +package io.aklivity.zilla.runtime.filesystem.http.internal; -public final class HttpsFileSystemProvider extends AbstractHttpFileSystemProvider +public class HttpsFileSystemProvider extends HttpFileSystemProvider { - private static final String SCHEME = "https"; - @Override public String getScheme() { - return SCHEME; + return "https"; } } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/ReadOnlyByteArrayChannel.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/ReadOnlyByteArrayChannel.java similarity index 94% rename from runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/ReadOnlyByteArrayChannel.java rename to runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/ReadOnlyByteArrayChannel.java index d075596dcd..b18bc64c1a 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/ReadOnlyByteArrayChannel.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/ReadOnlyByteArrayChannel.java @@ -12,20 +12,20 @@ * WARRANTIES OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package io.aklivity.zilla.runtime.filesystem.http; +package io.aklivity.zilla.runtime.filesystem.http.internal; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.nio.channels.SeekableByteChannel; -public class ReadOnlyByteArrayChannel implements SeekableByteChannel +final class ReadOnlyByteArrayChannel implements SeekableByteChannel { private final byte[] data; private int position; private boolean closed; - public ReadOnlyByteArrayChannel( + ReadOnlyByteArrayChannel( byte[] data) { this.data = data; diff --git a/runtime/filesystem-http/src/main/moditect/module-info.java b/runtime/filesystem-http/src/main/moditect/module-info.java index d35f116893..a953179fba 100644 --- a/runtime/filesystem-http/src/main/moditect/module-info.java +++ b/runtime/filesystem-http/src/main/moditect/module-info.java @@ -18,6 +18,6 @@ requires org.agrona.core; provides java.nio.file.spi.FileSystemProvider with - io.aklivity.zilla.runtime.filesystem.http.HttpFileSystemProvider, - io.aklivity.zilla.runtime.filesystem.http.HttpsFileSystemProvider; + io.aklivity.zilla.runtime.filesystem.http.internal.HttpFileSystemProvider, + io.aklivity.zilla.runtime.filesystem.http.internal.HttpsFileSystemProvider; } diff --git a/runtime/filesystem-http/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider b/runtime/filesystem-http/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider index 5628d02fb6..0801cb9df9 100644 --- a/runtime/filesystem-http/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider +++ b/runtime/filesystem-http/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider @@ -1,2 +1,2 @@ -io.aklivity.zilla.runtime.filesystem.http.HttpFileSystemProvider -io.aklivity.zilla.runtime.filesystem.http.HttpsFileSystemProvider +io.aklivity.zilla.runtime.filesystem.http.internal.HttpFileSystemProvider +io.aklivity.zilla.runtime.filesystem.http.internal.HttpsFileSystemProvider diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java index e95d780bf5..4201306075 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java @@ -14,6 +14,8 @@ */ package io.aklivity.zilla.runtime.filesystem.http; +import static io.aklivity.zilla.runtime.filesystem.http.internal.HttpFileSystemConfiguration.POLL_INTERVAL_PROPERTY_NAME; +import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; import static java.util.concurrent.TimeUnit.SECONDS; import static org.hamcrest.MatcherAssert.assertThat; @@ -22,12 +24,15 @@ import java.io.InputStream; import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.WatchEvent; import java.nio.file.WatchKey; -import java.nio.file.spi.FileSystemProvider; +import java.nio.file.WatchService; import java.util.List; +import java.util.Map; import org.junit.Rule; import org.junit.Test; @@ -46,7 +51,6 @@ public class HttpFileSystemIT private final TestRule timeout = new DisableOnDebug(new Timeout(10, SECONDS)); @Rule - //public final TestRule chain = outerRule(timeout).around(k3po); // TODO: Ati public final TestRule chain = outerRule(k3po).around(timeout); @Test @@ -56,16 +60,19 @@ public class HttpFileSystemIT public void shouldReadString() throws Exception { // GIVEN - String helloUrl = "http://localhost:8080/hello.txt"; - Path helloPath = Path.of(new URI(helloUrl)); - - // WHEN - k3po.start(); - String helloBody = Files.readString(helloPath); - k3po.finish(); - - // THEN - assertThat(helloBody, equalTo("Hello World!")); + URI helloURI = URI.create("http://localhost:8080/hello.txt"); + try (FileSystem fs = FileSystems.newFileSystem(helloURI, Map.of())) + { + Path helloPath = fs.getPath(helloURI.toString()); + + // WHEN + k3po.start(); + String helloBody = Files.readString(helloPath); + k3po.finish(); + + // THEN + assertThat(helloBody, equalTo("Hello World!")); + } } @Test @@ -75,18 +82,21 @@ public void shouldReadString() throws Exception public void shouldReadStringEtagNotModified() throws Exception { // GIVEN - String helloUrl = "http://localhost:8080/hello.txt"; - Path helloPath = Path.of(new URI(helloUrl)); - - // WHEN - k3po.start(); - String helloBody1 = Files.readString(helloPath); - String helloBody2 = Files.readString(helloPath); - k3po.finish(); - - // THEN - assertThat(helloBody1, equalTo("Hello World!")); - assertThat(helloBody2, equalTo("Hello World!")); + URI helloURI = URI.create("http://localhost:8080/hello.txt"); + try (FileSystem fs = FileSystems.newFileSystem(helloURI, Map.of())) + { + Path helloPath = fs.getPath(helloURI.toString()); + + // WHEN + k3po.start(); + String helloBody1 = Files.readString(helloPath); + String helloBody2 = Files.readString(helloPath); + k3po.finish(); + + // THEN + assertThat(helloBody1, equalTo("Hello World!")); + assertThat(helloBody2, equalTo("Hello World!")); + } } @Test @@ -96,18 +106,21 @@ public void shouldReadStringEtagNotModified() throws Exception public void shouldReadStringEtagModified() throws Exception { // GIVEN - String helloUrl = "http://localhost:8080/hello.txt"; - Path helloPath = Path.of(new URI(helloUrl)); - - // WHEN - k3po.start(); - String helloBody1 = Files.readString(helloPath); - String helloBody2 = Files.readString(helloPath); - k3po.finish(); - - // THEN - assertThat(helloBody1, equalTo("Hello World!")); - assertThat(helloBody2, equalTo("Hello Universe!")); + URI helloURI = URI.create("http://localhost:8080/hello.txt"); + try (FileSystem fs = FileSystems.newFileSystem(helloURI, Map.of())) + { + Path helloPath = fs.getPath(helloURI.toString()); + + // WHEN + k3po.start(); + String helloBody1 = Files.readString(helloPath); + String helloBody2 = Files.readString(helloPath); + k3po.finish(); + + // THEN + assertThat(helloBody1, equalTo("Hello World!")); + assertThat(helloBody2, equalTo("Hello Universe!")); + } } @Test @@ -117,16 +130,19 @@ public void shouldReadStringEtagModified() throws Exception public void shouldReadStringNotFound() throws Exception { // GIVEN - String notFoundUrl = "http://localhost:8080/notfound.txt"; - Path notFoundPath = Path.of(new URI(notFoundUrl)); - - // WHEN - k3po.start(); - String notFoundBody = Files.readString(notFoundPath); - k3po.finish(); - - // THEN - assertThat(notFoundBody, equalTo("")); + URI notFoundURI = URI.create("http://localhost:8080/notfound.txt"); + try (FileSystem fs = FileSystems.newFileSystem(notFoundURI, Map.of())) + { + Path notFoundPath = fs.getPath(notFoundURI.toString()); + + // WHEN + k3po.start(); + String notFoundBody = Files.readString(notFoundPath); + k3po.finish(); + + // THEN + assertThat(notFoundBody, equalTo("")); + } } @Test @@ -136,18 +152,21 @@ public void shouldReadStringNotFound() throws Exception public void shouldReadStringNotFoundSuccess() throws Exception { // GIVEN - String helloUrl = "http://localhost:8080/hello.txt"; - Path helloPath = Path.of(new URI(helloUrl)); - - // WHEN - k3po.start(); - String helloBody1 = Files.readString(helloPath); - String helloBody2 = Files.readString(helloPath); - k3po.finish(); - - // THEN - assertThat(helloBody1, equalTo("")); - assertThat(helloBody2, equalTo("Hello World!")); + URI helloURI = URI.create("http://localhost:8080/hello.txt"); + try (FileSystem fs = FileSystems.newFileSystem(helloURI, Map.of())) + { + Path helloPath = fs.getPath(helloURI.toString()); + + // WHEN + k3po.start(); + String helloBody1 = Files.readString(helloPath); + String helloBody2 = Files.readString(helloPath); + k3po.finish(); + + // THEN + assertThat(helloBody1, equalTo("")); + assertThat(helloBody2, equalTo("Hello World!")); + } } @Test @@ -157,19 +176,21 @@ public void shouldReadStringNotFoundSuccess() throws Exception public void shouldReadInputStream() throws Exception { // GIVEN - String helloUrl = "http://localhost:8080/hello.txt"; - Path helloPath = Path.of(new URI(helloUrl)); - FileSystemProvider fsp = helloPath.getFileSystem().provider(); - - // WHEN - k3po.start(); - InputStream helloIs = fsp.newInputStream(helloPath); - String helloBody = new String(helloIs.readAllBytes()); - helloIs.close(); - k3po.finish(); - - // THEN - assertThat(helloBody, equalTo("Hello World!")); + URI helloURI = URI.create("http://localhost:8080/hello.txt"); + try (FileSystem fs = FileSystems.newFileSystem(helloURI, Map.of())) + { + Path helloPath = fs.getPath(helloURI.toString()); + + // WHEN + k3po.start(); + InputStream helloIs = Files.newInputStream(helloPath); + String helloBody = new String(helloIs.readAllBytes()); + helloIs.close(); + k3po.finish(); + + // THEN + assertThat(helloBody, equalTo("Hello World!")); + } } @Test @@ -179,19 +200,21 @@ public void shouldReadInputStream() throws Exception public void shouldReadInputStreamNotFound() throws Exception { // GIVEN - String notFoundUrl = "http://localhost:8080/notfound.txt"; - Path notFoundPath = Path.of(new URI(notFoundUrl)); - FileSystemProvider fsp = notFoundPath.getFileSystem().provider(); - - // WHEN - k3po.start(); - InputStream notFoundIs = fsp.newInputStream(notFoundPath); - String notFoundBody = new String(notFoundIs.readAllBytes()); - notFoundIs.close(); - k3po.finish(); - - // THEN - assertThat(notFoundBody, equalTo("")); + URI notFoundURI = URI.create("http://localhost:8080/notfound.txt"); + try (FileSystem fs = FileSystems.newFileSystem(notFoundURI, Map.of())) + { + Path notFoundPath = fs.getPath(notFoundURI.toString()); + + // WHEN + k3po.start(); + InputStream notFoundIs = Files.newInputStream(notFoundPath); + String notFoundBody = new String(notFoundIs.readAllBytes()); + notFoundIs.close(); + k3po.finish(); + + // THEN + assertThat(notFoundBody, equalTo("")); + } } @Test @@ -201,31 +224,39 @@ public void shouldReadInputStreamNotFound() throws Exception public void shouldWatch() throws Exception { // GIVEN - String url = "http://localhost:8080/hello.txt"; - Path path = Path.of(new URI(url)); - ((HttpPath) path).pollSeconds(1); // TODO: Ati - HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - - // WHEN - k3po.start(); - k3po.notifyBarrier("REGISTERED"); - path.register(watchService); - WatchKey key1 = watchService.take(); - List> events1 = key1.pollEvents(); - k3po.notifyBarrier("MODIFIED"); - WatchKey key2 = watchService.take(); - List> events2 = key2.pollEvents(); - watchService.close(); - ((HttpPath) path).shutdown(); - k3po.finish(); - - // THEN - assertThat(events1.size(), equalTo(1)); - assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); - assertThat(events1.get(0).context(), equalTo(path)); - assertThat(events2.size(), equalTo(1)); - assertThat(events1.get(0).kind(), equalTo(ENTRY_MODIFY)); - assertThat(events2.get(0).context(), equalTo(path)); + URI uri = URI.create("http://localhost:8080/hello.txt"); + Map env = Map.of(POLL_INTERVAL_PROPERTY_NAME, "PT0S"); + try (FileSystem fs = FileSystems.newFileSystem(uri, env)) + { + Path path = fs.getPath(uri.toString()); + + try (WatchService watcher = fs.newWatchService()) + { + // WHEN + k3po.start(); + + k3po.notifyBarrier("REGISTERED"); + path.register(watcher); + + WatchKey key1 = watcher.take(); + List> events1 = key1.pollEvents(); + + k3po.notifyBarrier("MODIFIED"); + + WatchKey key2 = watcher.take(); + List> events2 = key2.pollEvents(); + + k3po.finish(); + + // THEN + assertThat(events1.size(), equalTo(1)); + assertThat(events1.get(0).kind(), equalTo(ENTRY_CREATE)); + assertThat(events1.get(0).context(), equalTo(path)); + assertThat(events2.size(), equalTo(1)); + assertThat(events2.get(0).kind(), equalTo(ENTRY_MODIFY)); + assertThat(events2.get(0).context(), equalTo(path)); + } + } } @Test @@ -235,29 +266,39 @@ public void shouldWatch() throws Exception public void shouldWatchRead() throws Exception { // GIVEN - String url = "http://localhost:8080/hello.txt"; - Path path = Path.of(new URI(url)); - HttpWatchService watchService = (HttpWatchService) path.getFileSystem().newWatchService(); - ((HttpPath) path).pollSeconds(1); // TODO: Ati - - // WHEN - k3po.start(); - k3po.notifyBarrier("FIRST_READ"); - String body1 = Files.readString(path); - k3po.notifyBarrier("REGISTERED"); - path.register(watchService); - k3po.notifyBarrier("MODIFIED"); - watchService.take(); - k3po.notifyBarrier("NOT_MODIFIED"); - watchService.take(); - k3po.notifyBarrier("SECOND_READ"); - String body2 = Files.readString(path); - watchService.close(); - ((HttpPath) path).shutdown(); - k3po.finish(); - - // THEN - assertThat(body1, equalTo("Hello World!")); - assertThat(body2, equalTo("Hello Universe!")); + URI uri = URI.create("http://localhost:8080/hello.txt"); + Map env = Map.of(POLL_INTERVAL_PROPERTY_NAME, "PT0S"); + try (FileSystem fs = FileSystems.newFileSystem(uri, env)) + { + Path path = fs.getPath(uri.toString()); + + try (WatchService watcher = fs.newWatchService()) + { + // WHEN + k3po.start(); + + k3po.notifyBarrier("FIRST_READ"); + String body1 = Files.readString(path); + + k3po.notifyBarrier("REGISTERED"); + path.register(watcher); + + k3po.notifyBarrier("MODIFIED"); + watcher.take(); + + k3po.notifyBarrier("NOT_MODIFIED"); + watcher.take(); + + k3po.notifyBarrier("SECOND_READ"); + String body2 = Files.readString(path); + + k3po.finish(); + + + // THEN + assertThat(body1, equalTo("Hello World!")); + assertThat(body2, equalTo("Hello Universe!")); + } + } } } diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/TestHttpFileSystem.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemTest.java similarity index 98% rename from runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/TestHttpFileSystem.java rename to runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemTest.java index 1b0076c097..2682fafc56 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/TestHttpFileSystem.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemTest.java @@ -22,7 +22,7 @@ import org.junit.Test; -public class TestHttpFileSystem +public class HttpFileSystemTest { @Test public void testHttpPath() throws Exception diff --git a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardContext.java b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardContext.java index 551aa66dce..3c002ee908 100644 --- a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardContext.java +++ b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardContext.java @@ -44,7 +44,7 @@ public JwtGuardHandler attach( GuardConfig guard) { JwtOptionsConfig options = (JwtOptionsConfig) guard.options; - JwtGuardHandler handler = new JwtGuardHandler(options, context, supplyAuthorizedId, guard.readPath); + JwtGuardHandler handler = new JwtGuardHandler(options, context, supplyAuthorizedId); handlersById.put(guard.id, handler); return handler; } diff --git a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java index 4223e6454a..be90da20c5 100644 --- a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java +++ b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandler.java @@ -16,7 +16,8 @@ import static org.agrona.LangUtil.rethrowUnchecked; -import java.net.URI; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; @@ -28,7 +29,6 @@ import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.LongSupplier; import jakarta.json.bind.Jsonb; @@ -67,8 +67,7 @@ public class JwtGuardHandler implements GuardHandler public JwtGuardHandler( JwtOptionsConfig options, EngineContext context, - LongSupplier supplyAuthorizedId, - Function readPath) + LongSupplier supplyAuthorizedId) { this.issuer = options.issuer; this.audience = options.audience; @@ -82,7 +81,8 @@ public JwtGuardHandler( Jsonb jsonb = JsonbBuilder.newBuilder() .withConfig(config) .build(); - String keysText = readPath.apply(Path.of(URI.create(options.keysURL.get()))); + Path keysPath = context.resolvePath(options.keysURL.get()); + String keysText = readKeys(keysPath); JwtKeySetConfig jwks = jsonb.fromJson(keysText, JwtKeySetConfig.class); keysConfig = jwks.keys; } @@ -405,4 +405,21 @@ private void unshareIfNecessary() } } } + + private static String readKeys( + Path keysPath) + { + String content = null; + + try + { + content = Files.readString(keysPath); + } + catch (IOException ex) + { + rethrowUnchecked(ex); + } + + return content; + } } diff --git a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/config/JwtKeySetConfigAdapter.java b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/config/JwtKeySetConfigAdapter.java index d554efe2b4..e6b76f3bee 100644 --- a/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/config/JwtKeySetConfigAdapter.java +++ b/runtime/guard-jwt/src/main/java/io/aklivity/zilla/runtime/guard/jwt/internal/config/JwtKeySetConfigAdapter.java @@ -14,8 +14,6 @@ */ package io.aklivity.zilla.runtime.guard.jwt.internal.config; -import static java.util.stream.Collectors.toList; - import java.util.List; import jakarta.json.Json; @@ -36,15 +34,20 @@ public final class JwtKeySetConfigAdapter implements JsonbAdapter keysConfig = keysObject - .getJsonArray(KEYS_NAME) - .stream() + List keysConfig = keysObject.containsKey(KEYS_NAME) + ? keysObject.getJsonArray(KEYS_NAME).stream() .map(JsonValue::asJsonObject) .map(keyAdapter::adaptFromJson) - .collect(toList()); + .toList() + : null; + return new JwtKeySetConfig(keysConfig); } } diff --git a/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandlerTest.java b/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandlerTest.java index 0780ac914d..a19d95da95 100644 --- a/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandlerTest.java +++ b/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardHandlerTest.java @@ -27,12 +27,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.nio.file.Path; import java.security.KeyPair; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.function.Function; import org.agrona.collections.MutableLong; import org.jose4j.jws.JsonWebSignature; @@ -47,8 +45,6 @@ public class JwtGuardHandlerTest { - private static final Function READ_KEYS_URL = url -> "{}"; - private EngineContext context; @Before @@ -70,7 +66,7 @@ public void shouldAuthorize() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -103,7 +99,7 @@ public void shouldChallengeDuringChallengeWindow() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -132,7 +128,7 @@ public void shouldNotChallengeDuringWindowWithoutSubject() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -160,7 +156,7 @@ public void shouldNotChallengeBeforeChallengeWindow() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -189,7 +185,7 @@ public void shouldNotChallengeAgainDuringChallengeWindow() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -218,7 +214,7 @@ public void shouldNotAuthorizeWhenAlgorithmDiffers() throws Exception .audience("testAudience") .key(RFC7515_RS256_CONFIG) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); JwtClaims claims = new JwtClaims(); claims.setClaim("iss", "test issuer"); @@ -240,7 +236,7 @@ public void shouldNotAuthorizeWhenSignatureInvalid() throws Exception .audience("testAudience") .key(RFC7515_RS256_CONFIG) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); JwtClaims claims = new JwtClaims(); claims.setClaim("iss", "test issuer"); @@ -264,7 +260,7 @@ public void shouldNotAuthorizeWhenIssuerDiffers() throws Exception .audience("testAudience") .key(RFC7515_RS256_CONFIG) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); JwtClaims claims = new JwtClaims(); claims.setClaim("iss", "not test issuer"); @@ -286,7 +282,7 @@ public void shouldNotAuthorizeWhenAudienceDiffers() throws Exception .audience("testAudience") .key(RFC7515_RS256_CONFIG) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); JwtClaims claims = new JwtClaims(); claims.setClaim("iss", "test issuer"); @@ -308,7 +304,7 @@ public void shouldNotAuthorizeWhenExpired() throws Exception .audience("testAudience") .key(RFC7515_RS256_CONFIG) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -333,7 +329,7 @@ public void shouldNotAuthorizeWhenNotYetValid() throws Exception .audience("testAudience") .key(RFC7515_RS256_CONFIG) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -360,7 +356,7 @@ public void shouldNotVerifyAuthorizedWhenRolesInsufficient() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); JwtClaims claims = new JwtClaims(); claims.setClaim("iss", "test issuer"); @@ -386,7 +382,7 @@ public void shouldReauthorizeWhenExpirationLater() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -421,7 +417,7 @@ public void shouldReauthorizeWhenScopeBroader() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -457,7 +453,7 @@ public void shouldNotReauthorizeWhenExpirationEarlier() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -492,7 +488,7 @@ public void shouldNotReauthorizeWhenScopeNarrower() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -529,7 +525,7 @@ public void shouldNotReauthorizeWhenSubjectDiffers() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -566,7 +562,7 @@ public void shouldNotReauthorizeWhenContextDiffers() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); @@ -602,7 +598,7 @@ public void shouldDeauthorize() throws Exception .key(RFC7515_RS256_CONFIG) .challenge(challenge) .build(); - JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement, READ_KEYS_URL); + JwtGuardHandler guard = new JwtGuardHandler(options, context, new MutableLong(1L)::getAndIncrement); Instant now = Instant.now(); diff --git a/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardIT.java b/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardIT.java index c89f04281c..77457d8578 100644 --- a/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardIT.java +++ b/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/JwtGuardIT.java @@ -23,11 +23,16 @@ import org.junit.rules.TestRule; import org.junit.rules.Timeout; +import io.aklivity.k3po.runtime.junit.annotation.Specification; +import io.aklivity.k3po.runtime.junit.rules.K3poRule; import io.aklivity.zilla.runtime.engine.test.EngineRule; import io.aklivity.zilla.runtime.engine.test.annotation.Configuration; public class JwtGuardIT { + private final K3poRule k3po = new K3poRule() + .addScriptRoot("keys", "io/aklivity/zilla/specs/guard/jwt/config/keys"); + private final TestRule timeout = new DisableOnDebug(new Timeout(10, SECONDS)); private final EngineRule engine = new EngineRule() @@ -37,7 +42,7 @@ public class JwtGuardIT .clean(); @Rule - public final TestRule chain = outerRule(engine).around(timeout); + public final TestRule chain = outerRule(k3po).around(engine).around(timeout); @Test @Configuration("guard.yaml") @@ -47,13 +52,21 @@ public void shouldInitialize() throws Exception @Test @Configuration("guard-keys-dynamic.yaml") + @Specification({ + "${keys}/issuer" + }) public void shouldInitializeGuardWithDynamicKeys() throws Exception { + k3po.finish(); } @Test @Configuration("guard-keys-implicit.yaml") + @Specification({ + "${keys}/issuer" + }) public void shouldInitializeGuardWithImplicitKeys() throws Exception { + k3po.finish(); } } diff --git a/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/config/JwtKeySetConfigAdapterTest.java b/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/config/JwtKeySetConfigAdapterTest.java index 225813bc1e..1514aff91f 100644 --- a/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/config/JwtKeySetConfigAdapterTest.java +++ b/runtime/guard-jwt/src/test/java/io/aklivity/zilla/runtime/guard/jwt/internal/config/JwtKeySetConfigAdapterTest.java @@ -44,6 +44,28 @@ public void initJson() jsonb = JsonbBuilder.create(config); } + @Test + public void shouldReadJwtKeySetWhenKeysMissing() + { + String text = + "{" + + "}"; + JwtKeySetConfig jwksConfig = jsonb.fromJson(text, JwtKeySetConfig.class); + assertThat(jwksConfig.keys, nullValue()); + } + + @Test + public void shouldWriteJwtKeySetWithKeysMissing() + { + JwtKeySetConfig keySetConfig = new JwtKeySetConfig(null); + String text = jsonb.toJson(keySetConfig); + + assertThat(text, not(nullValue())); + assertThat(text, equalTo( + "{" + + "}")); + } + @Test public void shouldReadJwtKeySet() { @@ -81,7 +103,6 @@ public void shouldReadJwtKeySet() @Test public void shouldWriteJwtKeySet() { - JwtKeySetConfig keySetConfig = new JwtKeySetConfig(List.of(RFC7515_RS256_CONFIG, RFC7515_ES256_CONFIG)); String text = jsonb.toJson(keySetConfig); diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.create.via.http/client.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.create.via.http/client.rpt index 5093a82e2c..db208d88cd 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.create.via.http/client.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.create.via.http/client.rpt @@ -14,7 +14,7 @@ # under the License. # -connect "http://localhost:8080/" +connect "http://localhost:8080/zilla.yaml" connected write http:method "GET" @@ -23,13 +23,13 @@ write close read http:status "404" "Not Found" read closed -connect "http://localhost:8080/" + +connect "http://localhost:8080/zilla.yaml" connected write http:method "GET" write close -write notify FIRST_ABORTED read http:status "200" "OK" read http:header "Etag" "AAAAAAA" read '---\n' @@ -42,6 +42,16 @@ read '---\n' read closed + +connect "http://localhost:8080/zilla.yaml" +connected + +write http:method "GET" +write http:header "If-None-Match" "AAAAAAA" +write http:header "Prefer" "wait=86400" +write close + + connect "zilla://streams/app0" option zilla:window 8192 option zilla:transmission "duplex" diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.create.via.http/server.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.create.via.http/server.rpt index 1e59cdb65c..53f943cb58 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.create.via.http/server.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.create.via.http/server.rpt @@ -14,7 +14,9 @@ # under the License. # -accept "http://localhost:8080/" +accept "http://localhost:8080/zilla.yaml" + + accepted connected @@ -25,13 +27,13 @@ write http:status "404" "Not Found" write http:content-length write close + accepted connected read http:method "GET" read closed -write await FIRST_ABORTED write http:status "200" "OK" write http:content-length write http:header "Etag" "AAAAAAA" @@ -45,6 +47,16 @@ write '---\n' write close + +accepted +connected + +read http:method "GET" +read http:header "If-None-Match" "AAAAAAA" +read http:header "Prefer" "wait=86400" +read closed + + accept "zilla://streams/app0" option zilla:window 8192 option zilla:transmission "duplex" diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.delete.via.http/client.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.delete.via.http/client.rpt index bcf7ee1383..6f658f4624 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.delete.via.http/client.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.delete.via.http/client.rpt @@ -14,7 +14,7 @@ # under the License. # -connect "http://localhost:8080/" +connect "http://localhost:8080/zilla.yaml" connected write http:method "GET" @@ -31,22 +31,15 @@ read '---\n' ' exit: app0\n' read closed -connect "http://localhost:8080/" + +connect "http://localhost:8080/zilla.yaml" connected write http:method "GET" +write http:header "If-None-Match" "AAAAAAA" +write http:header "Prefer" "wait=86400" write close -write notify FIRST_CONNECTED read http:status "404" "Not Found" read closed -connect "zilla://streams/app0" - option zilla:window 8192 - option zilla:transmission "duplex" - -connected - -write notify CONFIG_DELETED -write abort -read abort \ No newline at end of file diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.delete.via.http/server.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.delete.via.http/server.rpt index ca7389f572..803a7cba72 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.delete.via.http/server.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.delete.via.http/server.rpt @@ -14,7 +14,7 @@ # under the License. # -accept "http://localhost:8080/" +accept "http://localhost:8080/zilla.yaml" accepted connected @@ -33,24 +33,15 @@ write '---\n' ' exit: app0\n' write close + accepted connected read http:method "GET" +read http:header "If-None-Match" "AAAAAAA" +read http:header "Prefer" "wait=86400" read closed -write await FIRST_CONNECTED write http:status "404" "Not Found" write http:content-length write close - -accept "zilla://streams/app0" - option zilla:window 8192 - option zilla:transmission "duplex" - -accepted -connected - -read await CONFIG_DELETED -read aborted -write aborted diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.no.etag.via.http/client.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.no.etag.via.http/client.rpt index 791d9c4a1d..e5a646a6e9 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.no.etag.via.http/client.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.no.etag.via.http/client.rpt @@ -48,8 +48,8 @@ read closed connect "zilla://streams/app0" -option zilla:window 8192 -option zilla:transmission "duplex" + option zilla:window 8192 + option zilla:transmission "duplex" connected @@ -57,6 +57,6 @@ write abort read abort connect "zilla://streams/app1" -option zilla:window 8192 -option zilla:transmission "duplex" + option zilla:window 8192 + option zilla:transmission "duplex" connected \ No newline at end of file diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.no.etag.via.http/server.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.no.etag.via.http/server.rpt index a4130af69d..39d09dd14b 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.no.etag.via.http/server.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.no.etag.via.http/server.rpt @@ -57,7 +57,6 @@ accept "zilla://streams/app0" accepted connected -write notify CONNECTED read aborted write aborted diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.via.http/client.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.via.http/client.rpt index 99779dc0ff..6319e6f586 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.via.http/client.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.via.http/client.rpt @@ -14,7 +14,7 @@ # under the License. # -connect "http://localhost:8080/" +connect "http://localhost:8080/zilla.yaml" connected write http:method "GET" @@ -31,7 +31,8 @@ read '---\n' ' exit: app0\n' read closed -connect "http://localhost:8080/" + +connect "http://localhost:8080/zilla.yaml" connected write http:method "GET" @@ -51,16 +52,26 @@ read '---\n' read closed +connect "http://localhost:8080/zilla.yaml" +connected + +write http:method "GET" +write http:header "If-None-Match" "BBBBBBB" +write http:header "Prefer" "wait=86400" +write close + + connect "zilla://streams/app0" -option zilla:window 8192 -option zilla:transmission "duplex" + option zilla:window 8192 + option zilla:transmission "duplex" connected write abort read abort + connect "zilla://streams/app1" -option zilla:window 8192 -option zilla:transmission "duplex" -connected \ No newline at end of file + option zilla:window 8192 + option zilla:transmission "duplex" +connected diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.via.http/server.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.via.http/server.rpt index 1c15cfe74c..6f19b8e998 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.via.http/server.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.modify.via.http/server.rpt @@ -33,6 +33,7 @@ write '---\n' ' exit: app0\n' write close + accepted connected @@ -55,17 +56,26 @@ write '---\n' write close +accepted +connected + +read http:method "GET" +read http:header "If-None-Match" "BBBBBBB" +read http:header "Prefer" "wait=86400" +read closed + + accept "zilla://streams/app0" option zilla:window 8192 option zilla:transmission "duplex" accepted connected -write notify CONNECTED read aborted write aborted + accept "zilla://streams/app1" option zilla:window 8192 option zilla:transmission "duplex" diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.server.error.via.http/client.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.server.error.via.http/client.rpt index 26634d7379..c55613190b 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.server.error.via.http/client.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.server.error.via.http/client.rpt @@ -14,7 +14,7 @@ # under the License. # -connect "http://localhost:8080/" +connect "http://localhost:8080/zilla.yaml" connected write http:method "GET" @@ -31,7 +31,7 @@ read '---\n' ' exit: app0\n' read closed -connect "http://localhost:8080/" +connect "http://localhost:8080/zilla.yaml" connected write http:method "GET" @@ -44,11 +44,11 @@ read closed connect "zilla://streams/app0" -option zilla:window 8192 -option zilla:transmission "duplex" + option zilla:window 8192 + option zilla:transmission "duplex" connected connect "zilla://streams/app0" -option zilla:window 8192 -option zilla:transmission "duplex" -connected \ No newline at end of file + option zilla:window 8192 + option zilla:transmission "duplex" +connected diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.server.error.via.http/server.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.server.error.via.http/server.rpt index 7db63f1708..d51b937052 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.server.error.via.http/server.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/application/reconfigure.server.error.via.http/server.rpt @@ -14,7 +14,7 @@ # under the License. # -accept "http://localhost:8080/" +accept "http://localhost:8080/zilla.yaml" accepted connected @@ -53,6 +53,6 @@ accept "zilla://streams/app0" accepted connected -write notify CONNECTED + accepted -connected \ No newline at end of file +connected diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.create.via.http/client.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.create.via.http/client.rpt index 1aa54a359f..e1c5e4dec0 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.create.via.http/client.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.create.via.http/client.rpt @@ -14,13 +14,6 @@ # under the License. # - -connect "zilla://streams/net0" - option zilla:transmission "duplex" - option zilla:window 8192 -connect aborted -write notify FIRST_ABORTED - connect await CONFIG_CREATED "zilla://streams/net0" option zilla:transmission "duplex" diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.create.via.http/server.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.create.via.http/server.rpt index b11e21ce12..5867b43b1e 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.create.via.http/server.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.create.via.http/server.rpt @@ -14,13 +14,9 @@ # under the License. # - accept "zilla://streams/net0" option zilla:transmission "duplex" option zilla:window 8192 -rejected - -write notify CONFIG_CREATED accepted connected diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.delete.via.http/client.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.delete.via.http/client.rpt index 0cf89384dc..d7c3cbf35f 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.delete.via.http/client.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.delete.via.http/client.rpt @@ -14,15 +14,6 @@ # under the License. # -connect "zilla://streams/net0" - option zilla:transmission "duplex" - option zilla:window 8192 -connected -write notify FIRST_CONNECTED - -write aborted -read abort - connect await CONFIG_DELETED "zilla://streams/net0" option zilla:transmission "duplex" diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.delete.via.http/server.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.delete.via.http/server.rpt index b6386becc9..c288e29dbb 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.delete.via.http/server.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.delete.via.http/server.rpt @@ -14,15 +14,8 @@ # under the License. # - accept "zilla://streams/net0" option zilla:transmission "duplex" option zilla:window 8192 -accepted -connected - -write notify CONFIG_DELETED -read abort -write aborted rejected diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.no.etag.via.http/client.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.no.etag.via.http/client.rpt index 1c86918d66..b72ea7b4f6 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.no.etag.via.http/client.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.no.etag.via.http/client.rpt @@ -19,6 +19,8 @@ connect "zilla://streams/net0" option zilla:window 8192 connected +write notify CONNECTED + write aborted read abort diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.no.etag.via.http/server.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.no.etag.via.http/server.rpt index 0649d05dc8..87be4cb467 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.no.etag.via.http/server.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.no.etag.via.http/server.rpt @@ -17,14 +17,17 @@ accept "zilla://streams/net0" option zilla:transmission "duplex" option zilla:window 8192 + accepted connected write notify CONFIG_CHANGED read abort write aborted + rejected + accept "zilla://streams/net1" option zilla:transmission "duplex" option zilla:window 8192 diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.via.http/client.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.via.http/client.rpt index 1c86918d66..b72ea7b4f6 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.via.http/client.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.via.http/client.rpt @@ -19,6 +19,8 @@ connect "zilla://streams/net0" option zilla:window 8192 connected +write notify CONNECTED + write aborted read abort diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.via.http/server.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.via.http/server.rpt index 0649d05dc8..5804f1545f 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.via.http/server.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.modify.via.http/server.rpt @@ -17,14 +17,18 @@ accept "zilla://streams/net0" option zilla:transmission "duplex" option zilla:window 8192 + accepted connected write notify CONFIG_CHANGED read abort write aborted + + rejected + accept "zilla://streams/net1" option zilla:transmission "duplex" option zilla:window 8192 diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.server.error.via.http/client.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.server.error.via.http/client.rpt index bd64496704..b79f1b71f2 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.server.error.via.http/client.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.server.error.via.http/client.rpt @@ -19,9 +19,13 @@ connect "zilla://streams/net0" option zilla:window 8192 connected +read notify CONNECTED + + connect await SERVER_ERROR "zilla://streams/net0" option zilla:transmission "duplex" option zilla:window 8192 connected + write notify CHECK_RECONFIGURE diff --git a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.server.error.via.http/server.rpt b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.server.error.via.http/server.rpt index 5beb68d59e..0af1c48c59 100644 --- a/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.server.error.via.http/server.rpt +++ b/specs/engine.spec/src/main/scripts/io/aklivity/zilla/specs/engine/streams/network/reconfigure.server.error.via.http/server.rpt @@ -17,9 +17,9 @@ accept "zilla://streams/net0" option zilla:transmission "duplex" option zilla:window 8192 + accepted connected -write notify SERVER_ERROR accepted connected \ No newline at end of file diff --git a/specs/engine.spec/src/test/java/io/aklivity/zilla/specs/engine/streams/ApplicationIT.java b/specs/engine.spec/src/test/java/io/aklivity/zilla/specs/engine/streams/ApplicationIT.java index a165358034..ed9a73beea 100644 --- a/specs/engine.spec/src/test/java/io/aklivity/zilla/specs/engine/streams/ApplicationIT.java +++ b/specs/engine.spec/src/test/java/io/aklivity/zilla/specs/engine/streams/ApplicationIT.java @@ -87,7 +87,7 @@ public void shouldNotReconfigureWhenModifiedButParseFailed() throws Exception @Specification({ "${app}/reconfigure.modify.via.http/client", "${app}/reconfigure.modify.via.http/server" }) - public void shouldReconfigureWhenModifiedHTTP() throws Exception + public void shouldReconfigureWhenModifiedViaHttp() throws Exception { k3po.finish(); } @@ -96,7 +96,7 @@ public void shouldReconfigureWhenModifiedHTTP() throws Exception @Specification({ "${app}/reconfigure.create.via.http/client", "${app}/reconfigure.create.via.http/server" }) - public void shouldReconfigureWhenCreatedHTTP() throws Exception + public void shouldReconfigureWhenCreatedViaHttp() throws Exception { k3po.finish(); } @@ -105,7 +105,7 @@ public void shouldReconfigureWhenCreatedHTTP() throws Exception @Specification({ "${app}/reconfigure.delete.via.http/client", "${app}/reconfigure.delete.via.http/server" }) - public void shouldReconfigureWhenDeletedHTTP() throws Exception + public void shouldReconfigureWhenDeletedViaHttp() throws Exception { k3po.finish(); } @@ -115,7 +115,7 @@ public void shouldReconfigureWhenDeletedHTTP() throws Exception "${app}/reconfigure.modify.no.etag.via.http/server", "${app}/reconfigure.modify.no.etag.via.http/client" }) - public void shouldReconfigureWhenModifiedHTTPEtagNotSupported() throws Exception + public void shouldReconfigureWhenModifiedViaHttpEtagNotSupported() throws Exception { k3po.finish(); } @@ -125,7 +125,7 @@ public void shouldReconfigureWhenModifiedHTTPEtagNotSupported() throws Exception "${app}/reconfigure.server.error.via.http/server", "${app}/reconfigure.server.error.via.http/client" }) - public void shouldNotReconfigureWhen500Returned() throws Exception + public void shouldNotReconfigureViaHttpWhenServerError() throws Exception { k3po.finish(); } diff --git a/specs/engine.spec/src/test/java/io/aklivity/zilla/specs/engine/streams/NetworkIT.java b/specs/engine.spec/src/test/java/io/aklivity/zilla/specs/engine/streams/NetworkIT.java index c9a29907b2..8330551345 100644 --- a/specs/engine.spec/src/test/java/io/aklivity/zilla/specs/engine/streams/NetworkIT.java +++ b/specs/engine.spec/src/test/java/io/aklivity/zilla/specs/engine/streams/NetworkIT.java @@ -87,8 +87,10 @@ public void shouldNotReconfigureWhenModifiedButParseFailed() throws Exception @Specification({ "${net}/reconfigure.modify.via.http/client", "${net}/reconfigure.modify.via.http/server" }) - public void shouldReconfigureWhenModifiedHTTP() throws Exception + public void shouldReconfigureWhenModifiedViaHttp() throws Exception { + k3po.start(); + k3po.notifyBarrier("CONFIG_CHANGED"); k3po.finish(); } @@ -96,8 +98,10 @@ public void shouldReconfigureWhenModifiedHTTP() throws Exception @Specification({ "${net}/reconfigure.create.via.http/client", "${net}/reconfigure.create.via.http/server" }) - public void shouldReconfigureWhenCreatedHTTP() throws Exception + public void shouldReconfigureWhenCreatedViaHttp() throws Exception { + k3po.start(); + k3po.notifyBarrier("CONFIG_CREATED"); k3po.finish(); } @@ -105,8 +109,10 @@ public void shouldReconfigureWhenCreatedHTTP() throws Exception @Specification({ "${net}/reconfigure.delete.via.http/client", "${net}/reconfigure.delete.via.http/server" }) - public void shouldReconfigureWhenDeletedHTTP() throws Exception + public void shouldReconfigureWhenDeletedViaHttp() throws Exception { + k3po.start(); + k3po.notifyBarrier("CONFIG_DELETED"); k3po.finish(); } @@ -115,7 +121,7 @@ public void shouldReconfigureWhenDeletedHTTP() throws Exception "${net}/reconfigure.modify.no.etag.via.http/server", "${net}/reconfigure.modify.no.etag.via.http/client" }) - public void shouldReconfigureWhenModifiedHTTPEtagNotSupported() throws Exception + public void shouldReconfigureWhenModifiedViaHttpEtagNotSupported() throws Exception { k3po.finish(); } @@ -125,8 +131,10 @@ public void shouldReconfigureWhenModifiedHTTPEtagNotSupported() throws Exception "${net}/reconfigure.server.error.via.http/server", "${net}/reconfigure.server.error.via.http/client" }) - public void shouldNotReconfigureWhen500Returned() throws Exception + public void shouldNotReconfigureViaHttpWhenServerError() throws Exception { + k3po.start(); + k3po.notifyBarrier("SERVER_ERROR"); k3po.finish(); } } diff --git a/specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/guard-keys-dynamic.yaml b/specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/guard-keys-dynamic.yaml index f877141e5b..d4663cd0f9 100644 --- a/specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/guard-keys-dynamic.yaml +++ b/specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/guard-keys-dynamic.yaml @@ -13,20 +13,13 @@ # specific language governing permissions and limitations under the License. # -{ - "name": "test", - "guards": - { - "jwt0": - { - "type": "jwt", - "options": - { - "issuer": "https://aklivity.us.auth0.com", - "audience": "https://api.aklivity.com", - "keys": "https://aklivity.us.auth0.com/.well-known/jwks.json", - "challenge": 30 - } - } - } -} +--- +name: test +guards: + jwt0: + type: jwt + options: + issuer: http://localhost:8080 + audience: https://api.aklivity.com + keys: http://localhost:8080/.well-known/jwks.json + challenge: 30 diff --git a/specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/guard-keys-implicit.yaml b/specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/guard-keys-implicit.yaml index 8994c3c306..138928094f 100644 --- a/specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/guard-keys-implicit.yaml +++ b/specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/guard-keys-implicit.yaml @@ -19,6 +19,6 @@ guards: jwt0: type: jwt options: - issuer: https://aklivity.us.auth0.com + issuer: http://localhost:8080 audience: https://api.aklivity.com challenge: 30 diff --git a/specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/keys/issuer.rpt b/specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/keys/issuer.rpt new file mode 100644 index 0000000000..7be87bde1e --- /dev/null +++ b/specs/guard-jwt.spec/src/main/scripts/io/aklivity/zilla/specs/guard/jwt/config/keys/issuer.rpt @@ -0,0 +1,29 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "http://localhost:8080/.well-known/jwks.json" + + +accepted +connected + +read http:method "GET" +read closed + +write http:status "200" "OK" +write http:content-length +write '{}' + +write close From e7c6c56f75dbc22f5b9f1b040ee26aff3d87e112 Mon Sep 17 00:00:00 2001 From: John Fallows Date: Tue, 25 Jun 2024 21:39:10 -0700 Subject: [PATCH 42/43] No watch event needed for identical response body --- .../http/internal/HttpFileSystem.java | 22 ---------------- .../filesystem/http/internal/HttpPath.java | 10 ++++++-- .../filesystem/http/HttpFileSystemIT.java | 8 ------ .../http/application/watch.read/client.rpt | 25 ++++++++++--------- .../http/application/watch.read/server.rpt | 10 ++++---- 5 files changed, 26 insertions(+), 49 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystem.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystem.java index 4f84312cea..8f274fd89d 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystem.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpFileSystem.java @@ -17,7 +17,6 @@ import static java.net.http.HttpClient.Redirect.NORMAL; import static java.net.http.HttpClient.Version.HTTP_2; import static java.util.Objects.requireNonNull; -import static org.agrona.LangUtil.rethrowUnchecked; import java.net.URI; import java.net.http.HttpClient; @@ -26,8 +25,6 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.attribute.UserPrincipalLookupService; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.Map; import java.util.Set; @@ -39,7 +36,6 @@ public final class HttpFileSystem extends FileSystem private final URI root; private final HttpFileSystemConfiguration config; private final HttpClient client; - private final MessageDigest md5; HttpFileSystem( HttpFileSystemProvider provider, @@ -53,7 +49,6 @@ public final class HttpFileSystem extends FileSystem .version(HTTP_2) .followRedirects(NORMAL) .build(); - this.md5 = initMessageDigest("MD5"); } @Override @@ -148,21 +143,4 @@ HttpClient client() { return client; } - - private static MessageDigest initMessageDigest( - String algorithm) - { - MessageDigest md5 = null; - - try - { - md5 = MessageDigest.getInstance(algorithm); - } - catch (NoSuchAlgorithmException ex) - { - rethrowUnchecked(ex); - } - - return md5; - } } diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java index 39b0169df9..ae9a3a6766 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java @@ -32,6 +32,7 @@ import java.nio.file.WatchEvent; import java.nio.file.WatchKey; import java.nio.file.WatchService; +import java.util.Arrays; import java.util.Iterator; import java.util.Objects; @@ -292,10 +293,15 @@ void success( switch (response.statusCode()) { case HTTP_OK: + byte[] oldBody = body; body = response.body(); - // TODO: Ati - calculate and store hash if there is no etag etag = response.headers().firstValue("Etag").orElse(null); - changeCount++; + if (body == null || + oldBody == null || + !Arrays.equals(body, oldBody)) + { + changeCount++; + } break; case HTTP_NOT_FOUND: body = HttpPath.EMPTY_BODY; diff --git a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java index 4201306075..6fef361311 100644 --- a/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java +++ b/runtime/filesystem-http/src/test/java/io/aklivity/zilla/runtime/filesystem/http/HttpFileSystemIT.java @@ -277,24 +277,16 @@ public void shouldWatchRead() throws Exception // WHEN k3po.start(); - k3po.notifyBarrier("FIRST_READ"); String body1 = Files.readString(path); - k3po.notifyBarrier("REGISTERED"); path.register(watcher); - k3po.notifyBarrier("MODIFIED"); watcher.take(); - k3po.notifyBarrier("NOT_MODIFIED"); - watcher.take(); - - k3po.notifyBarrier("SECOND_READ"); String body2 = Files.readString(path); k3po.finish(); - // THEN assertThat(body1, equalTo("Hello World!")); assertThat(body2, equalTo("Hello Universe!")); diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt index e74045a90c..44ef7dc8fe 100644 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/client.rpt @@ -19,13 +19,15 @@ connected write http:method "GET" write close -read notify FIRST_READ read http:status "200" "OK" read http:header "Etag" "AAAAAAA" read "Hello World!" read closed +read notify CONFIG_INITIALIZED -connect "http://localhost:8080/hello.txt" + +connect await CONFIG_INITIALIZED + "http://localhost:8080/hello.txt" connected write http:method "GET" @@ -33,13 +35,14 @@ write http:header "If-None-Match" "AAAAAAA" write http:header "Prefer" "wait=86400" write close -read notify REGISTERED read http:status "200" "OK" read http:header "Etag" "AAAAAAA" read "Hello World!" read closed +read notify CONFIG_IDENTICAL -connect "http://localhost:8080/hello.txt" +connect await CONFIG_IDENTICAL + "http://localhost:8080/hello.txt" connected write http:method "GET" @@ -47,13 +50,14 @@ write http:header "If-None-Match" "AAAAAAA" write http:header "Prefer" "wait=86400" write close -read notify MODIFIED read http:status "200" "OK" read http:header "Etag" "BBBBBBB" read "Hello Universe!" read closed +read notify CONFIG_MODIFIED -connect "http://localhost:8080/hello.txt" +connect await CONFIG_MODIFIED + "http://localhost:8080/hello.txt" connected write http:method "GET" @@ -61,17 +65,14 @@ write http:header "If-None-Match" "BBBBBBB" write http:header "Prefer" "wait=86400" write close -read notify NOT_MODIFIED read http:status "304" "Not changed" read closed +read notify CONFIG_NOT_MODIFIED -connect "http://localhost:8080/hello.txt" +connect await CONFIG_NOT_MODIFIED + "http://localhost:8080/hello.txt" connected write http:method "GET" write http:header "If-None-Match" "BBBBBBB" write close - -read notify SECOND_READ -read http:status "304" "Not changed" -read closed diff --git a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/server.rpt b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/server.rpt index 4ca89ac7f3..c1d51dd9b1 100644 --- a/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/server.rpt +++ b/specs/filesystem-http.spec/src/main/scripts/io/aklivity/zilla/specs/filesystem/http/application/watch.read/server.rpt @@ -14,19 +14,20 @@ # accept "http://localhost:8080/hello.txt" + accepted connected read http:method "GET" read closed -write await FIRST_READ write http:status "200" "OK" write http:content-length write http:header "Etag" "AAAAAAA" write "Hello World!" write close + accepted connected @@ -35,13 +36,13 @@ read http:header "If-None-Match" "AAAAAAA" read http:header "Prefer" "wait=86400" read closed -write await REGISTERED write http:status "200" "OK" write http:content-length write http:header "Etag" "AAAAAAA" write "Hello World!" write close + accepted connected @@ -50,13 +51,13 @@ read http:header "If-None-Match" "AAAAAAA" read http:header "Prefer" "wait=86400" read closed -write await MODIFIED write http:status "200" "OK" write http:content-length write http:header "Etag" "BBBBBBB" write "Hello Universe!" write close + accepted connected @@ -65,11 +66,11 @@ read http:header "If-None-Match" "BBBBBBB" read http:header "Prefer" "wait=86400" read closed -write await NOT_MODIFIED write http:status "304" "Not changed" write http:content-length write close + accepted connected @@ -77,7 +78,6 @@ read http:method "GET" read http:header "If-None-Match" "BBBBBBB" read closed -write await SECOND_READ write http:status "304" "Not changed" write http:content-length write close From 5423718b0661f5409e2dca22fe5bf226fd96cb30 Mon Sep 17 00:00:00 2001 From: John Fallows Date: Tue, 25 Jun 2024 22:34:55 -0700 Subject: [PATCH 43/43] Handle status 204 with null body and infer delay for optional prefer wait --- .../runtime/filesystem/http/internal/HttpPath.java | 10 +++++++--- .../filesystem/http/internal/HttpWatchService.java | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java index ae9a3a6766..bb6d54c838 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpPath.java @@ -16,6 +16,7 @@ import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_NOT_MODIFIED; +import static java.net.HttpURLConnection.HTTP_NO_CONTENT; import static java.net.HttpURLConnection.HTTP_OK; import static java.util.Objects.requireNonNull; @@ -290,11 +291,14 @@ byte[] readBody() void success( HttpResponse response) { - switch (response.statusCode()) + final int status = response.statusCode(); + + switch (status) { case HTTP_OK: + case HTTP_NO_CONTENT: byte[] oldBody = body; - body = response.body(); + body = status == HTTP_NO_CONTENT ? EMPTY_BODY : response.body(); etag = response.headers().firstValue("Etag").orElse(null); if (body == null || oldBody == null || @@ -304,7 +308,7 @@ void success( } break; case HTTP_NOT_FOUND: - body = HttpPath.EMPTY_BODY; + body = EMPTY_BODY; etag = null; changeCount++; break; diff --git a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpWatchService.java b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpWatchService.java index 52d0b5e870..5c52e1cbde 100644 --- a/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpWatchService.java +++ b/runtime/filesystem-http/src/main/java/io/aklivity/zilla/runtime/filesystem/http/internal/HttpWatchService.java @@ -14,9 +14,12 @@ */ package io.aklivity.zilla.runtime.filesystem.http.internal; +import static java.lang.System.currentTimeMillis; import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.agrona.LangUtil.rethrowUnchecked; import java.net.http.HttpClient; @@ -159,8 +162,10 @@ private void checkModifiers( private void watchBody( HttpWatchKey watchKey) { - final long pollSeconds = pollInterval.getSeconds(); - executor.schedule(watchKey::watchBody, pollSeconds, TimeUnit.SECONDS); + long elapsed = currentTimeMillis() - watchKey.lastWatchAt; + long delay = Math.max(SECONDS.toMillis(pollInterval.getSeconds()) - elapsed, 0L); + + executor.schedule(watchKey::watchBody, delay, MILLISECONDS); } private void signalKey( @@ -184,6 +189,7 @@ private static final class HttpWatchKey implements WatchKey, Comparable future; + private long lastWatchAt; private HttpWatchKey() { @@ -278,6 +284,8 @@ private void watchBody() HttpClient client = path.getFileSystem().client(); HttpRequest request = path.newWatchRequest(); + this.lastWatchAt = currentTimeMillis(); + this.future = client.sendAsync(request, BodyHandlers.ofByteArray()) .thenAccept(this::success) .exceptionally(this::failure); @@ -305,6 +313,8 @@ else if (exists) { signalEvent(ENTRY_CREATE); } + + this.lastWatchAt = 0L; } watcher.watchBody(this);