From 68e31a01329dff97a2c5da8658a9a8b4de9eb331 Mon Sep 17 00:00:00 2001 From: Erik Rasmussen Date: Sat, 11 May 2024 15:42:00 -0500 Subject: [PATCH] Persist the broker between cli runs (#68) * Removed Broker.fs and its references The Broker.fs file has been deleted, along with all its code. References to this file in the UnMango.Tdl.FSharp.fsproj have also been removed. This change simplifies the project structure by eliminating unnecessary components. * Refactor client creation and rename modules - Replaced the method for creating a client channel in Handlers.cs - Renamed Client.fs to GrpcClient.fs, updating all references accordingly - Renamed Library.fs to Util.fs, updating all references accordingly - Updated module name from UnMango.Tdl.Client to UnMango.Tdl.GrpcClient * Updated F# code formatting The F# code has been reformatted for better readability. The changes include: - Adjusted the indentation style in .editorconfig for F# files. - Simplified async expressions by removing unnecessary newlines and brackets. - Reformatted function definitions to make them more concise. - Cleaned up some test cases by removing extra lines and spaces. * Refactor Docker image creation condition The conditional compilation directive for creating Docker images has been replaced with a runtime check. Now, the image creation process only occurs if the environment is set to 'Release'. This change enhances code readability and maintainability. * Refactored container creation and added label-based search Significant changes include: - Refactoring of the Container class to use static factory methods for instantiation. - Addition of a method in the Docker class to find containers based on labels. - Modification of the Docker.Start method to include label parameters when creating a container. - Changes in EnsureBroker class to check for socket existence before starting, and addition of labels during container start. --- .editorconfig | 1 + src/Cli.Tests/Internal/DockerRunnerTests.cs | 2 +- src/Cli/Handlers.cs | 2 +- src/Cli/Internal/Container.cs | 8 +- src/Cli/Internal/Docker.cs | 50 +++++++--- src/Cli/Internal/EnsureBroker.cs | 33 ++++--- src/FSharp/Broker.fs | 103 -------------------- src/FSharp/{Client.fs => GrpcClient.fs} | 2 +- src/FSharp/Tdl.fs | 18 ++-- src/FSharp/Tools.fs | 27 +++-- src/FSharp/UnMango.Tdl.FSharp.fsproj | 5 +- src/FSharp/{Library.fs => Util.fs} | 0 src/Language.Tests/Program.fs | 8 +- src/Language.Tests/Tests.fs | 15 ++- 14 files changed, 99 insertions(+), 175 deletions(-) delete mode 100644 src/FSharp/Broker.fs rename src/FSharp/{Client.fs => GrpcClient.fs} (97%) rename src/FSharp/{Library.fs => Util.fs} (100%) diff --git a/.editorconfig b/.editorconfig index 3ddf1b1f..83aeec43 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,6 +11,7 @@ indent_style = space [*.fs] indent_style = space +fsharp_newline_before_multiline_computation_expression = false [*.cs] indent_size = 4 diff --git a/src/Cli.Tests/Internal/DockerRunnerTests.cs b/src/Cli.Tests/Internal/DockerRunnerTests.cs index fe960497..ba377975 100644 --- a/src/Cli.Tests/Internal/DockerRunnerTests.cs +++ b/src/Cli.Tests/Internal/DockerRunnerTests.cs @@ -12,7 +12,7 @@ public async Task TestDocker() { var spec = new Spec(); var createResponse = new CreateContainerResponse { ID = "test-id" }; var client = Substitute.For(); - var startResult = new Container(client, createResponse); + var startResult = Container.From(client, createResponse); client.Start(Arg.Any(), CancellationToken.None) .Returns(startResult); diff --git a/src/Cli/Handlers.cs b/src/Cli/Handlers.cs index a086b5cb..7b8746d4 100644 --- a/src/Cli/Handlers.cs +++ b/src/Cli/Handlers.cs @@ -22,7 +22,7 @@ public static async Task Gen( IEnumerable files, CancellationToken cancellationToken) { Log.Verbose("Creating client channel"); - using var channel = Client.createChannel(Config.Socket); + using var channel = GrpcClient.createChannel(Config.Socket); var client = new UmlService.UmlServiceClient(channel); var retryCount = 0; diff --git a/src/Cli/Internal/Container.cs b/src/Cli/Internal/Container.cs index cbc6acd9..a20b4008 100644 --- a/src/Cli/Internal/Container.cs +++ b/src/Cli/Internal/Container.cs @@ -8,12 +8,16 @@ public interface IContainer : IAsyncDisposable string Id { get; } } -internal sealed class Container(IDocker docker, CreateContainerResponse create) : IContainer +internal sealed class Container(IDocker docker, string id) : IContainer { - public string Id => create.ID; + public string Id => id; public async ValueTask DisposeAsync() { Log.Verbose("Disposing container"); await docker.Stop(this); } + + public static Container From(IDocker docker, CreateContainerResponse create) => new(docker, create.ID); + + public static Container From(IDocker docker, ContainerListResponse list) => new(docker, list.ID); } diff --git a/src/Cli/Internal/Docker.cs b/src/Cli/Internal/Docker.cs index cfd0190c..ef975e80 100644 --- a/src/Cli/Internal/Docker.cs +++ b/src/Cli/Internal/Docker.cs @@ -12,10 +12,13 @@ public sealed record StartArgs public string? Name { get; init; } public IList Volumes { get; init; } = []; public string? User { get; init; } + public IDictionary Labels { get; init; } = new Dictionary(); } public interface IDocker { + Task FindMatching(IDictionary labels, CancellationToken cancellationToken); + Task FollowLogs(string id, CancellationToken cancellationToken = default); Task Start(StartArgs args, CancellationToken cancellationToken = default); @@ -49,6 +52,24 @@ internal sealed class Docker(IDockerClient docker, IDockerProgress progress) : I private static readonly Random Random = new(); private static string RandomName => $"tdl-{Random.Next()}"; + public async Task FindMatching( + IDictionary labels, + CancellationToken cancellationToken) { + Log.Debug("Listing containers"); + var containers = await docker.Containers.ListContainersAsync( + new ContainersListParameters { + Filters = labels.ToDictionary( + x => x.Key, + x => new Dictionary { + [x.Value] = true, + } as IDictionary), + }, + cancellationToken); + + var firstMatch = containers.FirstOrDefault(); + return firstMatch == null ? null : Container.From(this, firstMatch); + } + public Task FollowLogs(string id, CancellationToken cancellationToken) { Log.Debug("Getting container logs"); return docker.Containers.GetContainerLogsAsync( @@ -63,26 +84,27 @@ public Task FollowLogs(string id, CancellationToken cancellationToken) { } public async Task Start(StartArgs args, CancellationToken cancellationToken) { -#if !DEBUG - Log.Debug("Creating image"); - await docker.Images.CreateImageAsync( - new ImagesCreateParameters { - FromImage = args.Image, - Tag = args.Tag, - }, - new AuthConfig(), - progress, - cancellationToken); -#endif + if (Config.Env.IsRelease) { + Log.Debug("Creating image"); + await docker.Images.CreateImageAsync( + new ImagesCreateParameters { + FromImage = args.Image, + Tag = args.Tag, + }, + new AuthConfig(), + progress, + cancellationToken); + } Log.Debug("Creating container"); - var container = await docker.Containers.CreateContainerAsync( + var createResponse = await docker.Containers.CreateContainerAsync( new CreateContainerParameters { Image = $"{args.Image}:{args.Tag}", Name = args.Name ?? RandomName, Cmd = args.Cmd, User = args.User, Tty = true, + Labels = args.Labels, HostConfig = new HostConfig { Binds = args.Volumes, }, @@ -91,7 +113,7 @@ await docker.Images.CreateImageAsync( Log.Debug("Starting container"); var started = await docker.Containers.StartContainerAsync( - container.ID, + createResponse.ID, new ContainerStartParameters(), cancellationToken); @@ -100,7 +122,7 @@ await docker.Images.CreateImageAsync( } Log.Verbose("Started container"); - return new Container(this, container); + return Container.From(this, createResponse); } public async Task Stop(string id, CancellationToken cancellationToken) { diff --git a/src/Cli/Internal/EnsureBroker.cs b/src/Cli/Internal/EnsureBroker.cs index 1e8da3ec..5f94d75b 100644 --- a/src/Cli/Internal/EnsureBroker.cs +++ b/src/Cli/Internal/EnsureBroker.cs @@ -14,11 +14,25 @@ internal static partial class Patterns internal static class EnsureBroker { + private const string OwnerLabel = "tdl.owner", Owner = "tdl-cli"; private static readonly Regex ApplicationStarted = Patterns.ApplicationStarted(); public static InvocationMiddleware Middleware => async (context, next) => { - var cancellationToken = context.GetCancellationToken(); var docker = context.BindingContext.GetRequiredService(); + await EnsureStarted(docker, context.GetCancellationToken()); + await next(context); + }; + + private static Task EnsureStarted(IDocker docker, CancellationToken cancellationToken) { + Log.Verbose("Checking for socket existence"); + if (!File.Exists(Config.Socket)) + return Start(docker, cancellationToken); + + Log.Debug("Socket exists"); + return Task.CompletedTask; + } + + private static async Task Start(IDocker docker, CancellationToken cancellationToken) { var uid = await Config.Uid(); var gid = await Config.Gid(); @@ -26,22 +40,13 @@ internal static class EnsureBroker var container = await docker.Start(new StartArgs { Image = $"{Config.ContainerRepo}/tdl-broker", Tag = Config.ContainerTag, - Name = "tdl-test", User = $"{uid}:{gid}", Volumes = [$"{Config.SocketDir}:/var/run/tdl"], + Labels = { [OwnerLabel] = Owner }, }, cancellationToken); Log.Verbose("Started broker"); - try { - _ = docker.FollowLogs(container); - await docker.WaitFor(container, ApplicationStarted.IsMatch, cancellationToken); - Log.Verbose("Invoking next"); - await next(context); - Log.Verbose("After invoking next"); - } - finally { - Log.Debug("Stopping broker"); - await docker.Stop(container, cancellationToken); - } - }; + _ = docker.FollowLogs(container, cancellationToken); + await docker.WaitFor(container, ApplicationStarted.IsMatch, cancellationToken); + } } diff --git a/src/FSharp/Broker.fs b/src/FSharp/Broker.fs deleted file mode 100644 index 11561f78..00000000 --- a/src/FSharp/Broker.fs +++ /dev/null @@ -1,103 +0,0 @@ -namespace UnMango.Tdl - -open System -open System.IO -open System.Runtime.InteropServices -open System.Text.RegularExpressions -open System.Threading -open System.Threading.Tasks -open CliWrap.EventStream -open FSharp.Control.Reactive -open UnMango.CliWrap.FSharp - -type DevelopmentMeta = { Configuration: string; Tfm: string } -type BrokerLocation = Development of DevelopmentMeta - -type Broker = - { Endpoint: string - Location: BrokerLocation - Bin: string } - -module Broker = - let private dir = Path.Join("..", "Broker") - let private startedPattern = Regex("Content root path: .*") - let private stoppedPattern = Regex("Application is shutting down") - - let private matchPattern (pattern: Regex) (e: CommandEvent) = - match e with - | :? StandardOutputCommandEvent as o -> pattern.IsMatch(o.Text) - | _ -> false - - let private started e = matchPattern startedPattern e - let private stopped e = matchPattern stoppedPattern e - - type Stop(obs, cts: CancellationTokenSource) = - interface IDisposable with - member this.Dispose() = - do cts.Cancel() - do obs |> Observable.firstIf stopped |> Observable.wait |> ignore - do cts.Dispose() - - interface IAsyncDisposable with - member this.DisposeAsync() = - task { - do! cts.CancelAsync() - // TODO: Is there any async version like Rx.NET? - do obs |> Observable.firstIf stopped |> Observable.wait |> ignore - do cts.Dispose() - } - |> ValueTask - - let debugMeta = - { Configuration = "Debug" - Tfm = "net9.0" } - - let dev = - { Endpoint = "http://127.0.0.1:6969" - Location = Development debugMeta - Bin = "UnMango.Tdl.Broker.dll" } - - let binDir broker = - match broker.Location with - | Development meta -> Path.Join(dir, "bin", meta.Configuration, meta.Tfm) - - let buildCmd = - command "dotnet" { - workingDirectory dir - args [ "build" ] - stdout (PipeTo.f Console.WriteLine) - } - - let startCmd broker = - match broker.Location with - | Development _ -> - command "dotnet" { - workingDirectory (binDir broker) - env [ "ASPNETCORE_URLS", broker.Endpoint ] - args [ broker.Bin ] - stdout (PipeTo.f Console.WriteLine) - } - - let start broker : Async = - async { - match broker.Location with - | Development _ -> - do! buildCmd |> Cli.exec |> Async.Ignore - let! ct = Async.CancellationToken - use forceful = CancellationTokenSource.CreateLinkedTokenSource(ct) - let graceful = new CancellationTokenSource() - - forceful.CancelAfter(TimeSpan.FromSeconds(30L)) - let obs = startCmd broker |> _.Observe(forceful.Token, graceful.Token) - do obs |> Observable.firstIf started |> Observable.wait |> ignore - - return new Stop(obs, graceful) - } - -type Broker with - static member Dev = Broker.dev - - member this.Start(endpoint, [] cancellationToken) = - { this with Endpoint = endpoint } - |> Broker.start - |> fun t -> Async.StartAsTask(t, cancellationToken = cancellationToken) diff --git a/src/FSharp/Client.fs b/src/FSharp/GrpcClient.fs similarity index 97% rename from src/FSharp/Client.fs rename to src/FSharp/GrpcClient.fs index 1c012a87..6172651f 100644 --- a/src/FSharp/Client.fs +++ b/src/FSharp/GrpcClient.fs @@ -1,4 +1,4 @@ -module UnMango.Tdl.Client +module UnMango.Tdl.GrpcClient open System open System.IO diff --git a/src/FSharp/Tdl.fs b/src/FSharp/Tdl.fs index 55018781..49588182 100644 --- a/src/FSharp/Tdl.fs +++ b/src/FSharp/Tdl.fs @@ -10,18 +10,16 @@ module From = let convert i o = async { failwith "TODO" } let wrap (c: IConverter) : From = - fun o -> - async { - let! ct = Async.CancellationToken - return! c.FromAsync(o, ct) |> Async.AwaitTask - } + fun o -> async { + let! ct = Async.CancellationToken + return! c.FromAsync(o, ct) |> Async.AwaitTask + } module Gen = let generate i o = async { failwith "TODO" } let wrap (g: IGenerator) : Gen = - fun i o -> - async { - let! ct = Async.CancellationToken - return! g.GenerateAsync(i, o, ct) |> Async.AwaitTask - } + fun i o -> async { + let! ct = Async.CancellationToken + return! g.GenerateAsync(i, o, ct) |> Async.AwaitTask + } diff --git a/src/FSharp/Tools.fs b/src/FSharp/Tools.fs index 54f58747..d3feac1d 100644 --- a/src/FSharp/Tools.fs +++ b/src/FSharp/Tools.fs @@ -3,21 +3,20 @@ module UnMango.Tdl.Tools open System.Text open UnMango.CliWrap.FSharp -let id (opts: string list) = - async { - let sb = StringBuilder() +let id (opts: string list) = async { + let sb = StringBuilder() - do! - command "id" { - args opts - stdout (PipeTo.string sb) - async - } - |> Async.Ignore + do! + command "id" { + args opts + stdout (PipeTo.string sb) + async + } + |> Async.Ignore - return sb.ToString().Trim() - } + return sb.ToString().Trim() +} -let uid = id ["-u"] +let uid = id [ "-u" ] -let gid = id ["-g"] +let gid = id [ "-g" ] diff --git a/src/FSharp/UnMango.Tdl.FSharp.fsproj b/src/FSharp/UnMango.Tdl.FSharp.fsproj index 2489342f..a0fffe18 100644 --- a/src/FSharp/UnMango.Tdl.FSharp.fsproj +++ b/src/FSharp/UnMango.Tdl.FSharp.fsproj @@ -7,11 +7,10 @@ - + - - + diff --git a/src/FSharp/Library.fs b/src/FSharp/Util.fs similarity index 100% rename from src/FSharp/Library.fs rename to src/FSharp/Util.fs diff --git a/src/Language.Tests/Program.fs b/src/Language.Tests/Program.fs index 96b133fa..31dc4f73 100644 --- a/src/Language.Tests/Program.fs +++ b/src/Language.Tests/Program.fs @@ -1,4 +1,4 @@ -module Program - -[] -let main _ = 0 +module Program + +[] +let main _ = 0 diff --git a/src/Language.Tests/Tests.fs b/src/Language.Tests/Tests.fs index a08be475..b4f96809 100644 --- a/src/Language.Tests/Tests.fs +++ b/src/Language.Tests/Tests.fs @@ -1,8 +1,7 @@ -module Tests - -open System -open Xunit - -[] -let ``My test`` () = - Assert.True(true) +module Tests + +open System +open Xunit + +[] +let ``My test`` () = Assert.True(true)