From 90e465003e21f2a4f1e03981770e12ff1dfa5d86 Mon Sep 17 00:00:00 2001 From: Milosz Krajewski Date: Sat, 6 May 2023 17:16:24 +0100 Subject: [PATCH 1/5] sqs mostly working --- Directory.Packages.props | 5 +- src/K4os.Shared/Extensions.cs | 31 +- src/K4os.Xpovoc.Core/Db/ISchedulerConfig.cs | 3 +- .../Queue/IJobQueueAdapter.cs | 12 + .../Queue/QueueJobScheduler.cs | 186 +++++----- .../K4os.Xpovoc.Sqs.Test.csproj | 29 ++ .../SqsFactoryAndQueueTests.cs | 148 ++++++++ .../SqsQueueAdapterTests.cs | 282 +++++++++++++++ src/K4os.Xpovoc.Sqs.Test/TestConfig.cs | 10 + src/K4os.Xpovoc.Sqs.Test/TestJob.cs | 19 + src/K4os.Xpovoc.Sqs.Test/TestLogger.cs | 31 ++ src/K4os.Xpovoc.Sqs.Test/TestLoggerFactory.cs | 17 + src/K4os.Xpovoc.Sqs.Test/Usings.cs | 1 + .../ISqsJobQueueAdapterConfig.cs | 17 + src/K4os.Xpovoc.Sqs/Internal/ISqsQueue.cs | 26 ++ .../Internal/ISqsQueueFactory.cs | 8 + .../Internal/ISqsQueueSettings.cs | 11 + src/K4os.Xpovoc.Sqs/Internal/SqsAttributes.cs | 15 + src/K4os.Xpovoc.Sqs/Internal/SqsConstants.cs | 13 + src/K4os.Xpovoc.Sqs/Internal/SqsQueue.cs | 99 ++++++ .../Internal/SqsQueueFactory.cs | 127 +++++++ .../Internal/SqsQueueSettings.cs | 11 + src/K4os.Xpovoc.Sqs/Internal/SqsResult.cs | 13 + src/K4os.Xpovoc.Sqs/K4os.Xpovoc.Sqs.csproj | 8 + src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs | 328 ++++++++++++++++++ src/K4os.Xpovoc.Sqs/SqsReceivedJob.cs | 39 +++ src/K4os.Xpovoc.sln | 7 + src/K4os.Xpovoc.sln.DotSettings | 1 + src/Playground/Program.cs | 17 +- 29 files changed, 1414 insertions(+), 100 deletions(-) create mode 100644 src/K4os.Xpovoc.Core/Queue/IJobQueueAdapter.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/K4os.Xpovoc.Sqs.Test.csproj create mode 100644 src/K4os.Xpovoc.Sqs.Test/SqsFactoryAndQueueTests.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/TestConfig.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/TestJob.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/TestLogger.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/TestLoggerFactory.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/Usings.cs create mode 100644 src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterConfig.cs create mode 100644 src/K4os.Xpovoc.Sqs/Internal/ISqsQueue.cs create mode 100644 src/K4os.Xpovoc.Sqs/Internal/ISqsQueueFactory.cs create mode 100644 src/K4os.Xpovoc.Sqs/Internal/ISqsQueueSettings.cs create mode 100644 src/K4os.Xpovoc.Sqs/Internal/SqsAttributes.cs create mode 100644 src/K4os.Xpovoc.Sqs/Internal/SqsConstants.cs create mode 100644 src/K4os.Xpovoc.Sqs/Internal/SqsQueue.cs create mode 100644 src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs create mode 100644 src/K4os.Xpovoc.Sqs/Internal/SqsQueueSettings.cs create mode 100644 src/K4os.Xpovoc.Sqs/Internal/SqsResult.cs create mode 100644 src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs create mode 100644 src/K4os.Xpovoc.Sqs/SqsReceivedJob.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index adf7a0c..f0b7ac2 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,14 +4,15 @@ --> + + + - - diff --git a/src/K4os.Shared/Extensions.cs b/src/K4os.Shared/Extensions.cs index f92c5f9..b761a20 100644 --- a/src/K4os.Shared/Extensions.cs +++ b/src/K4os.Shared/Extensions.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; // ReSharper disable UnusedMember.Global // ReSharper disable UnusedType.Global @@ -13,7 +14,8 @@ namespace System; internal static class Extensions { - public static T Required(this T? subject, string? name = null) where T: class => + public static T Required( + this T? subject, string? name = null) where T: class => subject ?? throw new ArgumentNullException(name ?? ""); public static void TryDispose(this object subject) @@ -68,7 +70,32 @@ public static T[] NotNull(this T[]? subject) => subject ?? Array.Empty(); public static T[] EnsureArray(this IEnumerable? subject) => - subject switch { null => Array.Empty(), T[] a => a, var e => e.ToArray(), }; + subject switch { null => Array.Empty(), T[] a => a, _ => subject.ToArray() }; + + public static void AddIfNotNull( + this IDictionary dictionary, K key, V? value) where V: class + { + if (value is not null) + dictionary.Add(key, value); + } + + public static V? TryGetOrDefault( + this IDictionary dictionary, K key, V? fallback = default) => + dictionary.TryGetValue(key, out var value) ? value : fallback; + + public static IEnumerable WhereNotNull( + this IEnumerable sequence) => + sequence.Where(x => x is not null)!; + + public static IEnumerable SelectNotNull( + this IEnumerable sequence, Func map) => + sequence.Select(map).Where(x => x is not null)!; + + public static void Await(this Task task) => task.GetAwaiter().GetResult(); + public static T Await(this Task task) => task.GetAwaiter().GetResult(); + public static void Forget(this Task task) => + task.ContinueWith(_ => _.Exception, TaskContinuationOptions.OnlyOnFaulted); + } internal class SharedNotNull where T: class, new() diff --git a/src/K4os.Xpovoc.Core/Db/ISchedulerConfig.cs b/src/K4os.Xpovoc.Core/Db/ISchedulerConfig.cs index e5d8a98..fbcf1e1 100644 --- a/src/K4os.Xpovoc.Core/Db/ISchedulerConfig.cs +++ b/src/K4os.Xpovoc.Core/Db/ISchedulerConfig.cs @@ -15,4 +15,5 @@ public interface ISchedulerConfig TimeSpan MaximumRetryInterval { get; } TimeSpan KeepFinishedJobsPeriod { get; } TimeSpan PruneInterval { get; } -} \ No newline at end of file +} + diff --git a/src/K4os.Xpovoc.Core/Queue/IJobQueueAdapter.cs b/src/K4os.Xpovoc.Core/Queue/IJobQueueAdapter.cs new file mode 100644 index 0000000..13a8529 --- /dev/null +++ b/src/K4os.Xpovoc.Core/Queue/IJobQueueAdapter.cs @@ -0,0 +1,12 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using K4os.Xpovoc.Abstractions; + +namespace K4os.Xpovoc.Core.Queue; + +public interface IJobQueueAdapter: IDisposable +{ + Task Publish(TimeSpan delay, IJob job, CancellationToken token); + IDisposable Subscribe(Func handler); +} diff --git a/src/K4os.Xpovoc.Core/Queue/QueueJobScheduler.cs b/src/K4os.Xpovoc.Core/Queue/QueueJobScheduler.cs index 5be1ce9..1538e02 100644 --- a/src/K4os.Xpovoc.Core/Queue/QueueJobScheduler.cs +++ b/src/K4os.Xpovoc.Core/Queue/QueueJobScheduler.cs @@ -1,86 +1,100 @@ -// using System; -// using System.Reactive.Concurrency; -// using System.Threading; -// using System.Threading.Tasks; -// using K4os.Xpovoc.Abstractions; -// using Microsoft.Extensions.Logging; -// using Microsoft.Extensions.Logging.Abstractions; -// -// namespace K4os.Xpovoc.Core.Queue -// { -// public class QueueJobScheduler: IJobScheduler, IDisposable -// { -// private static readonly TimeSpan AtLeastOneSecond = TimeSpan.FromSeconds(1); -// private static readonly TimeSpan PublishTimeout = TimeSpan.FromSeconds(5); -// -// public ILogger Log { get; } -// -// private readonly IDisposable _subscription; -// private readonly IJobHandler _handler; -// private readonly IScheduler _scheduler; -// private IJobQueue _jobQueue; -// -// public DateTimeOffset Now => _scheduler.Now; -// -// public QueueJobScheduler( -// ILoggerFactory logFactory, -// IJobHandler handler, -// IJobQueue jobQueue, -// IScheduler scheduler = null) -// { -// Log = (logFactory ?? NullLoggerFactory.Instance).CreateLogger(GetType()); -// -// _scheduler = scheduler ?? Scheduler.Default; -// _handler = handler.Required(nameof(handler)); -// _jobQueue = jobQueue.Required(nameof(jobQueue)); -// _subscription = jobQueue.Subscribe(Handle); -// } -// -// private Task Handle(IJob job) { throw new NotImplementedException(); } -// -// public Task Schedule(DateTimeOffset time, object payload) -// { -// try -// { -// _jobQueue.Publish(payload, time, 1) -// .Publish(new Envelope { Body = body, Time = time }) -// .Wait(PublishTimeout); -// } -// catch (Exception e) -// { -// throw e.Unwrap().Rethrow(); -// } -// -// } -// -// -// private async Task Handle(CancellationToken token, Envelope envelope) -// { -// var now = Now; -// -// if (now >= envelope.Time) -// { -// // time has come - execute action -// Log.LogDebug($"Handle.Execute({envelope.Time})"); -// var message = _deserializer(envelope.Body); -// await _handler.Execute(message); -// } -// else -// { -// // still time - put it back to the queue -// Log.LogDebug($"Handle.Republish({envelope.Time})"); -// await _publisher.Publish(envelope); -// } -// } -// -// public void Dispose() -// { -// _subscription?.Dispose(); -// } -// } -// -// public interface IJobQueue -// { -// IDisposable Subscribe(Func handle); -// } -// } +using System; +using System.Threading; +using System.Threading.Tasks; +using K4os.Xpovoc.Abstractions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace K4os.Xpovoc.Core.Queue; + +public class QueueJobScheduler: IJobScheduler +{ + public ILogger Log { get; } + + private readonly IDateTimeSource _dateTimeSource; + private readonly IJobQueueAdapter _jobQueueAdapter; + private readonly IJobHandler _jobHandler; + private readonly IDisposable _jobQueueSubscription; + + public QueueJobScheduler( + ILoggerFactory? loggerFactory, + IJobQueueAdapter jobStorage, + IJobHandler jobHandler, + IDateTimeSource? dateTimeSource = null) + { + Log = loggerFactory?.CreateLogger(GetType()) ?? NullLogger.Instance; + _dateTimeSource = dateTimeSource ?? SystemDateTimeSource.Default; + _jobHandler = jobHandler.Required(nameof(jobHandler)); + _jobQueueAdapter = jobStorage.Required(nameof(jobStorage)); + _jobQueueSubscription = _jobQueueAdapter.Subscribe(TryHandle); + } + + private Task TryHandle(CancellationToken token, IJob job) => + Now < job.UtcTime ? Reschedule(job) : Handle(token, job); + + private async Task Handle(CancellationToken token, IJob job) + { + var jobId = job.JobId; + + Log.LogInformation("Job {Job} has been started", jobId); + + try + { + var payload = job.Payload; + + if (payload is null) + { + Log.LogWarning("Job {Job} had no payload and has been ignored", jobId); + } + else + { + await _jobHandler.Handle(token, payload); + } + } + catch (Exception e) + { + // NOTE: error handling and retry policy is NOT concern of this class + Log.LogError(e, "Job {Job} execution failed", jobId); + throw; + } + } + + public DateTimeOffset Now => _dateTimeSource.Now; + + protected class JobEnvelope: IJob + { + public Guid JobId { get; set; } + public DateTime UtcTime { get; set; } + public object? Payload { get; set; } + public object? Context { get; set; } + } + + public async Task Schedule(DateTimeOffset time, object payload) + { + var jobId = Guid.NewGuid(); + var job = new JobEnvelope { + JobId = jobId, + UtcTime = time.UtcDateTime, + Payload = payload, + Context = null, + }; + await Schedule(job); + return jobId; + } + + private Task Reschedule(IJob envelope) => + Schedule(envelope); + + private async Task Schedule(IJob job) + { + var when = job.UtcTime; + var delay = when - Now; + await _jobQueueAdapter.Publish(delay, job, CancellationToken.None); + } + + public void Dispose() + { + _jobQueueSubscription.Dispose(); + _jobQueueAdapter.TryDispose(); + } +} diff --git a/src/K4os.Xpovoc.Sqs.Test/K4os.Xpovoc.Sqs.Test.csproj b/src/K4os.Xpovoc.Sqs.Test/K4os.Xpovoc.Sqs.Test.csproj new file mode 100644 index 0000000..f845af6 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/K4os.Xpovoc.Sqs.Test.csproj @@ -0,0 +1,29 @@ + + + + net6.0 + enable + enable + + false + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + diff --git a/src/K4os.Xpovoc.Sqs.Test/SqsFactoryAndQueueTests.cs b/src/K4os.Xpovoc.Sqs.Test/SqsFactoryAndQueueTests.cs new file mode 100644 index 0000000..4116425 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/SqsFactoryAndQueueTests.cs @@ -0,0 +1,148 @@ +using Amazon.SQS; +using Amazon.SQS.Model; +using K4os.Xpovoc.Sqs.Internal; +using Xunit.Abstractions; + +namespace K4os.Xpovoc.Sqs.Test; + +public class SqsFactoryAndQueueTests +{ + private readonly ITestOutputHelper _testOutputHelper; + + public SqsFactoryAndQueueTests(ITestOutputHelper testOutputHelper) + { + _testOutputHelper = testOutputHelper; + } + + [Fact] + public async Task QueueCanBeCreatedOrFound() + { + var queueName = "mk-scheduler-test"; + var factory = new SqsQueueFactory(new AmazonSQSClient()); + var queue = await factory.Create(queueName, new SqsQueueSettings()); + Assert.NotNull(queue); + } + + private static SendMessageBatchRequestEntry CreateMessage() + { + var id = Guid.NewGuid().ToString("N"); + var jitterFive = Random.Shared.NextDouble() * 5; + return new SendMessageBatchRequestEntry { + Id = id, + MessageAttributes = new Dictionary { + { + "Xpovoc-ScheduledFor", + new MessageAttributeValue { + DataType = "String", + StringValue = DateTime.UtcNow.AddSeconds(jitterFive).ToString("O"), + } + }, + }, + MessageBody = $"message {id}", + }; + } + + [Fact] + public async Task MessagesCanBySent() + { + var factory = new SqsQueueFactory(new AmazonSQSClient()); + var queue = await factory.Create("mk-scheduler-test", new SqsQueueSettings()); + + var response = await queue.Send( + new List { + CreateMessage(), + CreateMessage(), + CreateMessage(), + }, CancellationToken.None); + + Assert.True(response.Count > 0); + Assert.True(response.All(r => r.Error == null)); + } + + [Fact] + public async Task MessagesCanBySentAndReceived() + { + var factory = new SqsQueueFactory(new AmazonSQSClient()); + var queue = await factory.Create("mk-scheduler-test", new SqsQueueSettings()); + + await queue.Send( + new List { + CreateMessage(), + CreateMessage(), + CreateMessage(), + }, CancellationToken.None); + + var messages = await queue.Receive(CancellationToken.None); + var count = messages.Count; + + _testOutputHelper.WriteLine($"Received {count} messages"); + + Assert.True(count > 0); + } + + [Fact] + public async Task MessagesCanBeDeleted() + { + var factory = new SqsQueueFactory(new AmazonSQSClient()); + var queue = await factory.Create("mk-scheduler-test", new SqsQueueSettings()); + + await queue.Send( + new List { + CreateMessage(), + CreateMessage(), + CreateMessage(), + }, CancellationToken.None); + + var messages = await queue.Receive(CancellationToken.None); + + var deletes = messages + .Select( + m => new DeleteMessageBatchRequestEntry { + Id = Guid.NewGuid().ToString("N"), + ReceiptHandle = m.ReceiptHandle, + }) + .ToList(); + + var response = await queue.Delete(deletes, CancellationToken.None); + var count = response.Count; + + _testOutputHelper.WriteLine($"Deleted {count} messages"); + + Assert.True(response.Count == messages.Count); + Assert.True(response.All(r => r.Error == null)); + } + + [Fact] + public async Task MessagesCanBeTouched() + { + var factory = new SqsQueueFactory(new AmazonSQSClient()); + var queue = await factory.Create("mk-scheduler-test", new SqsQueueSettings()); + + await queue.Send( + new List { + CreateMessage(), + CreateMessage(), + CreateMessage(), + }, CancellationToken.None); + + var messages = await queue.Receive(CancellationToken.None); + + var touches = messages + .Select( + m => new ChangeMessageVisibilityBatchRequestEntry { + Id = Guid.NewGuid().ToString("N"), + ReceiptHandle = m.ReceiptHandle, + VisibilityTimeout = 1, + }) + .ToList(); + + var response = await queue.Touch(touches, CancellationToken.None); + var count = response.Count; + + _testOutputHelper.WriteLine($"Touched {count} messages"); + + Assert.True(response.Count == messages.Count); + Assert.True(response.All(r => r.Error == null)); + } + +} diff --git a/src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs b/src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs new file mode 100644 index 0000000..c18f0d3 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs @@ -0,0 +1,282 @@ +using System.Diagnostics; +using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Reactive.Threading.Tasks; +using Amazon.SQS; +using K4os.Xpovoc.Abstractions; +using K4os.Xpovoc.Json; +using K4os.Xpovoc.Sqs.Internal; +using Newtonsoft.Json; +using Xunit.Abstractions; + +namespace K4os.Xpovoc.Sqs.Test; + +public class SqsJobQueueAdapterTests +{ + private readonly ITestOutputHelper _output; + private readonly TestLoggerFactory _loggerFactory; + private static readonly AmazonSQSClient AmazonSqsClient = new(); + + private static readonly JsonJobSerializer JsonJobSerializer = new( + new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Auto }); + + private static readonly SqsJobQueueAdapterConfig DefaultAdapterConfig = new() { + QueueName = "mk-SqsJobQueueAdapter-tests", + Concurrency = 1, + }; + + private static readonly TestConfig FastRoundtripConfig = new() { + QueueName = "mk-SqsJobQueueAdapter-5s-tests", + Concurrency = 1, + SqsQueueSettings = new SqsQueueSettings { + VisibilityTimeout = TimeSpan.FromSeconds(5), + }, + }; + + private static readonly TestConfig BatchingConfig = new() { + QueueName = "mk-SqsJobQueueAdapter-10s-tests", + Concurrency = 16, + SqsQueueSettings = new SqsQueueSettings { + VisibilityTimeout = TimeSpan.FromSeconds(10), + }, + }; + + public SqsJobQueueAdapterTests(ITestOutputHelper output) + { + _output = output; + _loggerFactory = new TestLoggerFactory(_output); + } + + [Fact] + public void AdapterCanBeInstantiatedAndDisposed() + { + var adapter = new SqsJobQueueAdapter( + _loggerFactory, + AmazonSqsClient, + JsonJobSerializer, + DefaultAdapterConfig); + adapter.Dispose(); + } + + [Fact] + public async Task CanPublishJob() + { + var adapter = new SqsJobQueueAdapter( + _loggerFactory, + AmazonSqsClient, + JsonJobSerializer, + DefaultAdapterConfig); + + await adapter.Publish( + TimeSpan.Zero, + new TestJob(DateTime.UtcNow, Guid.NewGuid()), + CancellationToken.None); + + adapter.Dispose(); + } + + [Fact] + public async Task CanSubscribe() + { + var adapter = new SqsJobQueueAdapter( + _loggerFactory, + AmazonSqsClient, + JsonJobSerializer, + DefaultAdapterConfig); + + var guid = Guid.NewGuid().ToString(); + _output.WriteLine("Expecting: {0}", guid); + + await adapter.Publish( + TimeSpan.Zero, + new TestJob(DateTime.UtcNow, guid), + CancellationToken.None); + + var jobs = new ReplaySubject(); + var subscription = Subscribe(adapter, jobs); + var found = jobs + .Do(j => _output.WriteLine("Received: {0}", j.Payload)) + .FirstOrDefaultAsync(j => guid.Equals(j.Payload)) + .ToTask(); + + WaitForTask(found); + + subscription.Dispose(); + adapter.Dispose(); + } + + [Fact] + public async Task CanDelayMessage() + { + var adapter = new SqsJobQueueAdapter( + _loggerFactory, + AmazonSqsClient, + JsonJobSerializer, + DefaultAdapterConfig); + + var guid = Guid.NewGuid().ToString(); + _output.WriteLine("Expecting: {0}", guid); + + var delay = TimeSpan.FromSeconds(5); + var started = DateTime.UtcNow; + + await adapter.Publish( + delay, + new TestJob(DateTime.UtcNow.Add(delay), guid), + CancellationToken.None); + + var jobs = new ReplaySubject(); + var subscription = Subscribe(adapter, jobs); + var found = jobs + .Do(j => _output.WriteLine("Received: {0}", j.Payload)) + .FirstOrDefaultAsync(j => guid.Equals(j.Payload)) + .ToTask(); + + WaitForTask(found); + + var finished = DateTime.UtcNow; + var elapsed = finished - started; + + Assert.True(elapsed >= delay); + + subscription.Dispose(); + adapter.Dispose(); + } + + [Fact] + public async Task HandledMessageIsKeptInvisible() + { + var adapter = new SqsJobQueueAdapter( + _loggerFactory, + AmazonSqsClient, + JsonJobSerializer, + FastRoundtripConfig); + + // make some artificial crowd + for (var i = 0; i < 10; i++) + { + await adapter.Publish( + TimeSpan.Zero, + new TestJob(DateTime.UtcNow, Guid.NewGuid()), + CancellationToken.None); + } + + var guid = Guid.NewGuid().ToString(); + _output.WriteLine("Expecting: {0}", guid); + + await adapter.Publish( + TimeSpan.Zero, + new TestJob(DateTime.UtcNow, guid), + CancellationToken.None); + + var done = new TaskCompletionSource(); + var flag = new[] { 0 }; + var subscription = adapter.Subscribe((_, m) => LongAction(flag, m, guid, done)); + + WaitForTask(done.Task, TimeSpan.FromMinutes(1)); + + subscription.Dispose(); + adapter.Dispose(); + } + + private async Task LongAction( + int[] flag, + IJob job, + string guid, + TaskCompletionSource done) + { + if (job.Payload as string != guid) + return; + + if (Interlocked.CompareExchange(ref flag[0], 1, 0) != 0) + { + done.SetException(new InvalidOperationException("Already handled")); + return; + } + + for (var i = 0; i < 30; i++) + { + await Task.Delay(1000); + _output.WriteLine($"Long action... {i}"); + } + + done.SetResult(true); + } + + [Fact] + public async Task InConcurrentSettingItUsesBatching() + { + var adapter = new SqsJobQueueAdapter( + _loggerFactory, + AmazonSqsClient, + JsonJobSerializer, + BatchingConfig); + + var expected = Enumerable + .Range(0, 100) + .Select(_ => Guid.NewGuid().ToString()) + .ToHashSet(); + var original = expected.ToHashSet(); // clone + int concurrent = 0; + var done = new TaskCompletionSource(); + + var publishTasks = expected + .Select(g => new TestJob(DateTime.UtcNow, g)) + .Select(j => adapter.Publish(TimeSpan.Zero, j, CancellationToken.None)) + .ToArray(); + + await Task.WhenAll(publishTasks); + + async Task ActOnMessage(IJob job, int time) + { + var id = (string)job.Payload!; + + lock (expected) + { + if (!original.Contains(id)) return; + if (!expected.Contains(id)) + throw new InvalidOperationException("Duplicate message"); + } + + var saturation = Interlocked.Increment(ref concurrent); + _output.WriteLine($"Started {id} ({time}s left, {saturation} concurrent, ~{expected.Count} left)"); + + await Task.Delay(TimeSpan.FromSeconds(time)); + + lock (expected) + { + expected.Remove(id); + if (expected.Count == 0) done.SetResult(true); + } + + _output.WriteLine($"Done {id}"); + Interlocked.Decrement(ref concurrent); + } + + var subscription = adapter.Subscribe((_, m) => ActOnMessage(m, 10/*Random.Shared.Next(8, 13)*/)); + + // 100 messages, 8-12 seconds each, but 16 at the time = 100*12/16 = 75 seconds + WaitForTask(done.Task, TimeSpan.FromSeconds(80)); + + subscription.Dispose(); + adapter.Dispose(); + } + + public T WaitForTask(Task task, TimeSpan? timeout = null) + { + var wait = timeout ?? ( + Debugger.IsAttached ? TimeSpan.FromMinutes(1) : TimeSpan.FromSeconds(10) + ); + var done = task.Wait(wait); + if (!done) throw new TimeoutException(); + + return task.GetAwaiter().GetResult(); + } + + private static IDisposable Subscribe(SqsJobQueueAdapter adapter, IObserver jobs) => + adapter.Subscribe( + (_, j) => { + jobs.OnNext(j); + return Task.CompletedTask; + }); +} diff --git a/src/K4os.Xpovoc.Sqs.Test/TestConfig.cs b/src/K4os.Xpovoc.Sqs.Test/TestConfig.cs new file mode 100644 index 0000000..a43c934 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/TestConfig.cs @@ -0,0 +1,10 @@ +using K4os.Xpovoc.Sqs.Internal; + +namespace K4os.Xpovoc.Sqs.Test; + +public class TestConfig: ISqsJobQueueAdapterConfig +{ + public string QueueName { get; set; } = null!; + public int Concurrency { get; set; } + public SqsQueueSettings? SqsQueueSettings { get; set; } +} diff --git a/src/K4os.Xpovoc.Sqs.Test/TestJob.cs b/src/K4os.Xpovoc.Sqs.Test/TestJob.cs new file mode 100644 index 0000000..b1bd471 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/TestJob.cs @@ -0,0 +1,19 @@ +using K4os.Xpovoc.Abstractions; + +namespace K4os.Xpovoc.Sqs.Test; + +public class TestJob: IJob +{ + public Guid JobId { get; } + public DateTime UtcTime { get; } + public object? Payload { get; } + public object? Context { get; } + + public TestJob(DateTime time, object payload) + { + JobId = Guid.NewGuid(); + UtcTime = time; + Payload = payload; + Context = null; + } +} diff --git a/src/K4os.Xpovoc.Sqs.Test/TestLogger.cs b/src/K4os.Xpovoc.Sqs.Test/TestLogger.cs new file mode 100644 index 0000000..562f0d6 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/TestLogger.cs @@ -0,0 +1,31 @@ +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; + +namespace K4os.Xpovoc.Sqs.Test; + +public class TestLogger: ILogger +{ + private readonly ITestOutputHelper _output; + private readonly string _categoryName; + + public TestLogger(ITestOutputHelper output, string categoryName) + { + _output = output; + _categoryName = categoryName; + } + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + _output.WriteLine( + "[{0}] ({1}) {2}", logLevel, _categoryName, formatter(state, exception)); + } + + public bool IsEnabled(LogLevel logLevel) => true; + + public IDisposable? BeginScope(TState state) where TState: notnull => null; +} diff --git a/src/K4os.Xpovoc.Sqs.Test/TestLoggerFactory.cs b/src/K4os.Xpovoc.Sqs.Test/TestLoggerFactory.cs new file mode 100644 index 0000000..fdd9552 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/TestLoggerFactory.cs @@ -0,0 +1,17 @@ +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; + +namespace K4os.Xpovoc.Sqs.Test; + +public class TestLoggerFactory: ILoggerFactory +{ + private readonly ITestOutputHelper _output; + + public TestLoggerFactory(ITestOutputHelper output) { _output = output; } + + public ILogger CreateLogger(string categoryName) => new TestLogger(_output, categoryName); + + public void AddProvider(ILoggerProvider provider) { } + + public void Dispose() { } +} diff --git a/src/K4os.Xpovoc.Sqs.Test/Usings.cs b/src/K4os.Xpovoc.Sqs.Test/Usings.cs new file mode 100644 index 0000000..c802f44 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/Usings.cs @@ -0,0 +1 @@ +global using Xunit; diff --git a/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterConfig.cs b/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterConfig.cs new file mode 100644 index 0000000..60ecc15 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterConfig.cs @@ -0,0 +1,17 @@ +using K4os.Xpovoc.Sqs.Internal; + +namespace K4os.Xpovoc.Sqs; + +public interface ISqsJobQueueAdapterConfig +{ + public string QueueName { get; } + int Concurrency { get; set; } + SqsQueueSettings? SqsQueueSettings { get; } +} + +public class SqsJobQueueAdapterConfig: ISqsJobQueueAdapterConfig +{ + public string QueueName { get; set; } = null!; + public int Concurrency { get; set; } + SqsQueueSettings? ISqsJobQueueAdapterConfig.SqsQueueSettings => default; +} diff --git a/src/K4os.Xpovoc.Sqs/Internal/ISqsQueue.cs b/src/K4os.Xpovoc.Sqs/Internal/ISqsQueue.cs new file mode 100644 index 0000000..6edd59d --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/Internal/ISqsQueue.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS.Model; + +namespace K4os.Xpovoc.Sqs.Internal; + +public interface ISqsQueue +{ + Task GetAttributes( + CancellationToken token = default); + + Task>> Send( + List messages, + CancellationToken token); + + Task> Receive(CancellationToken token); + + Task>> Delete( + List messages, + CancellationToken token); + + Task>> Touch( + List messages, + CancellationToken token); +} diff --git a/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueFactory.cs b/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueFactory.cs new file mode 100644 index 0000000..2eef38d --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueFactory.cs @@ -0,0 +1,8 @@ +using System.Threading.Tasks; + +namespace K4os.Xpovoc.Sqs.Internal; + +public interface ISqsQueueFactory +{ + Task Create(string queueName, SqsQueueSettings settings); +} \ No newline at end of file diff --git a/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueSettings.cs b/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueSettings.cs new file mode 100644 index 0000000..c165e46 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueSettings.cs @@ -0,0 +1,11 @@ +using System; + +namespace K4os.Xpovoc.Sqs.Internal; + +public interface ISqsQueueSettings +{ + int? ReceiveCount { get; } + TimeSpan? RetentionPeriod { get; } + TimeSpan? VisibilityTimeout { get; } + TimeSpan? ReceiveMessageWait { get; } +} diff --git a/src/K4os.Xpovoc.Sqs/Internal/SqsAttributes.cs b/src/K4os.Xpovoc.Sqs/Internal/SqsAttributes.cs new file mode 100644 index 0000000..988f3ad --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/Internal/SqsAttributes.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using Amazon.SQS.Model; + +namespace K4os.Xpovoc.Sqs.Internal; + +internal static class SqsAttributes +{ + public static readonly List All = new() { "All" }; + public const string JobId = "Xpovoc.JobId"; + public const string ScheduledFor = "Xpovoc.ScheduledFor"; + + public static string? TryGetStringAttribute(this Message message, string name) => + message.MessageAttributes.TryGetOrDefault(name)?.StringValue; +} diff --git a/src/K4os.Xpovoc.Sqs/Internal/SqsConstants.cs b/src/K4os.Xpovoc.Sqs/Internal/SqsConstants.cs new file mode 100644 index 0000000..d780978 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/Internal/SqsConstants.cs @@ -0,0 +1,13 @@ +using System; + +namespace K4os.Xpovoc.Sqs.Internal; + +internal static class SqsConstants +{ + public const int MaximumReceiveCount = 10; + public const int MaximumNumberOfMessages = 10; + public static readonly TimeSpan MaximumRetentionPeriod = TimeSpan.FromDays(14); + public static readonly TimeSpan DefaultVisibilityTimeout = TimeSpan.FromSeconds(30); + public static readonly TimeSpan DefaultReceiveMessageWait = TimeSpan.FromSeconds(20); + public static readonly TimeSpan MaximumDelay = TimeSpan.FromMinutes(15); +} diff --git a/src/K4os.Xpovoc.Sqs/Internal/SqsQueue.cs b/src/K4os.Xpovoc.Sqs/Internal/SqsQueue.cs new file mode 100644 index 0000000..5628392 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/Internal/SqsQueue.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS; +using Amazon.SQS.Model; + +namespace K4os.Xpovoc.Sqs.Internal; + +//public TimeSpan RetentionPeriod => TimeSpan.FromSeconds(Attributes.MessageRetentionPeriod); +//public TimeSpan VisibilityTimeout => TimeSpan.FromSeconds(Attributes.VisibilityTimeout); + + +public class SqsQueue: ISqsQueue +{ + private readonly IAmazonSQS _client; + private readonly string _queueUrl; + private GetQueueAttributesResponse? _attributes; + + public string Url => _queueUrl; + + public SqsQueue(IAmazonSQS client, string queueUrl) + { + _client = client; + _queueUrl = queueUrl; + } + + public async Task GetAttributes( + CancellationToken token = default) => + _attributes ??= await _client.GetQueueAttributesAsync( + new GetQueueAttributesRequest { + QueueUrl = _queueUrl, + AttributeNames = SqsAttributes.All, + }, token); + + public async Task>> Send( + List messages, + CancellationToken token) + { + var response = await _client.SendMessageBatchAsync( + new SendMessageBatchRequest { + QueueUrl = _queueUrl, + Entries = messages, + }, token); + + return ComposeResponse(response.Successful, response.Failed); + } + + public async Task> Receive(CancellationToken token) + { + var response = await _client.ReceiveMessageAsync( + new ReceiveMessageRequest { + QueueUrl = _queueUrl, + MaxNumberOfMessages = SqsConstants.MaximumNumberOfMessages, + AttributeNames = SqsAttributes.All, // ApproximateReceiveCount, SentTimestamp + MessageAttributeNames = SqsAttributes.All, + }, token); + + return response.Messages; + } + + public async Task>> Delete( + List messages, + CancellationToken token) + { + var response = await _client.DeleteMessageBatchAsync( + new DeleteMessageBatchRequest { + QueueUrl = _queueUrl, + Entries = messages, + }, token); + + return ComposeResponse(response.Successful, response.Failed); + } + + public async Task>> Touch( + List messages, + CancellationToken token) + { + var response = await _client.ChangeMessageVisibilityBatchAsync( + new ChangeMessageVisibilityBatchRequest { + QueueUrl = _queueUrl, + Entries = messages, + }, token); + + return ComposeResponse(response.Successful, response.Failed); + } + + private static List> ComposeResponse( + IReadOnlyCollection success, + IReadOnlyCollection failure) + { + var count = success.Count + failure.Count; + var result = new List>(count); + result.AddRange(success.Select(r => new SqsResult(r))); + result.AddRange(failure.Select(r => new SqsResult(r))); + return result; + } +} diff --git a/src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs b/src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs new file mode 100644 index 0000000..3714a71 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs @@ -0,0 +1,127 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS; +using Amazon.SQS.Model; +using ThirdParty.Json.LitJson; + +namespace K4os.Xpovoc.Sqs.Internal; + +public class SqsQueueFactory: ISqsQueueFactory +{ + private readonly IAmazonSQS _client; + + public SqsQueueFactory(IAmazonSQS client) { _client = client; } + + public async Task Create(string queueName, SqsQueueSettings settings) + { + var queueUrl = + await FindOrCreateQueue(queueName, settings) ?? + throw new InvalidOperationException($"Failed to find queue: {queueName}"); + return new SqsQueue(_client, queueUrl); + } + + private async Task FindOrCreateQueue(string queueName, SqsQueueSettings settings) + { + if (await TryFindQueue(queueName) is { } queueUrl) + return queueUrl; + + var deadLetterName = queueName + "-dlq"; + + var deadLetterUrl = + await TryCreateQueue(deadLetterName, null, new SqsQueueSettings()) ?? + throw new InvalidOperationException( + $"Failed to create dead letter queue: {deadLetterName}"); + + return + await TryCreateQueue(queueName, deadLetterUrl, settings) ?? + throw new InvalidOperationException($"Failed to create queue: {queueName}"); + } + + private async Task TryCreateQueue( + string queueName, string? deadLetterUrl, SqsQueueSettings settings) + { + var queueSettings = new Dictionary(); + + queueSettings.AddIfNotNull( + QueueAttributeName.MessageRetentionPeriod, + ToQueueAttribute( + settings.RetentionPeriod ?? SqsConstants.MaximumRetentionPeriod)); + + queueSettings.AddIfNotNull( + QueueAttributeName.VisibilityTimeout, + ToQueueAttribute( + settings.VisibilityTimeout ?? SqsConstants.DefaultVisibilityTimeout)); + + queueSettings.AddIfNotNull( + QueueAttributeName.ReceiveMessageWaitTimeSeconds, + ToQueueAttribute( + settings.ReceiveMessageWait ?? SqsConstants.DefaultReceiveMessageWait)); + + if (deadLetterUrl is not null) + { + var deadLetterArn = await GetQueueArn(deadLetterUrl); + queueSettings.Add( + QueueAttributeName.RedrivePolicy, + ToRedrivePolicy( + deadLetterArn, + settings.ReceiveCount)); + } + + try + { + var response = await _client.CreateQueueAsync( + new CreateQueueRequest { + QueueName = queueName, + Attributes = queueSettings, + }); + + return response.QueueUrl; + } + catch (QueueNameExistsException) + { + return await TryFindQueue(queueName); + } + } + + private async Task GetQueueArn(string queueUrl) + { + var response = await _client.GetQueueAttributesAsync( + new GetQueueAttributesRequest { + QueueUrl = queueUrl, + AttributeNames = SqsAttributes.All, + }); + + return response.QueueARN; + } + + private async Task TryFindQueue(string queueName) + { + try + { + var response = await _client.GetQueueUrlAsync( + new GetQueueUrlRequest { QueueName = queueName }); + return response.QueueUrl; + } + catch (QueueDoesNotExistException) + { + return null; + } + } + + private static string? ToQueueAttribute(TimeSpan? timespan) => + timespan is not null + ? ((int)timespan.Value.TotalSeconds.NotLessThan(0)).ToString() + : null; + + private static string ToRedrivePolicy(string queueArn, int? maxReceiveCount) + { + var values = new Dictionary { + { "deadLetterTargetArn", queueArn }, + { "maxReceiveCount", maxReceiveCount ?? SqsConstants.MaximumReceiveCount }, + }; + + return JsonMapper.ToJson(values); + } +} \ No newline at end of file diff --git a/src/K4os.Xpovoc.Sqs/Internal/SqsQueueSettings.cs b/src/K4os.Xpovoc.Sqs/Internal/SqsQueueSettings.cs new file mode 100644 index 0000000..b5b873c --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/Internal/SqsQueueSettings.cs @@ -0,0 +1,11 @@ +using System; + +namespace K4os.Xpovoc.Sqs.Internal; + +public class SqsQueueSettings: ISqsQueueSettings +{ + public int? ReceiveCount { get; set; } + public TimeSpan? RetentionPeriod { get; set; } + public TimeSpan? VisibilityTimeout { get; set; } + public TimeSpan? ReceiveMessageWait { get; set; } +} \ No newline at end of file diff --git a/src/K4os.Xpovoc.Sqs/Internal/SqsResult.cs b/src/K4os.Xpovoc.Sqs/Internal/SqsResult.cs new file mode 100644 index 0000000..1c88124 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/Internal/SqsResult.cs @@ -0,0 +1,13 @@ +using Amazon.SQS.Model; + +namespace K4os.Xpovoc.Sqs.Internal; + +public readonly record struct SqsResult( + T? Result, BatchResultErrorEntry? Error) +{ + public SqsResult(T result): this(result, null) { } + public SqsResult(BatchResultErrorEntry error): this(default, error) { } + + public bool IsSuccess => !IsError; + public bool IsError => Error != null; +} diff --git a/src/K4os.Xpovoc.Sqs/K4os.Xpovoc.Sqs.csproj b/src/K4os.Xpovoc.Sqs/K4os.Xpovoc.Sqs.csproj index 220c990..3439e12 100644 --- a/src/K4os.Xpovoc.Sqs/K4os.Xpovoc.Sqs.csproj +++ b/src/K4os.Xpovoc.Sqs/K4os.Xpovoc.Sqs.csproj @@ -12,4 +12,12 @@ Extensions.cs + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + \ No newline at end of file diff --git a/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs b/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs new file mode 100644 index 0000000..fe5a46b --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs @@ -0,0 +1,328 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS; +using Amazon.SQS.Model; +using K4os.Async.Toys; +using K4os.Xpovoc.Abstractions; +using K4os.Xpovoc.Core.Queue; +using K4os.Xpovoc.Sqs.Internal; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace K4os.Xpovoc.Sqs; + +public class SqsJobQueueAdapter: IJobQueueAdapter +{ + protected ILogger Log { get; } + + private readonly IJobSerializer _serializer; + private readonly ISqsJobQueueAdapterConfig _config; + private readonly ISqsQueueFactory _factory; + private readonly Task _ready; + + private ISqsQueue _queue; + private TimeSpan _visibility; + + private IBatchBuilder> + _batchSender; + + private IAliveKeeper _aliveKeeper; + + public SqsJobQueueAdapter( + ILoggerFactory? loggerFactory, + IAmazonSQS client, + IJobSerializer serializer, + ISqsJobQueueAdapterConfig config): + this( + loggerFactory, + new SqsQueueFactory(client), + serializer, + config) { } + + public SqsJobQueueAdapter( + ILoggerFactory? loggerFactory, + ISqsQueueFactory queueFactory, + IJobSerializer serializer, + ISqsJobQueueAdapterConfig config) + { + Log = loggerFactory?.CreateLogger(GetType()) ?? NullLogger.Instance; + + _factory = queueFactory; + _config = config; + _serializer = serializer; + + _queue = null!; + _visibility = SqsConstants.DefaultVisibilityTimeout; // might not be true + _batchSender = null!; + _aliveKeeper = null!; + + _ready = Task.Run(Startup); + } + + private async Task Startup() + { + _queue = await CreateQueue(); + var attributes = await _queue.GetAttributes(); + _visibility = TimeSpan.FromSeconds(attributes.VisibilityTimeout); + _batchSender = CreateBatchSender(); + _aliveKeeper = CreateAliveKeeper(); + } + + private static string TextId => Guid.NewGuid().ToString("D"); + + private async Task CreateQueue() + { + var queueSettings = _config.SqsQueueSettings ?? new SqsQueueSettings(); + var queue = await _factory.Create(_config.QueueName, queueSettings); + return queue; + } + + private IAliveKeeper CreateAliveKeeper() + { + + const int retryCount = 5; + var retryInterval = TimeSpan.FromSeconds(1); + var touchInterval = CalculateTouchInterval( + _visibility, retryInterval, retryCount); + var touchDelay = CalculateTouchDelay( + _visibility, touchInterval, retryInterval, retryCount); + + var aliveKeeperSettings = new AliveKeeperSettings { + DeleteBatchSize = SqsConstants.MaximumNumberOfMessages, + TouchBatchSize = SqsConstants.MaximumNumberOfMessages, + RetryLimit = retryCount, + RetryInterval = retryInterval, + TouchInterval = touchInterval, + TouchBatchDelay = touchDelay, + }; + + var aliveKeeper = AliveKeeper.Create( + TouchMany, + DeleteMany, + m => m.MessageId, + aliveKeeperSettings, + Log); + + return aliveKeeper; + } + + private IBatchBuilder> + CreateBatchSender() + { + var batchSenderSettings = new BatchBuilderSettings { + BatchDelay = TimeSpan.Zero, + BatchSize = SqsConstants.MaximumNumberOfMessages, + Concurrency = 4, + }; + + return BatchBuilder + .Create>( + rq => rq.Id, + rs => rs.Error?.Id ?? rs.Result?.Id ?? string.Empty, + SendMany, + batchSenderSettings, + Log); + } + + private static TimeSpan CalculateTouchInterval( + TimeSpan visibilityTimeout, TimeSpan retryInterval, int retryCount) + { + var retryMargin = retryInterval.TotalSeconds * retryCount; + var visibility = visibilityTimeout.TotalSeconds; + var interval = Math.Min(visibility / 2, visibility - retryMargin).NotLessThan(1); + return TimeSpan.FromSeconds(interval); + } + + private static TimeSpan CalculateTouchDelay( + TimeSpan visibilityTimeout, + TimeSpan touchInterval, + TimeSpan retryInterval, + int retryCount) + { + var visibility = visibilityTimeout.TotalSeconds; + var interval = touchInterval.TotalSeconds; + var retryMargin = retryInterval.TotalSeconds * retryCount; + var delta = (visibility - interval - retryMargin) / retryCount; + return TimeSpan.FromSeconds(delta.NotLessThan(0).NotMoreThan(1)); + } + + private async Task[]> SendMany( + SendMessageBatchRequestEntry[] messages) + { + await _ready; + var requests = messages.ToList(); + Log.LogDebug("Sending batch of {Count} messages", requests.Count); + var responses = await _queue.Send(requests, CancellationToken.None); + return responses.ToArray(); + } + + private async Task DeleteMany(Message[] messages) + { + await _ready; + var map = messages.ToDictionary(m => m.MessageId); + var requests = messages.Select(ToDeleteRequest).ToList(); + Log.LogDebug("Deleting batch of {Count} messages", requests.Count); + if (requests.Count == 0) Debugger.Break(); + var response = await _queue.Delete(requests, CancellationToken.None); + return response + .SelectNotNull(r => r.Result?.Id) + .SelectNotNull(id => map.TryGetOrDefault(id)) + .ToArray(); + } + + private async Task TouchMany(Message[] messages) + { + await _ready; + var timeout = (int)_visibility.TotalSeconds; + var map = messages.ToDictionary(m => m.MessageId); + var requests = messages.Select(m => ToTouchRequest(m, timeout)).ToList(); + Log.LogDebug("Touching batch of {Count} messages", requests.Count); + if (requests.Count == 0) Debugger.Break(); + var response = await _queue.Touch(requests, CancellationToken.None); + return response + .SelectNotNull(r => r.Result?.Id) + .SelectNotNull(id => map.TryGetOrDefault(id)) + .ToArray(); + } + + private static DeleteMessageBatchRequestEntry ToDeleteRequest( + Message message) => + new() { + Id = message.MessageId, + ReceiptHandle = message.ReceiptHandle, + }; + + private static ChangeMessageVisibilityBatchRequestEntry ToTouchRequest( + Message message, int timeout) => + new() { + Id = message.MessageId, + ReceiptHandle = message.ReceiptHandle, + VisibilityTimeout = timeout, + }; + + private static SendMessageBatchRequestEntry CreateSendOneRequest( + TimeSpan delay, Guid jobId, DateTime scheduledFor, string? payload) + { + delay = delay.NotLessThan(TimeSpan.Zero).NotMoreThan(SqsConstants.MaximumDelay); + + return new SendMessageBatchRequestEntry { + Id = TextId, + DelaySeconds = (int)delay.TotalSeconds, + MessageBody = payload ?? string.Empty, + MessageAttributes = new Dictionary { + [SqsAttributes.JobId] = CreateAttribute(jobId), + [SqsAttributes.ScheduledFor] = CreateAttribute(scheduledFor), + }, + }; + } + + private SendMessageBatchRequestEntry CreateSendOneRequest( + TimeSpan delay, IJob job) => + CreateSendOneRequest(delay, job.JobId, job.UtcTime, Serialize(job)); + + private string? Serialize(IJob job) => + (job.Context as Message)?.Body ?? ( + job.Payload is null ? null : _serializer.Serialize(job.Payload) + ); + + private static MessageAttributeValue CreateAttribute(string text) => + new() { DataType = "String", StringValue = text }; + + private static MessageAttributeValue CreateAttribute(Guid guid) => + CreateAttribute(guid.ToString("D")); + + private static MessageAttributeValue CreateAttribute(DateTime time) => + CreateAttribute(time.ToString("O")); + + public async Task Publish(TimeSpan delay, IJob job, CancellationToken token) + { + await _ready; + + var jobId = job.JobId; + var request = CreateSendOneRequest(delay, job); + var response = await _batchSender.Request(request); + var error = response.Error; + + if (error is null) + return; + + throw new ArgumentException( + $"Failed to publish job {jobId}: {error.Message}", + nameof(job)); + } + + public IDisposable Subscribe(Func handler) => + Agent.Launch(c => LongPollLoop(c, handler, _config.Concurrency), Log); + + private async Task LongPollLoop( + IAgentContext context, + Func handler, + int concurrency) + { + await _ready; + + var token = context.Token; + var semaphore = new SemaphoreSlim(concurrency.NotLessThan(1)); + + while (!token.IsCancellationRequested) + { + await LongPoll(handler, semaphore, token); + } + } + + private async Task LongPoll( + Func handler, + SemaphoreSlim semaphore, + CancellationToken token) + { + var messages = (await _queue.Receive(token)).ToArray(); + if (messages.Length <= 0) return; + + foreach (var message in messages) + { + _aliveKeeper.Upkeep(message, token); + } + + foreach (var message in messages) + { + await semaphore.WaitAsync(token); + // every execution is in separate "thread" and is not awaited + Task.Run(() => HandleMessage(message, handler, semaphore, token), token).Forget(); + } + } + + private async Task HandleMessage( + Message message, + Func handler, + SemaphoreSlim semaphore, + CancellationToken token) + { + try + { + token.ThrowIfCancellationRequested(); + var job = new SqsReceivedJob(message, _serializer); + await handler(token, job); + await _aliveKeeper.Delete(message, token); + } + catch (Exception e) + { + Log.LogError(e, "Failed to process message {MessageId}", message.MessageId); + } + finally + { + _aliveKeeper.Forget(message); + semaphore.Release(); + } + } + + public void Dispose() + { + _ready.Await(); // it is safe to finish async initialization + _batchSender.Dispose(); + _aliveKeeper.Dispose(); + } +} diff --git a/src/K4os.Xpovoc.Sqs/SqsReceivedJob.cs b/src/K4os.Xpovoc.Sqs/SqsReceivedJob.cs new file mode 100644 index 0000000..b4b6311 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/SqsReceivedJob.cs @@ -0,0 +1,39 @@ +using System; +using Amazon.SQS.Model; +using K4os.Xpovoc.Abstractions; +using K4os.Xpovoc.Sqs.Internal; + +namespace K4os.Xpovoc.Sqs; + +internal class SqsReceivedJob: IJob +{ + private readonly Message _message; + private readonly IJobSerializer _serializer; + + private Guid? _jobId; + private DateTime? _scheduledFor; + private object? _payload; + + public SqsReceivedJob(Message message, IJobSerializer serializer) + { + _message = message; + _serializer = serializer; + } + + public Guid JobId => + _jobId ??= _message.TryGetStringAttribute(SqsAttributes.JobId) switch { + { } attr when Guid.TryParse(attr, out var parsed) => parsed, + _ => throw new ArgumentException($"Invalid '{SqsAttributes.JobId}' attribute"), + }; + + public DateTime UtcTime => + _scheduledFor ??= _message.TryGetStringAttribute(SqsAttributes.ScheduledFor) switch { + { } attr when DateTime.TryParse(attr, out var parsed) => parsed, + _ => throw new ArgumentException($"Invalid '{SqsAttributes.ScheduledFor}' attribute"), + }; + + public object Payload => + _payload ??= _serializer.Deserialize(_message.Body); + + public object Context => _message; +} diff --git a/src/K4os.Xpovoc.sln b/src/K4os.Xpovoc.sln index c0cc3ac..200a3ae 100644 --- a/src/K4os.Xpovoc.sln +++ b/src/K4os.Xpovoc.sln @@ -62,6 +62,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "K4os.Xpovoc.Hosting", "K4os EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "_build", "..\.nuke\build\_build.csproj", "{F7D34DD7-E82F-41A3-AE97-789FEE5A9C39}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "K4os.Xpovoc.Sqs.Test", "K4os.Xpovoc.Sqs.Test\K4os.Xpovoc.Sqs.Test.csproj", "{26039CB2-C5F9-40E6-9F16-85A0CD6897EF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -142,6 +144,10 @@ Global {309DF87F-E5C2-47D4-B4D2-DAA857C7EA16}.Debug|Any CPU.Build.0 = Debug|Any CPU {309DF87F-E5C2-47D4-B4D2-DAA857C7EA16}.Release|Any CPU.ActiveCfg = Release|Any CPU {309DF87F-E5C2-47D4-B4D2-DAA857C7EA16}.Release|Any CPU.Build.0 = Release|Any CPU + {26039CB2-C5F9-40E6-9F16-85A0CD6897EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {26039CB2-C5F9-40E6-9F16-85A0CD6897EF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {26039CB2-C5F9-40E6-9F16-85A0CD6897EF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {26039CB2-C5F9-40E6-9F16-85A0CD6897EF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -163,5 +169,6 @@ Global {48B30C5C-B95A-4690-921C-7D7C0BB0EC1C} = {F032EA99-FA15-4FFD-9261-0B7E77613C5E} {ED3837B6-3434-4213-B6D1-9DA0E7F9A258} = {4964B568-60A9-45DC-BFB5-F09772ECB635} {9A66EC7B-41D6-4C72-94CE-D359164EE0AA} = {4964B568-60A9-45DC-BFB5-F09772ECB635} + {26039CB2-C5F9-40E6-9F16-85A0CD6897EF} = {05CC704E-DDB0-4335-82E4-460AD6E89D20} EndGlobalSection EndGlobal diff --git a/src/K4os.Xpovoc.sln.DotSettings b/src/K4os.Xpovoc.sln.DotSettings index cde2741..bdb03c5 100644 --- a/src/K4os.Xpovoc.sln.DotSettings +++ b/src/K4os.Xpovoc.sln.DotSettings @@ -5,6 +5,7 @@ True True True + True True True True \ No newline at end of file diff --git a/src/Playground/Program.cs b/src/Playground/Program.cs index 543a338..96a7ff9 100644 --- a/src/Playground/Program.cs +++ b/src/Playground/Program.cs @@ -32,7 +32,7 @@ internal static class Program private static readonly int ConsumeDelay = 0; private static readonly int ConsumeThreads = 4; private static readonly bool EnablePruning = true; - + public static Task Compositions(string[] args) { var collection = new ServiceCollection(); @@ -65,7 +65,7 @@ private static void Configure(ServiceCollection serviceCollection) ConfigureMsSql(serviceCollection, secrets); ConfigureMongo(serviceCollection, secrets); ConfigureRedis(serviceCollection, secrets); - + serviceCollection.AddSingleton( new SchedulerConfig { WorkerCount = ConsumeThreads, @@ -122,7 +122,7 @@ private static void ConfigureMongo(ServiceCollection serviceCollection, XDocumen serviceCollection.AddSingleton( p => p.GetRequiredService().GetDatabase("test")); } - + private static void ConfigureRedis(ServiceCollection serviceCollection, XDocument secrets) { var connectionString = secrets.XPathSelectElement("/secrets/redis")?.Value; @@ -131,7 +131,7 @@ private static void ConfigureRedis(ServiceCollection serviceCollection, XDocumen serviceCollection.AddSingleton( p => new RedisJobStorageConfig { Prefix = "xpovoc" }); } - + private static async Task Execute( ILoggerFactory loggerFactory, IServiceProvider serviceProvider, string[] args) { @@ -157,14 +157,13 @@ private static async Task Execute( serviceProvider.GetRequiredService(), serializer); - var handler = new AdHocJobHandler(ConsumeOne); var schedulerConfig = serviceProvider.GetRequiredService(); // var scheduler = new DbJobScheduler(null, mysqlStorage, handler, schedulerConfig); // var scheduler = new RxJobScheduler(loggerFactory, handler, Scheduler.Default); var scheduler = new DbJobScheduler(null, redisStorage, handler, schedulerConfig); - + // var producer = Task.CompletedTask; // var producerSpeed = Task.CompletedTask; var producer = Task.Run(() => Producer(token, scheduler), token); @@ -224,7 +223,7 @@ private static async Task Producer(CancellationToken token, IJobScheduler schedu var random = new Random(); while (!token.IsCancellationRequested) { - if (ProduceDelay > 0) + if (ProduceDelay > 0) await Task.Delay(ProduceDelay, token); var delay = TimeSpan.FromSeconds(random.NextDouble() * 5); var message = Guid.NewGuid(); @@ -237,9 +236,9 @@ private static async Task Producer(CancellationToken token, IJobScheduler schedu private static void ConsumeOne(object payload) { - if (ConsumeDelay > 0) + if (ConsumeDelay > 0) Thread.Sleep(ConsumeDelay); - var guid = (Guid) payload; + var guid = (Guid)payload; var result = Guids.TryAdd(guid, null); if (!result) { From 642ac6a1f35a1f8f3c7b3769a3a204451ad65c1e Mon Sep 17 00:00:00 2001 From: Milosz Krajewski Date: Sat, 18 Nov 2023 15:14:32 +0000 Subject: [PATCH 2/5] single poller --- Directory.Build.props | 5 +- Directory.Build.targets | 3 + Directory.Packages.props | 33 +- docker-compose.yml | 22 +- src/K4os.Shared/Extensions.cs | 2 - src/K4os.Shared/Secrets.cs | 2 - .../IDateTimeSource.cs | 8 - src/K4os.Xpovoc.Abstractions/IJobScheduler.cs | 2 +- src/K4os.Xpovoc.Abstractions/ITimeSource.cs | 11 + .../K4os.Xpovoc.Abstractions.csproj | 4 +- .../SystemTimeSource.cs | 9 +- .../K4os.Xpovoc.Brighter.csproj | 13 +- src/K4os.Xpovoc.Core/Db/DbAgent.cs | 8 +- src/K4os.Xpovoc.Core/Db/DbCleaner.cs | 4 +- src/K4os.Xpovoc.Core/Db/DbJobScheduler.cs | 12 +- src/K4os.Xpovoc.Core/Db/DbPoller.cs | 4 +- src/K4os.Xpovoc.Core/K4os.Xpovoc.Core.csproj | 15 +- .../Queue/QueueJobScheduler.cs | 8 +- .../K4os.Xpovoc.Db.Test.csproj | 24 +- .../K4os.Xpovoc.Hosting.csproj | 10 +- src/K4os.Xpovoc.Json/K4os.Xpovoc.Json.csproj | 10 +- .../K4os.Xpovoc.MediatR.csproj | 13 +- .../K4os.Xpovoc.Mongo.csproj | 13 +- .../K4os.Xpovoc.MsSql.csproj | 22 +- .../K4os.Xpovoc.MySql.csproj | 8 +- .../K4os.Xpovoc.PgSql.csproj | 7 +- .../K4os.Xpovoc.Quarterback.csproj | 12 +- .../K4os.Xpovoc.Redis.csproj | 10 +- .../K4os.Xpovoc.SqLite.csproj | 10 +- .../K4os.Xpovoc.Sqs.Test.csproj | 18 +- .../Move10KUsingRawSqsAccess.cs | 104 ++++++ .../Move10KUsingSqsAdapter.cs | 107 ++++++ .../SqsQueueAdapterTests.cs | 10 +- src/K4os.Xpovoc.Sqs.Test/TestConfig.cs | 10 - .../Utilities/TestSqsQueue.cs | 230 +++++++++++++ .../Utilities/TestSqsQueueFactory.cs | 36 ++ .../Utilities/TestTimeSource.cs | 33 ++ src/K4os.Xpovoc.Sqs.Test/VeryLongRun.cs | 163 +++++++++ .../ISqsJobQueueAdapterConfig.cs | 17 - .../ISqsJobQueueAdapterSettings.cs | 26 ++ .../Internal/ISqsQueueFactory.cs | 2 +- src/K4os.Xpovoc.Sqs/Internal/SqsQueue.cs | 22 +- .../Internal/SqsQueueFactory.cs | 11 +- src/K4os.Xpovoc.Sqs/K4os.Xpovoc.Sqs.csproj | 20 +- src/K4os.Xpovoc.Sqs/SqsBatchAdapter.cs | 93 ++++++ src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs | 260 ++++++--------- src/K4os.Xpovoc.Sqs/ToysTimeSourceAdapter.cs | 14 + src/K4os.Xpovoc.Test/JobHandlerTests.cs | 4 + src/K4os.Xpovoc.Test/K4os.Xpovoc.Test.csproj | 22 +- src/Playground/ColorConsoleProvider.cs | 63 ---- src/Playground/InfiniteScheduling.cs | 309 ++++++++++++++++++ src/Playground/Playground.csproj | 18 +- src/Playground/Program.cs | 269 +-------------- src/Playground/SqsThroughput.cs | 132 ++++++++ src/Playground/TrueDeferredJobs.cs | 133 ++++++++ .../{ => Utilities}/MySqlExamples.cs | 2 +- .../{ => Utilities}/NullJobHandler.cs | 2 +- .../{ => Utilities}/PgSqlExamples.cs | 2 +- 58 files changed, 1769 insertions(+), 667 deletions(-) delete mode 100644 src/K4os.Xpovoc.Abstractions/IDateTimeSource.cs create mode 100644 src/K4os.Xpovoc.Abstractions/ITimeSource.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/Move10KUsingRawSqsAccess.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/Move10KUsingSqsAdapter.cs delete mode 100644 src/K4os.Xpovoc.Sqs.Test/TestConfig.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueue.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueueFactory.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/Utilities/TestTimeSource.cs create mode 100644 src/K4os.Xpovoc.Sqs.Test/VeryLongRun.cs delete mode 100644 src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterConfig.cs create mode 100644 src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterSettings.cs create mode 100644 src/K4os.Xpovoc.Sqs/SqsBatchAdapter.cs create mode 100644 src/K4os.Xpovoc.Sqs/ToysTimeSourceAdapter.cs delete mode 100644 src/Playground/ColorConsoleProvider.cs create mode 100644 src/Playground/InfiniteScheduling.cs create mode 100644 src/Playground/SqsThroughput.cs create mode 100644 src/Playground/TrueDeferredJobs.cs rename src/Playground/{ => Utilities}/MySqlExamples.cs (96%) rename src/Playground/{ => Utilities}/NullJobHandler.cs (87%) rename src/Playground/{ => Utilities}/PgSqlExamples.cs (96%) diff --git a/Directory.Build.props b/Directory.Build.props index ff019f1..e1d93f9 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -4,6 +4,7 @@ false enable true + true false $(NoWarn);NU5125;NU5048 @@ -20,7 +21,7 @@ https://github.com/MiloszKrajewski/K4os.Xpovoc https://github.com/MiloszKrajewski/K4os.Xpovoc/blob/master/doc/icon.png?raw=true - - + + \ No newline at end of file diff --git a/Directory.Build.targets b/Directory.Build.targets index c1df222..6d20f19 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -1,2 +1,5 @@ + + + \ No newline at end of file diff --git a/Directory.Packages.props b/Directory.Packages.props index f0b7ac2..e886c7c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,16 +4,33 @@ --> - - - + + + - - - - - + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index bbcdc96..f4ea771 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,11 +43,17 @@ services: ports: - "6379:6379" -# localstack: -# image: "localstack/localstack" -# network_mode: bridge -# ports: -# - "127.0.0.1:4566:4566" -# - "127.0.0.1:4571:4571" -# environment: -# SERVICES: sqs + sqs: + image: softwaremill/elasticmq-native + ports: + - "9324:9324" + - "9325:9325" + + # dynamodb: + # image: amazon/dynamodb-local + # command: "-jar DynamoDBLocal.jar -sharedDb -dbPath ./data" + # ports: + # - "8000:8000" + # # volumes: + # # - "./docker/dynamodb:/home/dynamodblocal/data" + # working_dir: /home/dynamodblocal diff --git a/src/K4os.Shared/Extensions.cs b/src/K4os.Shared/Extensions.cs index b761a20..ffe4908 100644 --- a/src/K4os.Shared/Extensions.cs +++ b/src/K4os.Shared/Extensions.cs @@ -1,5 +1,3 @@ -#nullable enable - using System; using System.Collections.Generic; using System.Linq; diff --git a/src/K4os.Shared/Secrets.cs b/src/K4os.Shared/Secrets.cs index b6d22ef..8162403 100644 --- a/src/K4os.Shared/Secrets.cs +++ b/src/K4os.Shared/Secrets.cs @@ -1,5 +1,3 @@ -#nullable enable - using System.IO; using System.Xml.Linq; diff --git a/src/K4os.Xpovoc.Abstractions/IDateTimeSource.cs b/src/K4os.Xpovoc.Abstractions/IDateTimeSource.cs deleted file mode 100644 index 53d389e..0000000 --- a/src/K4os.Xpovoc.Abstractions/IDateTimeSource.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System; - -namespace K4os.Xpovoc.Abstractions; - -public interface IDateTimeSource -{ - DateTimeOffset Now { get; } -} \ No newline at end of file diff --git a/src/K4os.Xpovoc.Abstractions/IJobScheduler.cs b/src/K4os.Xpovoc.Abstractions/IJobScheduler.cs index 92236d8..1a5124d 100644 --- a/src/K4os.Xpovoc.Abstractions/IJobScheduler.cs +++ b/src/K4os.Xpovoc.Abstractions/IJobScheduler.cs @@ -3,7 +3,7 @@ namespace K4os.Xpovoc.Abstractions; -public interface IJobScheduler: IDisposable, IDateTimeSource +public interface IJobScheduler: IDisposable { Task Schedule(DateTimeOffset time, object payload); } \ No newline at end of file diff --git a/src/K4os.Xpovoc.Abstractions/ITimeSource.cs b/src/K4os.Xpovoc.Abstractions/ITimeSource.cs new file mode 100644 index 0000000..42d4c83 --- /dev/null +++ b/src/K4os.Xpovoc.Abstractions/ITimeSource.cs @@ -0,0 +1,11 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace K4os.Xpovoc.Abstractions; + +public interface ITimeSource +{ + DateTimeOffset Now { get; } + Task Delay(TimeSpan delay, CancellationToken token); +} \ No newline at end of file diff --git a/src/K4os.Xpovoc.Abstractions/K4os.Xpovoc.Abstractions.csproj b/src/K4os.Xpovoc.Abstractions/K4os.Xpovoc.Abstractions.csproj index 7a3a252..f837fda 100644 --- a/src/K4os.Xpovoc.Abstractions/K4os.Xpovoc.Abstractions.csproj +++ b/src/K4os.Xpovoc.Abstractions/K4os.Xpovoc.Abstractions.csproj @@ -1,6 +1,8 @@ + - netstandard2.0;net462;net5.0 + netstandard2.0 true + \ No newline at end of file diff --git a/src/K4os.Xpovoc.Abstractions/SystemTimeSource.cs b/src/K4os.Xpovoc.Abstractions/SystemTimeSource.cs index 42521ce..e63a5c0 100644 --- a/src/K4os.Xpovoc.Abstractions/SystemTimeSource.cs +++ b/src/K4os.Xpovoc.Abstractions/SystemTimeSource.cs @@ -1,10 +1,13 @@ using System; +using System.Threading; +using System.Threading.Tasks; namespace K4os.Xpovoc.Abstractions; -public class SystemDateTimeSource: IDateTimeSource +public class SystemTimeSource: ITimeSource { - public static readonly IDateTimeSource Default = new SystemDateTimeSource(); + public static readonly ITimeSource Default = new SystemTimeSource(); public DateTimeOffset Now => DateTimeOffset.UtcNow; - private SystemDateTimeSource() { } + public Task Delay(TimeSpan delay, CancellationToken token) => Task.Delay(delay, token); + private SystemTimeSource() { } } \ No newline at end of file diff --git a/src/K4os.Xpovoc.Brighter/K4os.Xpovoc.Brighter.csproj b/src/K4os.Xpovoc.Brighter/K4os.Xpovoc.Brighter.csproj index 8e5ea46..5d10ccf 100644 --- a/src/K4os.Xpovoc.Brighter/K4os.Xpovoc.Brighter.csproj +++ b/src/K4os.Xpovoc.Brighter/K4os.Xpovoc.Brighter.csproj @@ -1,15 +1,18 @@ - + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 $(PackageTags) adapter brighter true + - - + + + - + + \ No newline at end of file diff --git a/src/K4os.Xpovoc.Core/Db/DbAgent.cs b/src/K4os.Xpovoc.Core/Db/DbAgent.cs index 4e4900f..bf7fc22 100644 --- a/src/K4os.Xpovoc.Core/Db/DbAgent.cs +++ b/src/K4os.Xpovoc.Core/Db/DbAgent.cs @@ -14,21 +14,21 @@ internal abstract class DbAgent protected readonly ILogger Log; - protected DateTime Now => _dateTimeSource.Now.UtcDateTime; + protected DateTime Now => _timeSource.Now.UtcDateTime; protected readonly IDbJobStorage JobStorage; protected readonly ISchedulerConfig Configuration; - private readonly IDateTimeSource _dateTimeSource; + private readonly ITimeSource _timeSource; private int _started; protected DbAgent( ILoggerFactory? loggerFactory, - IDateTimeSource dateTimeSource, + ITimeSource timeSource, IDbJobStorage storage, ISchedulerConfig config) { Log = (loggerFactory ?? NullLoggerFactory.Instance).CreateLogger(ObjectId); - _dateTimeSource = dateTimeSource; + _timeSource = timeSource; Configuration = config; JobStorage = storage; } diff --git a/src/K4os.Xpovoc.Core/Db/DbCleaner.cs b/src/K4os.Xpovoc.Core/Db/DbCleaner.cs index c71577d..3974008 100644 --- a/src/K4os.Xpovoc.Core/Db/DbCleaner.cs +++ b/src/K4os.Xpovoc.Core/Db/DbCleaner.cs @@ -13,10 +13,10 @@ internal class DbCleaner: DbAgent public DbCleaner( ILoggerFactory loggerFactory, - IDateTimeSource dateTimeSource, + ITimeSource timeSource, IDbJobStorage storage, ISchedulerConfig config): - base(loggerFactory, dateTimeSource, storage, config) + base(loggerFactory, timeSource, storage, config) { PruneInterval = config.PruneInterval.NotLessThan(ShortInterval); } diff --git a/src/K4os.Xpovoc.Core/Db/DbJobScheduler.cs b/src/K4os.Xpovoc.Core/Db/DbJobScheduler.cs index 9af3021..94a32a6 100644 --- a/src/K4os.Xpovoc.Core/Db/DbJobScheduler.cs +++ b/src/K4os.Xpovoc.Core/Db/DbJobScheduler.cs @@ -11,7 +11,7 @@ namespace K4os.Xpovoc.Core.Db; public class DbJobScheduler: IJobScheduler { private readonly ILoggerFactory _loggerFactory; - private readonly IDateTimeSource _dateTimeSource; + private readonly ITimeSource _timeSource; private readonly IDbJobStorage _jobStorage; private readonly IJobHandler _jobHandler; private readonly Task[] _pollers; @@ -28,14 +28,14 @@ public DbJobScheduler( IDbJobStorage jobStorage, IJobHandler jobHandler, ISchedulerConfig? configuration = null, - IDateTimeSource? dateTimeSource = null) + ITimeSource? dateTimeSource = null) { _loggerFactory = loggerFactory ?? NullLoggerFactory.Instance; Log = _loggerFactory.CreateLogger(GetType()); _jobStorage = jobStorage.Required(nameof(jobStorage)); _jobHandler = jobHandler.Required(nameof(jobHandler)); - _dateTimeSource = dateTimeSource ?? SystemDateTimeSource.Default; + _timeSource = dateTimeSource ?? SystemTimeSource.Default; _cancel = new CancellationTokenSource(); _ready = new TaskCompletionSource(); @@ -71,7 +71,7 @@ private Task Poll() { var poller = new DbPoller( _loggerFactory, - _dateTimeSource, + _timeSource, _jobStorage, _jobHandler, _configuration); return poller.Start(_cancel.Token, _ready.Task); @@ -80,11 +80,11 @@ private Task Poll() private Task Cleanup() { var cleaner = new DbCleaner( - _loggerFactory, _dateTimeSource, _jobStorage, _configuration); + _loggerFactory, _timeSource, _jobStorage, _configuration); return cleaner.Start(_cancel.Token, _ready.Task); } - public DateTimeOffset Now => _dateTimeSource.Now; + public DateTimeOffset Now => _timeSource.Now; public async Task Schedule(DateTimeOffset time, object payload) { diff --git a/src/K4os.Xpovoc.Core/Db/DbPoller.cs b/src/K4os.Xpovoc.Core/Db/DbPoller.cs index 7a04098..2088a0f 100644 --- a/src/K4os.Xpovoc.Core/Db/DbPoller.cs +++ b/src/K4os.Xpovoc.Core/Db/DbPoller.cs @@ -14,11 +14,11 @@ internal class DbPoller: DbAgent public DbPoller( ILoggerFactory loggerFactory, - IDateTimeSource dateTimeSource, + ITimeSource timeSource, IDbJobStorage storage, IJobHandler handler, ISchedulerConfig config): - base(loggerFactory, dateTimeSource, storage, config) + base(loggerFactory, timeSource, storage, config) { _workerId = Guid.NewGuid(); _jobHandler = handler.Required(nameof(handler)); diff --git a/src/K4os.Xpovoc.Core/K4os.Xpovoc.Core.csproj b/src/K4os.Xpovoc.Core/K4os.Xpovoc.Core.csproj index 46d641a..14df50a 100644 --- a/src/K4os.Xpovoc.Core/K4os.Xpovoc.Core.csproj +++ b/src/K4os.Xpovoc.Core/K4os.Xpovoc.Core.csproj @@ -1,19 +1,24 @@ + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 true + Extensions.cs + - + + - - - + + + + \ No newline at end of file diff --git a/src/K4os.Xpovoc.Core/Queue/QueueJobScheduler.cs b/src/K4os.Xpovoc.Core/Queue/QueueJobScheduler.cs index 1538e02..7f1af30 100644 --- a/src/K4os.Xpovoc.Core/Queue/QueueJobScheduler.cs +++ b/src/K4os.Xpovoc.Core/Queue/QueueJobScheduler.cs @@ -11,7 +11,7 @@ public class QueueJobScheduler: IJobScheduler { public ILogger Log { get; } - private readonly IDateTimeSource _dateTimeSource; + private readonly ITimeSource _timeSource; private readonly IJobQueueAdapter _jobQueueAdapter; private readonly IJobHandler _jobHandler; private readonly IDisposable _jobQueueSubscription; @@ -20,10 +20,10 @@ public QueueJobScheduler( ILoggerFactory? loggerFactory, IJobQueueAdapter jobStorage, IJobHandler jobHandler, - IDateTimeSource? dateTimeSource = null) + ITimeSource? dateTimeSource = null) { Log = loggerFactory?.CreateLogger(GetType()) ?? NullLogger.Instance; - _dateTimeSource = dateTimeSource ?? SystemDateTimeSource.Default; + _timeSource = dateTimeSource ?? SystemTimeSource.Default; _jobHandler = jobHandler.Required(nameof(jobHandler)); _jobQueueAdapter = jobStorage.Required(nameof(jobStorage)); _jobQueueSubscription = _jobQueueAdapter.Subscribe(TryHandle); @@ -59,7 +59,7 @@ private async Task Handle(CancellationToken token, IJob job) } } - public DateTimeOffset Now => _dateTimeSource.Now; + public DateTimeOffset Now => _timeSource.Now; protected class JobEnvelope: IJob { diff --git a/src/K4os.Xpovoc.Db.Test/K4os.Xpovoc.Db.Test.csproj b/src/K4os.Xpovoc.Db.Test/K4os.Xpovoc.Db.Test.csproj index 7beee45..e87b767 100644 --- a/src/K4os.Xpovoc.Db.Test/K4os.Xpovoc.Db.Test.csproj +++ b/src/K4os.Xpovoc.Db.Test/K4os.Xpovoc.Db.Test.csproj @@ -1,8 +1,10 @@ + net6.0 false + Extensions.cs @@ -11,17 +13,21 @@ Secrets.cs + - - - - - - + + + + + + + - - - + + + + + \ No newline at end of file diff --git a/src/K4os.Xpovoc.Hosting/K4os.Xpovoc.Hosting.csproj b/src/K4os.Xpovoc.Hosting/K4os.Xpovoc.Hosting.csproj index 74b0ea8..d437cb5 100644 --- a/src/K4os.Xpovoc.Hosting/K4os.Xpovoc.Hosting.csproj +++ b/src/K4os.Xpovoc.Hosting/K4os.Xpovoc.Hosting.csproj @@ -1,13 +1,17 @@ + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 true $(PackageTags) hosting + - + + - + + diff --git a/src/K4os.Xpovoc.Json/K4os.Xpovoc.Json.csproj b/src/K4os.Xpovoc.Json/K4os.Xpovoc.Json.csproj index 52020a4..ddc28f6 100644 --- a/src/K4os.Xpovoc.Json/K4os.Xpovoc.Json.csproj +++ b/src/K4os.Xpovoc.Json/K4os.Xpovoc.Json.csproj @@ -1,13 +1,17 @@ + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 true $(PackageTags) serialization json + - + + - + + \ No newline at end of file diff --git a/src/K4os.Xpovoc.MediatR/K4os.Xpovoc.MediatR.csproj b/src/K4os.Xpovoc.MediatR/K4os.Xpovoc.MediatR.csproj index 6abca17..dcefcad 100644 --- a/src/K4os.Xpovoc.MediatR/K4os.Xpovoc.MediatR.csproj +++ b/src/K4os.Xpovoc.MediatR/K4os.Xpovoc.MediatR.csproj @@ -1,15 +1,18 @@ - + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 $(PackageTags) adapter mediatr true + - - + + + - + + \ No newline at end of file diff --git a/src/K4os.Xpovoc.Mongo/K4os.Xpovoc.Mongo.csproj b/src/K4os.Xpovoc.Mongo/K4os.Xpovoc.Mongo.csproj index 07a40dd..3783366 100644 --- a/src/K4os.Xpovoc.Mongo/K4os.Xpovoc.Mongo.csproj +++ b/src/K4os.Xpovoc.Mongo/K4os.Xpovoc.Mongo.csproj @@ -1,19 +1,24 @@ + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 xpovoc scheduler cqrs mongo true + Extensions.cs + - - + + + - + + \ No newline at end of file diff --git a/src/K4os.Xpovoc.MsSql/K4os.Xpovoc.MsSql.csproj b/src/K4os.Xpovoc.MsSql/K4os.Xpovoc.MsSql.csproj index 7fec2b4..6697484 100644 --- a/src/K4os.Xpovoc.MsSql/K4os.Xpovoc.MsSql.csproj +++ b/src/K4os.Xpovoc.MsSql/K4os.Xpovoc.MsSql.csproj @@ -1,25 +1,31 @@ + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 xpovoc scheduler cqrs mssql true + - - + + + - - - - + + + + + - + + Extensions.cs + \ No newline at end of file diff --git a/src/K4os.Xpovoc.MySql/K4os.Xpovoc.MySql.csproj b/src/K4os.Xpovoc.MySql/K4os.Xpovoc.MySql.csproj index 85f9da8..d3f3faa 100644 --- a/src/K4os.Xpovoc.MySql/K4os.Xpovoc.MySql.csproj +++ b/src/K4os.Xpovoc.MySql/K4os.Xpovoc.MySql.csproj @@ -1,25 +1,29 @@ + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 xpovoc scheduler cqrs mysql true + Extensions.cs + + - + diff --git a/src/K4os.Xpovoc.PgSql/K4os.Xpovoc.PgSql.csproj b/src/K4os.Xpovoc.PgSql/K4os.Xpovoc.PgSql.csproj index 4351a1e..0be752d 100644 --- a/src/K4os.Xpovoc.PgSql/K4os.Xpovoc.PgSql.csproj +++ b/src/K4os.Xpovoc.PgSql/K4os.Xpovoc.PgSql.csproj @@ -1,24 +1,29 @@ + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 xpovoc scheduler cqrs postgres true + Extensions.cs + + + diff --git a/src/K4os.Xpovoc.Quarterback/K4os.Xpovoc.Quarterback.csproj b/src/K4os.Xpovoc.Quarterback/K4os.Xpovoc.Quarterback.csproj index 48d8f01..9b174cb 100644 --- a/src/K4os.Xpovoc.Quarterback/K4os.Xpovoc.Quarterback.csproj +++ b/src/K4os.Xpovoc.Quarterback/K4os.Xpovoc.Quarterback.csproj @@ -1,15 +1,17 @@ - + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 $(PackageTags) adapter quarterback true + - - + + + - + \ No newline at end of file diff --git a/src/K4os.Xpovoc.Redis/K4os.Xpovoc.Redis.csproj b/src/K4os.Xpovoc.Redis/K4os.Xpovoc.Redis.csproj index b3362b1..d4cd563 100644 --- a/src/K4os.Xpovoc.Redis/K4os.Xpovoc.Redis.csproj +++ b/src/K4os.Xpovoc.Redis/K4os.Xpovoc.Redis.csproj @@ -1,22 +1,28 @@ + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 xpovoc scheduler cqrs redis true + Extensions.cs + + + - + + diff --git a/src/K4os.Xpovoc.SqLite/K4os.Xpovoc.SqLite.csproj b/src/K4os.Xpovoc.SqLite/K4os.Xpovoc.SqLite.csproj index d29c2b2..984e1df 100644 --- a/src/K4os.Xpovoc.SqLite/K4os.Xpovoc.SqLite.csproj +++ b/src/K4os.Xpovoc.SqLite/K4os.Xpovoc.SqLite.csproj @@ -1,25 +1,31 @@ + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 $(PackageTags) sqlite true + Extensions.cs + + + - + + \ No newline at end of file diff --git a/src/K4os.Xpovoc.Sqs.Test/K4os.Xpovoc.Sqs.Test.csproj b/src/K4os.Xpovoc.Sqs.Test/K4os.Xpovoc.Sqs.Test.csproj index f845af6..e59ef92 100644 --- a/src/K4os.Xpovoc.Sqs.Test/K4os.Xpovoc.Sqs.Test.csproj +++ b/src/K4os.Xpovoc.Sqs.Test/K4os.Xpovoc.Sqs.Test.csproj @@ -7,18 +7,16 @@ false + + + + - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - + + + + diff --git a/src/K4os.Xpovoc.Sqs.Test/Move10KUsingRawSqsAccess.cs b/src/K4os.Xpovoc.Sqs.Test/Move10KUsingRawSqsAccess.cs new file mode 100644 index 0000000..4c444af --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/Move10KUsingRawSqsAccess.cs @@ -0,0 +1,104 @@ +using System.Diagnostics; +using Amazon.SQS; +using Amazon.SQS.Model; +using K4os.Xpovoc.Sqs.Internal; +using Xunit.Abstractions; + +namespace K4os.Xpovoc.Sqs.Test; + +public class Move10KUsingRawSqsAccess +{ + private const int SampleSize = 10_000; + + private readonly ITestOutputHelper _output; + + public Move10KUsingRawSqsAccess(ITestOutputHelper output) + { + _output = output; + } + + [Fact] + public async Task NoMessagesAreLost() + { + var factory = new SqsQueueFactory(new AmazonSQSClient()); + var settings = new SqsQueueSettings(); + var queue = await factory.Create("mk-test-move10k", settings); + + var guids = Enumerable.Range(0, SampleSize).Select(_ => Guid.NewGuid()).ToList(); + var stopwatch = Stopwatch.StartNew(); + + var push = SendAll(queue, guids, stopwatch); + var pull = PullAll(queue, guids, stopwatch); + + await Task.WhenAll(push, pull); + } + + private async Task SendAll(ISqsQueue queue, ICollection guids, Stopwatch stopwatch) + { + var semaphore = new SemaphoreSlim(10); + var batches = guids + .Select(g => g.ToString()) + .Select(g => new SendMessageBatchRequestEntry(g, g)) + .Chunk(10) + .Select(b => b.ToList()); + + await Task.WhenAll(batches.Select(b => SendBatch(queue, semaphore, b))); + + var rate = guids.Count / stopwatch.Elapsed.TotalSeconds; + _output.WriteLine($"All messages sent {rate:0.0}/s"); + } + + private static async Task SendBatch( + ISqsQueue queue, SemaphoreSlim semaphore, List batch) + { + await semaphore.WaitAsync(); + try + { + await queue.Send(batch, CancellationToken.None); + } + finally + { + semaphore.Release(); + } + } + + private async Task PullAll(ISqsQueue queue, ICollection guids, Stopwatch stopwatch) + { + var left = guids.ToHashSet(); + + while (left.Count > 0) + { + var messages = await queue.Receive(CancellationToken.None); + if (messages.Count == 0) + { + _output.WriteLine("No messages received"); + continue; + } + + foreach (var message in messages) + { + var body = message.Body; + + if (!Guid.TryParse(body, out var guid)) + { + _output.WriteLine($"Unexpected body: {body}"); + continue; + } + + if (!left.Remove(guid)) + { + _output.WriteLine($"Unexpected guid: {guid}"); + } + } + + var receipts = messages + .Select(m => new DeleteMessageBatchRequestEntry(m.MessageId, m.ReceiptHandle)) + .ToList(); + + await queue.Delete(receipts, CancellationToken.None); + } + + var rate = guids.Count / stopwatch.Elapsed.TotalSeconds; + _output.WriteLine($"All messages received {rate:0.0}/s"); + } +} diff --git a/src/K4os.Xpovoc.Sqs.Test/Move10KUsingSqsAdapter.cs b/src/K4os.Xpovoc.Sqs.Test/Move10KUsingSqsAdapter.cs new file mode 100644 index 0000000..12a84a9 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/Move10KUsingSqsAdapter.cs @@ -0,0 +1,107 @@ +using System.Diagnostics; +using Amazon.SQS; +using K4os.Xpovoc.Abstractions; +using K4os.Xpovoc.Core.Queue; +using K4os.Xpovoc.Core.Sql; +using K4os.Xpovoc.Sqs.Internal; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit.Abstractions; + +namespace K4os.Xpovoc.Sqs.Test; + +public class Move10KUsingSqsAdapter +{ + private const int SampleSize = 100_000; + private readonly ITestOutputHelper _output; + + public Move10KUsingSqsAdapter(ITestOutputHelper output) + { + _output = output; + } + + [Theory] + [InlineData(16, 128)] + [InlineData(32, 4)] + [InlineData(16, 4)] + [InlineData(8, 4)] + [InlineData(1, 4)] + public async Task NoMessagesAreLost(int sqsConcurrency, int jobConcurrency) + { + var loggerFactory = NullLoggerFactory.Instance; + // var loggerFactory = new TestLoggerFactory(_output); + var queueFactory = new SqsQueueFactory(new AmazonSQSClient()); + var adapter = new SqsJobQueueAdapter( + loggerFactory, + queueFactory, + new DefaultJobSerializer(), + new SqsJobQueueAdapterConfig { + QueueName = "mk-test-move10k-adapter", + SqsConcurrency = sqsConcurrency, + JobConcurrency = jobConcurrency, + }); + + var guids = Enumerable.Range(0, SampleSize).Select(_ => Guid.NewGuid()).ToList(); + var stopwatch = Stopwatch.StartNew(); + + var push = SendAll(adapter, guids, stopwatch); + var pull = PullAll(adapter, guids, stopwatch); + + await Task.WhenAll(push, pull); + + adapter.Dispose(); + } + + private async Task SendAll(IJobQueueAdapter queue, ICollection guids, Stopwatch stopwatch) + { + var publishTasks = guids + .Select(g => new FakeJob(g)) + .Select(j => queue.Publish(TimeSpan.Zero, j, CancellationToken.None)); + await Task.WhenAll(publishTasks); + + var rate = guids.Count / stopwatch.Elapsed.TotalSeconds; + _output.WriteLine($"All messages sent {rate:0.0}/s"); + } + + private async Task PullAll(IJobQueueAdapter queue, ICollection guids, Stopwatch stopwatch) + { + var left = guids.ToHashSet(); + var counter = left.Count; + var done = new TaskCompletionSource(); + + Task HandleOne(IJob job) + { + lock (left) + { + left.Remove((Guid) job.Payload!); + if (--counter <= 0) done.TrySetResult(true); + } + + return Task.CompletedTask; + } + + var subscription = queue.Subscribe((_, j) => HandleOne(j)); + + await done.Task; + + subscription.Dispose(); + + var rate = guids.Count / stopwatch.Elapsed.TotalSeconds; + _output.WriteLine($"All messages received {rate:0.0}/s"); + } +} + +internal class FakeJob: IJob +{ + public Guid JobId { get; } + public DateTime UtcTime { get; } + public object? Payload { get; } + public object? Context { get; } + + public FakeJob(Guid guid) + { + JobId = guid; + UtcTime = DateTime.UtcNow; + Payload = guid; + Context = null; + } +} diff --git a/src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs b/src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs index c18f0d3..649e6df 100644 --- a/src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs +++ b/src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs @@ -22,21 +22,21 @@ public class SqsJobQueueAdapterTests private static readonly SqsJobQueueAdapterConfig DefaultAdapterConfig = new() { QueueName = "mk-SqsJobQueueAdapter-tests", - Concurrency = 1, + JobConcurrency = 1, }; private static readonly TestConfig FastRoundtripConfig = new() { QueueName = "mk-SqsJobQueueAdapter-5s-tests", - Concurrency = 1, - SqsQueueSettings = new SqsQueueSettings { + JobConcurrency = 1, + QueueSettings = new SqsQueueSettings { VisibilityTimeout = TimeSpan.FromSeconds(5), }, }; private static readonly TestConfig BatchingConfig = new() { QueueName = "mk-SqsJobQueueAdapter-10s-tests", - Concurrency = 16, - SqsQueueSettings = new SqsQueueSettings { + JobConcurrency = 16, + QueueSettings = new SqsQueueSettings { VisibilityTimeout = TimeSpan.FromSeconds(10), }, }; diff --git a/src/K4os.Xpovoc.Sqs.Test/TestConfig.cs b/src/K4os.Xpovoc.Sqs.Test/TestConfig.cs deleted file mode 100644 index a43c934..0000000 --- a/src/K4os.Xpovoc.Sqs.Test/TestConfig.cs +++ /dev/null @@ -1,10 +0,0 @@ -using K4os.Xpovoc.Sqs.Internal; - -namespace K4os.Xpovoc.Sqs.Test; - -public class TestConfig: ISqsJobQueueAdapterConfig -{ - public string QueueName { get; set; } = null!; - public int Concurrency { get; set; } - public SqsQueueSettings? SqsQueueSettings { get; set; } -} diff --git a/src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueue.cs b/src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueue.cs new file mode 100644 index 0000000..d4f6cdd --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueue.cs @@ -0,0 +1,230 @@ +using System.Net; +using System.Reactive.Concurrency; +using Amazon.SQS; +using Amazon.SQS.Model; +using K4os.Xpovoc.Sqs.Internal; + +namespace K4os.Xpovoc.Sqs.Test.Utilities; + +internal class TestSqsQueue: ISqsQueue +{ + private readonly object _sync = new(); + private readonly IScheduler _scheduler; + private readonly TimeSpan _visibility; + private readonly Queue _messages = new(); + private readonly SemaphoreSlim _messagesLock = new(0, int.MaxValue); + private readonly Dictionary _receipts = new(); + + public TestSqsQueue( + ISqsQueueSettings settings, + IScheduler scheduler) + { + _scheduler = scheduler; + _visibility = settings.VisibilityTimeout ?? SqsConstants.DefaultVisibilityTimeout; + } + + public int InQueue + { + get + { + lock (_sync) return _messages.Count; + } + } + + public int InFlight + { + get + { + lock (_sync) return _receipts.Count; + } + } + + public Task GetAttributes(CancellationToken token = default) + { + var visibilityTimeout = (int)Math.Ceiling(_visibility.TotalSeconds); + + var attributes = new GetQueueAttributesResponse { + Attributes = { + [QueueAttributeName.VisibilityTimeout] = visibilityTimeout.ToString(), + // maybe more later + }, + HttpStatusCode = HttpStatusCode.OK, + ContentLength = 1337, + }; + + return Task.FromResult(attributes); + } + + public Task>> Send( + List entries, CancellationToken token) + { + var now = _scheduler.Now; + var results = new List>(); + + foreach (var entry in entries) + { + var messageId = Guid.NewGuid().ToString("D"); + var delay = entry.DelaySeconds.NotLessThan(0); + results.Add(ToResultEntry(messageId, entry)); + + if (delay > 0) + { + _scheduler.Schedule(now.AddSeconds(delay), () => EnqueueOne(messageId, entry)); + } + else + { + EnqueueOne(messageId, entry); + } + } + + return Task.FromResult(results); + } + + public async Task> Receive(CancellationToken token) + { + await _messagesLock.WaitAsync(token); + + var result = new List(); + while (result.Count < SqsConstants.MaximumReceiveCount) + { + var message = DequeueOne(); + if (message is null) break; + + result.Add(message); + } + + return result; + } + + public Task>> Delete( + List entries, CancellationToken token) + { + var result = new List>(); + foreach (var entry in entries) + { + DeleteOne(entry.ReceiptHandle); + result.Add(ToResultEntry(entry)); + } + + return Task.FromResult(result); + } + + private void DeleteOne(string receiptId) + { + lock (_sync) + { + _receipts.Remove(receiptId); + } + } + + public Task>> Touch( + List entries, CancellationToken token) + { + var result = new List>(); + foreach (var entry in entries) + { + TouchOne(entry.ReceiptHandle, entry.VisibilityTimeout); + result.Add(ToResultEntry(entry)); + } + + return Task.FromResult(result); + } + + private void TouchOne(string receiptId, int visibilityTimeout) + { + lock (_sync) + { + if (!_receipts.TryGetValue(receiptId, out var message)) return; + + var invisibleUntil = _scheduler.Now.AddSeconds(visibilityTimeout); + message.InvisibleUntil = invisibleUntil; + _scheduler.Schedule(invisibleUntil, () => TryRequeueOne(receiptId)); + } + } + + private void EnqueueOne(string messageId, SendMessageBatchRequestEntry entry) + { + lock (_sync) + { + _messages.Enqueue(ToMessage(messageId, entry)); + _messagesLock.Release(); + } + } + + private void TryRequeueOne(string receiptId) + { + lock (_sync) + { + if (!_receipts.TryGetValue(receiptId, out var message)) return; + + var now = _scheduler.Now; + if (message.InvisibleUntil > now) return; + + message.InvisibleUntil = null; + _receipts.Remove(receiptId); + _messages.Enqueue(message); + _messagesLock.Release(); + } + } + + private Message? DequeueOne() + { + var now = _scheduler.Now; + + lock (_sync) + { + if (!_messages.TryDequeue(out var message)) return null; + + if (message.InvisibleUntil.HasValue) + throw new InvalidOperationException("Message is already in flight"); + + var invisibleUntil = now.Add(_visibility); + message.InvisibleUntil = invisibleUntil; + var receiptId = Guid.NewGuid().ToString("D"); + _receipts.Add(receiptId, message); + _scheduler.Schedule(invisibleUntil, () => TryRequeueOne(receiptId)); + + return ToMessage(receiptId, message); + } + } + + private static FakeSqsMessage ToMessage(string messageId, SendMessageBatchRequestEntry entry) => + new() { + Id = messageId, + Body = entry.MessageBody, + Attributes = entry.MessageAttributes.ToDictionary(x => x.Key, x => x.Value.StringValue), + }; + + private static Message ToMessage(string receiptId, FakeSqsMessage entry) + { + return new Message { + MessageId = entry.Id, + ReceiptHandle = receiptId, + Body = entry.Body, + MessageAttributes = entry.Attributes.ToDictionary( + x => x.Key, x => new MessageAttributeValue { + DataType = "String", StringValue = x.Value, + }), + }; + } + + private static SqsResult ToResultEntry( + string messageId, SendMessageBatchRequestEntry entry) => + new(new SendMessageBatchResultEntry { Id = entry.Id, MessageId = messageId }); + + private static SqsResult ToResultEntry( + DeleteMessageBatchRequestEntry entry) => + new(new DeleteMessageBatchResultEntry { Id = entry.Id }); + + private static SqsResult ToResultEntry( + ChangeMessageVisibilityBatchRequestEntry entry) => + new(new ChangeMessageVisibilityBatchResultEntry { Id = entry.Id }); +} + +internal class FakeSqsMessage +{ + public string Id { get; set; } = null!; + public string Body { get; set; } = null!; + public Dictionary Attributes { get; set; } = null!; + public DateTimeOffset? InvisibleUntil { get; set; } +} diff --git a/src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueueFactory.cs b/src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueueFactory.cs new file mode 100644 index 0000000..6ef8689 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueueFactory.cs @@ -0,0 +1,36 @@ +using System.Reactive.Concurrency; +using K4os.Xpovoc.Sqs.Internal; + +namespace K4os.Xpovoc.Sqs.Test.Utilities; + +internal class TestSqsQueueFactory: ISqsQueueFactory +{ + private readonly Dictionary _queues = new(); + private readonly IScheduler _scheduler; + + public TestSqsQueueFactory(IScheduler scheduler) + { + _scheduler = scheduler; + } + + public TestSqsQueue Create(string queueName, ISqsQueueSettings settings) + { + lock (_queues) + return TryGetQueue(queueName) ?? (_queues[queueName] = CreateNewQueue(settings)); + } + + public TestSqsQueue? Find(string queueName) + { + lock (_queues) + return TryGetQueue(queueName); + } + + Task ISqsQueueFactory.Create(string queueName, ISqsQueueSettings settings) => + Task.FromResult(Create(queueName, settings)); + + private TestSqsQueue CreateNewQueue(ISqsQueueSettings settings) => + new(settings, _scheduler); + + private TestSqsQueue? TryGetQueue(string queueName) => + _queues.TryGetValue(queueName, out var queue) ? queue : null; +} \ No newline at end of file diff --git a/src/K4os.Xpovoc.Sqs.Test/Utilities/TestTimeSource.cs b/src/K4os.Xpovoc.Sqs.Test/Utilities/TestTimeSource.cs new file mode 100644 index 0000000..1551f20 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/Utilities/TestTimeSource.cs @@ -0,0 +1,33 @@ +using System.Reactive.Concurrency; +using System.Reactive.Disposables; +using K4os.Xpovoc.Abstractions; + +namespace K4os.Xpovoc.Sqs.Test.Utilities; + +public class TestTimeSource: ITimeSource +{ + private readonly IScheduler _scheduler; + + public TestTimeSource(IScheduler scheduler) { _scheduler = scheduler; } + + public DateTimeOffset Now => _scheduler.Now; + + public Task Delay(TimeSpan delay, CancellationToken token) + { + var tcs = new TaskCompletionSource(); + var disposables = new CompositeDisposable(); + + void Done() + { + var first = token.IsCancellationRequested + ? tcs.TrySetCanceled(token) + : tcs.TrySetResult(); + if (first) disposables.Dispose(); + } + + disposables.Add(_scheduler.Schedule(Now.Add(delay), Done)); + disposables.Add(token.Register(Done)); + + return tcs.Task; + } +} diff --git a/src/K4os.Xpovoc.Sqs.Test/VeryLongRun.cs b/src/K4os.Xpovoc.Sqs.Test/VeryLongRun.cs new file mode 100644 index 0000000..6b0a331 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs.Test/VeryLongRun.cs @@ -0,0 +1,163 @@ +using K4os.Xpovoc.Abstractions; +using K4os.Xpovoc.Core.Queue; +using K4os.Xpovoc.Core.Sql; +using K4os.Xpovoc.Sqs.Internal; +using K4os.Xpovoc.Sqs.Test.Utilities; +using Microsoft.Extensions.Logging; +using Microsoft.Reactive.Testing; +using Xunit.Abstractions; + +namespace K4os.Xpovoc.Sqs.Test; + +public class VeryLongRun: IJobHandler +{ + protected readonly ILogger Log; + + private static readonly DateTimeOffset Time0 = + new DateTime(2000, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + + private readonly TestScheduler _scheduler; + private readonly QueueJobScheduler _jobScheduler; + private readonly TestSqsQueue _testQueue; + private readonly List<(DateTimeOffset, Guid)> _jobsHandled = new(); + + public Func? HijackHandler = null; + + public VeryLongRun(ITestOutputHelper output) + { + var loggerFactory = new TestLoggerFactory(output); + _scheduler = new TestScheduler(); + AdvanceTo(Time0); + var queueName = "test-queue"; + var queueFactory = new TestSqsQueueFactory(_scheduler); + _testQueue = queueFactory.Create(queueName, new SqsQueueSettings()); + var testTimeSource = new TestTimeSource(_scheduler); + + var adapter = new SqsJobQueueAdapter( + loggerFactory, + queueFactory, + new DefaultJobSerializer(), + new SqsJobQueueAdapterConfig { QueueName = queueName, JobConcurrency = 1 }, + testTimeSource); + _jobScheduler = new QueueJobScheduler( + loggerFactory, adapter, this, testTimeSource); + + Log = loggerFactory.CreateLogger("Test"); + } + + ~VeryLongRun() { _jobScheduler.Dispose(); } + + private void AdvanceTo(DateTimeOffset time) { _scheduler.AdvanceTo(time.Ticks); } + private void AdvanceTo(TimeSpan span) { AdvanceTo(Time0.Add(span)); } + private void AdvanceBy(TimeSpan span) { AdvanceTo(Now.Add(span)); } + + private static async Task WaitUntil( + Func condition, + int milliseconds = 1000) + { + if (milliseconds <= 0) + { + await Task.Yield(); + return condition(); + } + + if (condition()) return true; + + using var token = new CancellationTokenSource(milliseconds); + + while (true) + { + try + { + await Task.Delay(10, token.Token); + } + catch (OperationCanceledException) + { + return false; + } + + if (condition()) return true; + } + } + + private DateTimeOffset Now => _scheduler.Now; + + private IEnumerable<(DateTimeOffset At, Guid Id)> HandledJob + { + get + { + lock (_jobsHandled) return _jobsHandled.ToArray(); + } + } + + private int HandledCount + { + get + { + lock (_jobsHandled) return _jobsHandled.Count; + } + } + + [Fact] + public async Task MessagesScheduledInThePastAreHandledImmediately() + { + var guid = Guid.NewGuid(); + await _jobScheduler.Schedule(Now.AddSeconds(-1), guid); + await WaitUntil(() => HandledCount > 0); + Assert.Equal(1, HandledCount); + Assert.True(HandledJob.Any(x => x.Id == guid)); + } + + [Fact] + public async Task MessagesScheduledForNowAreHandledImmediately() + { + var guid = Guid.NewGuid(); + await _jobScheduler.Schedule(Now, guid); + await WaitUntil(() => HandledCount > 0); + Assert.Equal(1, HandledCount); + Assert.True(HandledJob.Any(x => x.Id == guid)); + } + + [Fact] + public async Task MessagesScheduledInTheFutureDoNoGetHandledPrematurely() + { + var guid = Guid.NewGuid(); + var when = Now.AddDays(1); + await _jobScheduler.Schedule(when, guid); + Assert.False(await WaitUntil(() => HandledCount > 0)); + + while (true) + { + var next = Now.AddMinutes(7); + if (next >= when) break; + + AdvanceTo(next); + // every 7 minutes we make sure job is still not handled + // NOTE: we cannot just jump ahead because underlying mechanism + // will assume that message wasn't touched for a long time and + // will put it back to the queue effectively duplicating it + // this is how real SQS works! + // This is not great but it is a price of not using Rx + // for scheduling "Touch" operations + Assert.False(await WaitUntil(() => HandledCount > 0, 100)); + Assert.True(await WaitUntil(() => _testQueue.InQueue + _testQueue.InFlight <= 1)); + } + + Assert.False(await WaitUntil(() => HandledCount > 0)); + AdvanceTo(when); + Assert.True(await WaitUntil(() => _testQueue.InQueue + _testQueue.InFlight == 0)); + Assert.Equal(1, HandledCount); + } + + Task IJobHandler.Handle(CancellationToken token, object payload) + { + if (HijackHandler != null) + return HijackHandler(payload); + + var job = (Guid)payload; + var now = _scheduler.Now; + lock (_jobsHandled) + _jobsHandled.Add((now, job)); + return Task.CompletedTask; + } +} diff --git a/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterConfig.cs b/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterConfig.cs deleted file mode 100644 index 60ecc15..0000000 --- a/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterConfig.cs +++ /dev/null @@ -1,17 +0,0 @@ -using K4os.Xpovoc.Sqs.Internal; - -namespace K4os.Xpovoc.Sqs; - -public interface ISqsJobQueueAdapterConfig -{ - public string QueueName { get; } - int Concurrency { get; set; } - SqsQueueSettings? SqsQueueSettings { get; } -} - -public class SqsJobQueueAdapterConfig: ISqsJobQueueAdapterConfig -{ - public string QueueName { get; set; } = null!; - public int Concurrency { get; set; } - SqsQueueSettings? ISqsJobQueueAdapterConfig.SqsQueueSettings => default; -} diff --git a/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterSettings.cs b/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterSettings.cs new file mode 100644 index 0000000..502ad7c --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterSettings.cs @@ -0,0 +1,26 @@ +using System; +using K4os.Xpovoc.Sqs.Internal; + +namespace K4os.Xpovoc.Sqs; + +public interface ISqsJobQueueAdapterConfig +{ + public string QueueName { get; } + int? PushConcurrency { get; set; } + int? PullConcurrency { get; set; } + int? ExecConcurrency { get; set; } + TimeSpan? RetryInterval { get; set; } + int? RetryCount { get; set; } + ISqsQueueSettings? QueueSettings { get; set; } +} + +public class SqsJobQueueAdapterConfig: ISqsJobQueueAdapterConfig +{ + public string QueueName { get; set; } = null!; + public int? PushConcurrency { get; set; } + public int? PullConcurrency { get; set; } + public int? ExecConcurrency { get; set; } + public TimeSpan? RetryInterval { get; set; } + public int? RetryCount { get; set; } + public ISqsQueueSettings? QueueSettings { get; set; } +} diff --git a/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueFactory.cs b/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueFactory.cs index 2eef38d..a615560 100644 --- a/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueFactory.cs +++ b/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueFactory.cs @@ -4,5 +4,5 @@ namespace K4os.Xpovoc.Sqs.Internal; public interface ISqsQueueFactory { - Task Create(string queueName, SqsQueueSettings settings); + Task Create(string queueName, ISqsQueueSettings settings); } \ No newline at end of file diff --git a/src/K4os.Xpovoc.Sqs/Internal/SqsQueue.cs b/src/K4os.Xpovoc.Sqs/Internal/SqsQueue.cs index 5628392..368ead9 100644 --- a/src/K4os.Xpovoc.Sqs/Internal/SqsQueue.cs +++ b/src/K4os.Xpovoc.Sqs/Internal/SqsQueue.cs @@ -8,18 +8,14 @@ namespace K4os.Xpovoc.Sqs.Internal; -//public TimeSpan RetentionPeriod => TimeSpan.FromSeconds(Attributes.MessageRetentionPeriod); -//public TimeSpan VisibilityTimeout => TimeSpan.FromSeconds(Attributes.VisibilityTimeout); - - -public class SqsQueue: ISqsQueue +internal class SqsQueue: ISqsQueue { private readonly IAmazonSQS _client; private readonly string _queueUrl; private GetQueueAttributesResponse? _attributes; public string Url => _queueUrl; - + public SqsQueue(IAmazonSQS client, string queueUrl) { _client = client; @@ -43,8 +39,7 @@ public async Task>> Send( QueueUrl = _queueUrl, Entries = messages, }, token); - - return ComposeResponse(response.Successful, response.Failed); + return Combine(response.Successful, response.Failed); } public async Task> Receive(CancellationToken token) @@ -56,7 +51,6 @@ public async Task> Receive(CancellationToken token) AttributeNames = SqsAttributes.All, // ApproximateReceiveCount, SentTimestamp MessageAttributeNames = SqsAttributes.All, }, token); - return response.Messages; } @@ -69,8 +63,7 @@ public async Task>> Delete( QueueUrl = _queueUrl, Entries = messages, }, token); - - return ComposeResponse(response.Successful, response.Failed); + return Combine(response.Successful, response.Failed); } public async Task>> Touch( @@ -82,11 +75,10 @@ public async Task>> Touc QueueUrl = _queueUrl, Entries = messages, }, token); - - return ComposeResponse(response.Successful, response.Failed); + return Combine(response.Successful, response.Failed); } - private static List> ComposeResponse( + private static List> Combine( IReadOnlyCollection success, IReadOnlyCollection failure) { @@ -96,4 +88,4 @@ private static List> ComposeResponse( result.AddRange(failure.Select(r => new SqsResult(r))); return result; } -} +} \ No newline at end of file diff --git a/src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs b/src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs index 3714a71..fa19ba7 100644 --- a/src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs +++ b/src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Threading; using System.Threading.Tasks; using Amazon.SQS; using Amazon.SQS.Model; @@ -10,11 +9,13 @@ namespace K4os.Xpovoc.Sqs.Internal; public class SqsQueueFactory: ISqsQueueFactory { + private static readonly SqsQueueSettings DeadLetterSqsQueueSettings = new(); + private readonly IAmazonSQS _client; public SqsQueueFactory(IAmazonSQS client) { _client = client; } - public async Task Create(string queueName, SqsQueueSettings settings) + public async Task Create(string queueName, ISqsQueueSettings settings) { var queueUrl = await FindOrCreateQueue(queueName, settings) ?? @@ -22,7 +23,7 @@ await FindOrCreateQueue(queueName, settings) ?? return new SqsQueue(_client, queueUrl); } - private async Task FindOrCreateQueue(string queueName, SqsQueueSettings settings) + private async Task FindOrCreateQueue(string queueName, ISqsQueueSettings settings) { if (await TryFindQueue(queueName) is { } queueUrl) return queueUrl; @@ -30,7 +31,7 @@ private async Task FindOrCreateQueue(string queueName, SqsQueueSettings var deadLetterName = queueName + "-dlq"; var deadLetterUrl = - await TryCreateQueue(deadLetterName, null, new SqsQueueSettings()) ?? + await TryCreateQueue(deadLetterName, null, DeadLetterSqsQueueSettings) ?? throw new InvalidOperationException( $"Failed to create dead letter queue: {deadLetterName}"); @@ -40,7 +41,7 @@ await TryCreateQueue(queueName, deadLetterUrl, settings) ?? } private async Task TryCreateQueue( - string queueName, string? deadLetterUrl, SqsQueueSettings settings) + string queueName, string? deadLetterUrl, ISqsQueueSettings settings) { var queueSettings = new Dictionary(); diff --git a/src/K4os.Xpovoc.Sqs/K4os.Xpovoc.Sqs.csproj b/src/K4os.Xpovoc.Sqs/K4os.Xpovoc.Sqs.csproj index 3439e12..2077908 100644 --- a/src/K4os.Xpovoc.Sqs/K4os.Xpovoc.Sqs.csproj +++ b/src/K4os.Xpovoc.Sqs/K4os.Xpovoc.Sqs.csproj @@ -1,23 +1,25 @@ + - netstandard2.0;net462;net5.0 + netstandard2.0;net462;net5.0;net6.0 $(PackageTags) sqs - false + true + - + + Extensions.cs + - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + + + + \ No newline at end of file diff --git a/src/K4os.Xpovoc.Sqs/SqsBatchAdapter.cs b/src/K4os.Xpovoc.Sqs/SqsBatchAdapter.cs new file mode 100644 index 0000000..76eb495 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/SqsBatchAdapter.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS.Model; +using K4os.Async.Toys; +using K4os.Xpovoc.Sqs.Internal; +using Microsoft.Extensions.Logging; + +namespace K4os.Xpovoc.Sqs; + +internal class SqsBatchAdapter: IBatchPoller +{ + protected readonly ILogger Log; + + private readonly ISqsQueue _queue; + private readonly TimeSpan _visibility; + + public SqsBatchAdapter(ILogger log, ISqsQueue queue, TimeSpan visibility) + { + Log = log; + _queue = queue; + _visibility = visibility; + } + + public string ReceiptFor(Message message) => message.ReceiptHandle; + public string IdentityOf(string receipt) => receipt; + + public async Task[]> Send( + SendMessageBatchRequestEntry[] messages) + { + var requests = messages.ToList(); + Log.LogDebug("Sending batch of {Count} messages", requests.Count); + var responses = await _queue.Send(requests, CancellationToken.None); + return responses.ToArray(); + } + + public async Task Receive(CancellationToken token) + { + var result = await _queue.Receive(token); + return result.ToArray(); + } + + public async Task Delete(string[] receipts, CancellationToken token) + { + var (map, requests) = BuildRequestMap(receipts, ToDeleteRequest); + Log.LogDebug("Deleting batch of {Count} messages", requests.Count); + var response = await _queue.Delete(requests, token); + return response + .SelectNotNull(r => r.Result?.Id) + .SelectNotNull(id => map.TryGetOrDefault(id)) + .ToArray(); + } + + public async Task Touch(string[] receipts, CancellationToken token) + { + var timeout = (int)_visibility.TotalSeconds; + var (map, requests) = BuildRequestMap(receipts, (id, m) => ToTouchRequest(id, m, timeout)); + Log.LogDebug("Touching batch of {Count} messages", requests.Count); + var response = await _queue.Touch(requests, token); + return response + .SelectNotNull(r => r.Result?.Id) + .SelectNotNull(id => map.TryGetOrDefault(id)) + .ToArray(); + } + + private static DeleteMessageBatchRequestEntry ToDeleteRequest(string id, string receipt) => + new() { Id = id, ReceiptHandle = receipt }; + + private static ChangeMessageVisibilityBatchRequestEntry ToTouchRequest( + string id, string receipt, int timeout) => + new() { Id = id, ReceiptHandle = receipt, VisibilityTimeout = timeout }; + + private static (Dictionary Receipts, List Requests) BuildRequestMap( + ICollection receipts, Func toRequest) + { + var capacity = receipts.Count; + var map = new Dictionary(capacity); + var list = new List(capacity); + var counter = 0; + + foreach (var receipt in receipts) + { + var itemId = (counter++).ToString(); + var request = toRequest(itemId, receipt); + map.Add(itemId, receipt); + list.Add(request); + } + + return (map, list); + } +} diff --git a/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs b/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs index fe5a46b..c20ad2d 100644 --- a/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs +++ b/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; +using System.Reactive.Disposables; using System.Threading; using System.Threading.Tasks; using Amazon.SQS; @@ -12,25 +12,42 @@ using K4os.Xpovoc.Sqs.Internal; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using ITimeAdapter = K4os.Async.Toys.ITimeSource; +using ITimeSource = K4os.Xpovoc.Abstractions.ITimeSource; namespace K4os.Xpovoc.Sqs; +using SqsBatchPublisher = + IBatchBuilder>; +using SqsBatchSubscriber = BatchSubscriber; + public class SqsJobQueueAdapter: IJobQueueAdapter { + private const int DefaultSqsConcurrency = 16; + private const int DefaultJobConcurrency = 4; + + private static readonly TimeSpan DefaultRetryInterval = TimeSpan.FromSeconds(1); + private static readonly TimeSpan MinimumRetryInterval = TimeSpan.Zero; + private const int DefaultRetryCount = 0; + protected ILogger Log { get; } - + + private static readonly SqsQueueSettings DefaultSqsQueueSettings = new(); + private readonly IJobSerializer _serializer; private readonly ISqsJobQueueAdapterConfig _config; private readonly ISqsQueueFactory _factory; private readonly Task _ready; - + private ISqsQueue _queue; private TimeSpan _visibility; - private IBatchBuilder> - _batchSender; + private readonly object _mutex = new(); + private long _messageId; - private IAliveKeeper _aliveKeeper; + private SqsBatchAdapter _adapter; + private SqsBatchPublisher _publisher; + private SqsBatchSubscriber? _subscriber; public SqsJobQueueAdapter( ILoggerFactory? loggerFactory, @@ -43,89 +60,94 @@ public SqsJobQueueAdapter( serializer, config) { } - public SqsJobQueueAdapter( + internal SqsJobQueueAdapter( ILoggerFactory? loggerFactory, ISqsQueueFactory queueFactory, IJobSerializer serializer, - ISqsJobQueueAdapterConfig config) + ISqsJobQueueAdapterConfig config, + ITimeSource? timeSource = null) { - Log = loggerFactory?.CreateLogger(GetType()) ?? NullLogger.Instance; + var log = Log = loggerFactory?.CreateLogger(GetType()) ?? NullLogger.Instance; _factory = queueFactory; _config = config; _serializer = serializer; + _visibility = SqsConstants.DefaultVisibilityTimeout; // might not be true + // those are set in Startup _queue = null!; - _visibility = SqsConstants.DefaultVisibilityTimeout; // might not be true - _batchSender = null!; - _aliveKeeper = null!; + _adapter = null!; + _publisher = null!; - _ready = Task.Run(Startup); + _ready = Task.Run(() => Startup(log, timeSource)); } - private async Task Startup() + private async Task Startup(ILogger log, ITimeSource? timeSource) { - _queue = await CreateQueue(); + var timeAdapter = timeSource is null ? null : new ToysTimeSourceAdapter(timeSource); + + var queue = _queue = await CreateQueue(); var attributes = await _queue.GetAttributes(); - _visibility = TimeSpan.FromSeconds(attributes.VisibilityTimeout); - _batchSender = CreateBatchSender(); - _aliveKeeper = CreateAliveKeeper(); + var visibility = _visibility = TimeSpan.FromSeconds(attributes.VisibilityTimeout); + var adapter = _adapter = new SqsBatchAdapter(log, queue, visibility); + _publisher = CreatePublisher(adapter, timeAdapter); } - private static string TextId => Guid.NewGuid().ToString("D"); + private Task CreateQueue() => + _factory.Create( + _config.QueueName, + // ReSharper disable once SuspiciousTypeConversion.Global + _config.QueueSettings ?? _config as ISqsQueueSettings ?? DefaultSqsQueueSettings); - private async Task CreateQueue() + private SqsBatchPublisher CreatePublisher(SqsBatchAdapter adapter, ITimeAdapter? timeAdapter) { - var queueSettings = _config.SqsQueueSettings ?? new SqsQueueSettings(); - var queue = await _factory.Create(_config.QueueName, queueSettings); - return queue; - } + var concurrency = (_config.SqsConcurrency ?? DefaultSqsConcurrency).NotLessThan(1); - private IAliveKeeper CreateAliveKeeper() - { - - const int retryCount = 5; - var retryInterval = TimeSpan.FromSeconds(1); - var touchInterval = CalculateTouchInterval( - _visibility, retryInterval, retryCount); - var touchDelay = CalculateTouchDelay( - _visibility, touchInterval, retryInterval, retryCount); - - var aliveKeeperSettings = new AliveKeeperSettings { - DeleteBatchSize = SqsConstants.MaximumNumberOfMessages, - TouchBatchSize = SqsConstants.MaximumNumberOfMessages, - RetryLimit = retryCount, - RetryInterval = retryInterval, - TouchInterval = touchInterval, - TouchBatchDelay = touchDelay, - }; - - var aliveKeeper = AliveKeeper.Create( - TouchMany, - DeleteMany, - m => m.MessageId, - aliveKeeperSettings, - Log); - - return aliveKeeper; - } - - private IBatchBuilder> - CreateBatchSender() - { var batchSenderSettings = new BatchBuilderSettings { BatchDelay = TimeSpan.Zero, BatchSize = SqsConstants.MaximumNumberOfMessages, - Concurrency = 4, + Concurrency = concurrency, }; - return BatchBuilder + var builder = BatchBuilder .Create>( rq => rq.Id, rs => rs.Error?.Id ?? rs.Result?.Id ?? string.Empty, - SendMany, + adapter.Send, batchSenderSettings, - Log); + Log, + timeAdapter); + + return builder; + } + + private BatchSubscriber CreateSubscriber( + SqsBatchAdapter adapter, + Func handler) + { + var retryInterval = (_config.RetryInterval ?? DefaultRetryInterval).NotLessThan(MinimumRetryInterval); + var retryCount = (_config.RetryCount ?? DefaultRetryCount).NotLessThan(0); + var sqsConcurrency = (_config.SqsConcurrency ?? DefaultSqsConcurrency).NotLessThan(1); + var jobConcurrency = (_config.JobConcurrency ?? DefaultJobConcurrency).NotLessThan(1); + + var touchInterval = CalculateTouchInterval(_visibility, retryInterval, retryCount); + var touchDelay = CalculateTouchDelay(_visibility, touchInterval, retryInterval, retryCount); + var subscriber = new BatchSubscriber( + adapter, + (m, t) => HandleMessage(m, handler, t), + new BatchSubscriberSettings { + AlternateBatches = true, + AsynchronousDeletes = true, + BatchConcurrency = sqsConcurrency, + HandlerCount = jobConcurrency, + RetryInterval = retryInterval, + RetryLimit = retryCount, + TouchInterval = touchInterval, + DeleteBatchSize = SqsConstants.MaximumNumberOfMessages, + TouchBatchSize = SqsConstants.MaximumNumberOfMessages, + TouchBatchDelay = touchDelay, + }); + return subscriber; } private static TimeSpan CalculateTouchInterval( @@ -146,72 +168,18 @@ private static TimeSpan CalculateTouchDelay( var visibility = visibilityTimeout.TotalSeconds; var interval = touchInterval.TotalSeconds; var retryMargin = retryInterval.TotalSeconds * retryCount; - var delta = (visibility - interval - retryMargin) / retryCount; + var delta = (visibility - interval - retryMargin) / (retryCount + 1); return TimeSpan.FromSeconds(delta.NotLessThan(0).NotMoreThan(1)); } - private async Task[]> SendMany( - SendMessageBatchRequestEntry[] messages) - { - await _ready; - var requests = messages.ToList(); - Log.LogDebug("Sending batch of {Count} messages", requests.Count); - var responses = await _queue.Send(requests, CancellationToken.None); - return responses.ToArray(); - } - - private async Task DeleteMany(Message[] messages) - { - await _ready; - var map = messages.ToDictionary(m => m.MessageId); - var requests = messages.Select(ToDeleteRequest).ToList(); - Log.LogDebug("Deleting batch of {Count} messages", requests.Count); - if (requests.Count == 0) Debugger.Break(); - var response = await _queue.Delete(requests, CancellationToken.None); - return response - .SelectNotNull(r => r.Result?.Id) - .SelectNotNull(id => map.TryGetOrDefault(id)) - .ToArray(); - } - - private async Task TouchMany(Message[] messages) - { - await _ready; - var timeout = (int)_visibility.TotalSeconds; - var map = messages.ToDictionary(m => m.MessageId); - var requests = messages.Select(m => ToTouchRequest(m, timeout)).ToList(); - Log.LogDebug("Touching batch of {Count} messages", requests.Count); - if (requests.Count == 0) Debugger.Break(); - var response = await _queue.Touch(requests, CancellationToken.None); - return response - .SelectNotNull(r => r.Result?.Id) - .SelectNotNull(id => map.TryGetOrDefault(id)) - .ToArray(); - } - - private static DeleteMessageBatchRequestEntry ToDeleteRequest( - Message message) => - new() { - Id = message.MessageId, - ReceiptHandle = message.ReceiptHandle, - }; - - private static ChangeMessageVisibilityBatchRequestEntry ToTouchRequest( - Message message, int timeout) => - new() { - Id = message.MessageId, - ReceiptHandle = message.ReceiptHandle, - VisibilityTimeout = timeout, - }; - - private static SendMessageBatchRequestEntry CreateSendOneRequest( + private SendMessageBatchRequestEntry CreateSendOneRequest( TimeSpan delay, Guid jobId, DateTime scheduledFor, string? payload) { delay = delay.NotLessThan(TimeSpan.Zero).NotMoreThan(SqsConstants.MaximumDelay); return new SendMessageBatchRequestEntry { - Id = TextId, - DelaySeconds = (int)delay.TotalSeconds, + Id = Interlocked.Increment(ref _messageId).ToString("x16"), + DelaySeconds = (int)Math.Ceiling(delay.TotalSeconds), MessageBody = payload ?? string.Empty, MessageAttributes = new Dictionary { [SqsAttributes.JobId] = CreateAttribute(jobId), @@ -244,7 +212,7 @@ public async Task Publish(TimeSpan delay, IJob job, CancellationToken token) var jobId = job.JobId; var request = CreateSendOneRequest(delay, job); - var response = await _batchSender.Request(request); + var response = await _publisher.Request(request); var error = response.Error; if (error is null) @@ -255,50 +223,33 @@ public async Task Publish(TimeSpan delay, IJob job, CancellationToken token) nameof(job)); } - public IDisposable Subscribe(Func handler) => - Agent.Launch(c => LongPollLoop(c, handler, _config.Concurrency), Log); - - private async Task LongPollLoop( - IAgentContext context, - Func handler, - int concurrency) + public IDisposable Subscribe(Func handler) { - await _ready; - - var token = context.Token; - var semaphore = new SemaphoreSlim(concurrency.NotLessThan(1)); + _ready.Await(); - while (!token.IsCancellationRequested) + lock (_mutex) { - await LongPoll(handler, semaphore, token); + if (_subscriber is not null) + throw new InvalidOperationException("Only one subscriber is allowed"); + + _subscriber = CreateSubscriber(_adapter, handler); + _subscriber.Start(); + return Disposable.Create(DisposeSubscriber); } } - private async Task LongPoll( - Func handler, - SemaphoreSlim semaphore, - CancellationToken token) + private void DisposeSubscriber() { - var messages = (await _queue.Receive(token)).ToArray(); - if (messages.Length <= 0) return; - - foreach (var message in messages) - { - _aliveKeeper.Upkeep(message, token); - } - - foreach (var message in messages) + lock (_mutex) { - await semaphore.WaitAsync(token); - // every execution is in separate "thread" and is not awaited - Task.Run(() => HandleMessage(message, handler, semaphore, token), token).Forget(); + _subscriber?.Dispose(); + _subscriber = null; } } private async Task HandleMessage( Message message, Func handler, - SemaphoreSlim semaphore, CancellationToken token) { try @@ -306,23 +257,18 @@ private async Task HandleMessage( token.ThrowIfCancellationRequested(); var job = new SqsReceivedJob(message, _serializer); await handler(token, job); - await _aliveKeeper.Delete(message, token); } catch (Exception e) { Log.LogError(e, "Failed to process message {MessageId}", message.MessageId); } - finally - { - _aliveKeeper.Forget(message); - semaphore.Release(); - } } + #warning double dispose is killing it (most likely BatchBuilder) public void Dispose() { _ready.Await(); // it is safe to finish async initialization - _batchSender.Dispose(); - _aliveKeeper.Dispose(); + _publisher.Dispose(); + DisposeSubscriber(); } } diff --git a/src/K4os.Xpovoc.Sqs/ToysTimeSourceAdapter.cs b/src/K4os.Xpovoc.Sqs/ToysTimeSourceAdapter.cs new file mode 100644 index 0000000..286fcb3 --- /dev/null +++ b/src/K4os.Xpovoc.Sqs/ToysTimeSourceAdapter.cs @@ -0,0 +1,14 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using K4os.Xpovoc.Abstractions; + +namespace K4os.Xpovoc.Sqs; + +internal class ToysTimeSourceAdapter: K4os.Async.Toys.ITimeSource +{ + private readonly ITimeSource _timeSource; + public ToysTimeSourceAdapter(ITimeSource timeSource) => _timeSource = timeSource; + public Task Delay(TimeSpan delay, CancellationToken token) => _timeSource.Delay(delay, token); + public DateTimeOffset Now => _timeSource.Now; +} diff --git a/src/K4os.Xpovoc.Test/JobHandlerTests.cs b/src/K4os.Xpovoc.Test/JobHandlerTests.cs index 8bdd610..ad39d88 100644 --- a/src/K4os.Xpovoc.Test/JobHandlerTests.cs +++ b/src/K4os.Xpovoc.Test/JobHandlerTests.cs @@ -33,8 +33,12 @@ public async Task SimpleJobHandlerResolvesRightHandler() var guid = Guid.NewGuid().ToString(); await scheduler.Schedule(DateTimeOffset.Now, guid); + #pragma warning disable xUnit1031 + Assert.True(result.Task.Wait(5000)); Assert.Equal(guid, result.Task.Result); + + #pragma warning restore xUnit1031 scheduler.Dispose(); } diff --git a/src/K4os.Xpovoc.Test/K4os.Xpovoc.Test.csproj b/src/K4os.Xpovoc.Test/K4os.Xpovoc.Test.csproj index d39c279..4d86196 100644 --- a/src/K4os.Xpovoc.Test/K4os.Xpovoc.Test.csproj +++ b/src/K4os.Xpovoc.Test/K4os.Xpovoc.Test.csproj @@ -1,17 +1,25 @@ + net6.0 false + - + + - - - - - - + + + + + + + + + + + \ No newline at end of file diff --git a/src/Playground/ColorConsoleProvider.cs b/src/Playground/ColorConsoleProvider.cs deleted file mode 100644 index 8631a4d..0000000 --- a/src/Playground/ColorConsoleProvider.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System; -using Microsoft.Extensions.Logging; - -namespace Playground; - -internal class ColorConsoleProvider: ILoggerProvider, ILogger -{ - public ILogger CreateLogger(string categoryName) => this; - - public void Log( - LogLevel logLevel, EventId eventId, TState state, Exception? exception, - Func formatter) - { - try - { - Log(logLevel, formatter(state, exception)); - if (exception is null) return; - - Log(logLevel, $"{exception.GetType().Name}: {exception.Message}"); - if (string.IsNullOrWhiteSpace(exception.StackTrace)) return; - - Log(logLevel, exception.StackTrace); - } - catch (Exception e) - { - Log(LogLevel.Warning, $""); - } - } - - private static void Log(LogLevel logLevel, string message) - { - if (string.IsNullOrWhiteSpace(message)) - return; - - lock (Console.Out) - { - var color = Console.ForegroundColor; - Console.ForegroundColor = ToColor(logLevel); - Console.WriteLine(message); - Console.ForegroundColor = color; - } - } - - private static ConsoleColor ToColor(LogLevel logLevel) - { - switch (logLevel) - { - case LogLevel.Debug: return ConsoleColor.Gray; - case LogLevel.Information: return ConsoleColor.Cyan; - case LogLevel.Warning: return ConsoleColor.Yellow; - case LogLevel.Error: return ConsoleColor.Red; - case LogLevel.Critical: return ConsoleColor.Magenta; - } - - return ConsoleColor.DarkGray; - } - - public bool IsEnabled(LogLevel logLevel) => true; - - public IDisposable? BeginScope(TState state) where TState: notnull => null; - - public void Dispose() { } -} \ No newline at end of file diff --git a/src/Playground/InfiniteScheduling.cs b/src/Playground/InfiniteScheduling.cs new file mode 100644 index 0000000..c20e057 --- /dev/null +++ b/src/Playground/InfiniteScheduling.cs @@ -0,0 +1,309 @@ +using System; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Xml.XPath; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using Amazon.SQS; +using K4os.Xpovoc.Abstractions; +using K4os.Xpovoc.Core.Db; +using K4os.Xpovoc.Core.Memory; +using K4os.Xpovoc.Core.Queue; +using K4os.Xpovoc.Core.Sql; +using K4os.Xpovoc.Mongo; +using K4os.Xpovoc.MsSql; +using K4os.Xpovoc.MySql; +using K4os.Xpovoc.PgSql; +using K4os.Xpovoc.Redis; +using K4os.Xpovoc.SqLite; +using K4os.Xpovoc.Sqs; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using Playground.Utilities; +using StackExchange.Redis; + +// ReSharper disable UnusedParameter.Local + +namespace Playground; + +internal static class InfiniteScheduling +{ + private static readonly int ProduceDelay = 0; + private static readonly int ConsumeDelay = 0; + private static readonly int ConsumeThreads = 4; + private static readonly bool EnablePruning = true; + + public static Task Compositions(string[] args) + { + var collection = new ServiceCollection(); + MySqlExamples.Configure(collection); + var provider = collection.BuildServiceProvider(); + MySqlExamples.Startup(provider); + return Task.CompletedTask; + } + + public static async Task Run(ILoggerFactory loggerFactory) + { + var serviceCollection = new ServiceCollection(); + serviceCollection.AddSingleton(loggerFactory); + + Configure(serviceCollection); + var serviceProvider = serviceCollection.BuildServiceProvider(); + + await Execute(loggerFactory, serviceProvider); + } + + private static void Configure(ServiceCollection serviceCollection) + { + var secrets = Secrets.Load("databases.xml").Required(); + + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton( + p => new AdHocJobHandler(ConsumeOne)); + + ConfigureMemory(serviceCollection); + ConfigureMySql(serviceCollection, secrets); + ConfigurePgSql(serviceCollection, secrets); + ConfigureSqLite(serviceCollection, secrets); + ConfigureMsSql(serviceCollection, secrets); + ConfigureMongo(serviceCollection, secrets); + ConfigureRedis(serviceCollection, secrets); + ConfigureSqs(serviceCollection, secrets); + + serviceCollection.AddSingleton( + new SchedulerConfig { + WorkerCount = ConsumeThreads, + KeepFinishedJobsPeriod = EnablePruning ? TimeSpan.Zero : TimeSpan.FromDays(90), + PruneInterval = EnablePruning ? TimeSpan.FromSeconds(1) : TimeSpan.FromDays(1), + }); + } + + private static void ConfigureMemory(ServiceCollection serviceCollection) + { + serviceCollection.AddTransient( + p => new MemoryJobStorage()); + } + + private static void ConfigureMySql(ServiceCollection serviceCollection, XDocument secrets) + { + var connectionString = secrets.XPathSelectElement("/secrets/mysql")?.Value; + serviceCollection.AddSingleton( + new MySqlJobStorageConfig { + ConnectionString = connectionString.Required(), + Prefix = "xpovoc_", + }); + serviceCollection.AddSingleton( + p => new MySqlJobStorage( + p.GetRequiredService(), + p.GetRequiredService())); + } + + private static void ConfigurePgSql(ServiceCollection serviceCollection, XDocument secrets) + { + var connectionString = secrets.XPathSelectElement("/secrets/pgsql")?.Value; + serviceCollection.AddSingleton( + new PgSqlJobStorageConfig { + ConnectionString = connectionString.Required(), + }); + serviceCollection.AddSingleton( + p => new PgSqlJobStorage( + p.GetRequiredService(), + p.GetRequiredService())); + } + + private static void ConfigureMsSql(ServiceCollection serviceCollection, XDocument secrets) + { + var connectionString = secrets.XPathSelectElement("/secrets/mssql")?.Value; + serviceCollection.AddSingleton( + new MsSqlJobStorageConfig { + ConnectionString = connectionString.Required(), + Schema = "xpovoc", + }); + serviceCollection.AddSingleton( + p => new MsSqlJobStorage( + p.GetRequiredService(), + p.GetRequiredService())); + } + + private static void ConfigureSqLite(ServiceCollection serviceCollection, XDocument secrets) + { + var connectionString = secrets.XPathSelectElement("/secrets/sqlite")?.Value; + serviceCollection.AddSingleton( + new SqLiteJobStorageConfig { + ConnectionString = connectionString.Required(), + Prefix = "xpovoc_", + PoolSize = 1, + }); + serviceCollection.AddSingleton( + p => new SqLiteJobStorage( + p.GetRequiredService(), + p.GetRequiredService())); + } + + private static void ConfigureMongo(ServiceCollection serviceCollection, XDocument secrets) + { + var connectionString = secrets.XPathSelectElement("/secrets/mongo")?.Value; + serviceCollection.AddSingleton( + p => new MongoClient(connectionString)); + serviceCollection.AddSingleton( + p => p.GetRequiredService().GetDatabase("test")); + serviceCollection.AddSingleton( + p => new MongoJobStorage( + p.GetRequiredService, + "xpovoc_devel_jobs", + DefaultMongoJobSerializer.Instance)); + } + + private static void ConfigureRedis(ServiceCollection serviceCollection, XDocument secrets) + { + var connectionString = secrets.XPathSelectElement("/secrets/redis")?.Value; + serviceCollection.AddSingleton( + p => ConnectionMultiplexer.Connect(connectionString).GetDatabase()); + serviceCollection.AddSingleton( + p => new RedisJobStorageConfig { Prefix = "xpovoc" }); + serviceCollection.AddSingleton( + p => new RedisJobStorage( + p.GetRequiredService(), + p.GetRequiredService(), + p.GetRequiredService())); + } + + private static void ConfigureSqs(ServiceCollection serviceCollection, XDocument secrets) + { + serviceCollection.AddSingleton( + p => new AmazonSQSClient( + new AmazonSQSConfig { ServiceURL = "http://localhost:9324", })); + serviceCollection.AddSingleton( + p => new SqsJobQueueAdapterConfig { + QueueName = "xpovoc-playground", + JobConcurrency = 16, + SqsConcurrency = 16, + }); + serviceCollection.AddSingleton( + p => new SqsJobQueueAdapter( + NullLoggerFactory.Instance, + // p.GetRequiredService(), + p.GetRequiredService(), + p.GetRequiredService(), + p.GetRequiredService())); + serviceCollection.AddSingleton( + p => new QueueJobScheduler( + NullLoggerFactory.Instance, + // p.GetRequiredService(), + p.GetRequiredService(), + p.GetRequiredService())); + } + + private static async Task Execute( + ILoggerFactory loggerFactory, IServiceProvider serviceProvider) + { + var cancel = new CancellationTokenSource(); + var token = cancel.Token; + +// var scheduler = new RxJobScheduler(loggerFactory, handler, Scheduler.Default); +// var scheduler = new DbJobScheduler(loggerFactory, mysqlStorage, handler, schedulerConfig); +// var scheduler = new DbJobScheduler(loggerFactory, redisStorage, handler, schedulerConfig); + var scheduler = serviceProvider.GetRequiredService(); + + // var producer = Task.CompletedTask; + // var producerSpeed = Task.CompletedTask; + var producer = Task.WhenAll( + Task.Run(() => Producer(token, scheduler), token), + Task.Run(() => Producer(token, scheduler), token), + Task.Run(() => Producer(token, scheduler), token), + Task.Run(() => Producer(token, scheduler), token) + ); + var producerSpeed = Task.Run( + () => Measure( + token, + loggerFactory, + "Produced", + () => Volatile.Read(ref _producedCount)), + token); + + var consumedSpeed = Task.Run( + () => Measure( + token, + loggerFactory, + "Consumed", + () => Volatile.Read(ref _consumedCount)), + token); + + // ReSharper disable once MethodSupportsCancellation + await Task.Run(Console.ReadLine); + + cancel.Cancel(); + await Task.WhenAny(producer, producerSpeed, consumedSpeed); + } + + private static async Task Measure( + CancellationToken token, ILoggerFactory loggerFactory, string name, Func probe) + { + var logger = loggerFactory.CreateLogger(name); + + logger.LogInformation("{Name} started", name); + + await Task.Delay(TimeSpan.FromSeconds(5), token); + + logger.LogInformation("{Name} active", name); + + var stopwatch = Stopwatch.StartNew(); + + var counter = probe(); + var timestamp = stopwatch.Elapsed.TotalSeconds; + + while (!token.IsCancellationRequested) + { + await Task.Delay(TimeSpan.FromSeconds(3), token); + var delta = probe() - counter; + var interval = stopwatch.Elapsed.TotalSeconds - timestamp; + logger.LogDebug("Rate({Name}): {Rate:0}/s ({Counter})", name, delta / interval, counter); + counter += delta; + timestamp += interval; + } + } + + private static long _producedCount; + private static long _consumedCount; + + private static async Task Producer(CancellationToken token, IJobScheduler scheduler) + { + var random = new Random(); + while (!token.IsCancellationRequested) + { + if (ProduceDelay > 0) + await Task.Delay(ProduceDelay, token); + var delay = TimeSpan.FromSeconds(random.NextDouble() * 5); + var message = Guid.NewGuid(); + var when = DateTimeOffset.UtcNow.Add(delay); + Produced.TryAdd(message, when); + await scheduler.Schedule(when, message); + Interlocked.Increment(ref _producedCount); + } + } + + private static readonly ConcurrentDictionary Produced = new(); + private static readonly ConcurrentDictionary Consumed = new(); + + private static void ConsumeOne(object payload) + { + if (ConsumeDelay > 0) + Thread.Sleep(ConsumeDelay); + var now = DateTimeOffset.UtcNow; + var guid = (Guid)payload; + var result = Consumed.TryAdd(guid, DateTimeOffset.UtcNow); + if (!result) + { + var consumed = Consumed[guid]; + var produced = Produced[guid]; + Console.WriteLine( + $"{guid}: scheduled {produced:HH:mm:ss.fff} consumed {consumed:HH:mm:ss.fff} now {now:HH:mm:ss.fff}"); + } + + Interlocked.Increment(ref _consumedCount); + } +} \ No newline at end of file diff --git a/src/Playground/Playground.csproj b/src/Playground/Playground.csproj index 5c973a9..9fc4ad8 100644 --- a/src/Playground/Playground.csproj +++ b/src/Playground/Playground.csproj @@ -21,14 +21,20 @@ + - - - - - + + - + + + + + + + + + \ No newline at end of file diff --git a/src/Playground/Program.cs b/src/Playground/Program.cs index 96a7ff9..13b1c19 100644 --- a/src/Playground/Program.cs +++ b/src/Playground/Program.cs @@ -1,252 +1,17 @@ -using System; -using System.Collections.Concurrent; -using System.Diagnostics; -using System.Xml.XPath; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using System.Xml.Linq; -using K4os.Xpovoc.Abstractions; -using K4os.Xpovoc.Core.Db; -using K4os.Xpovoc.Core.Memory; -using K4os.Xpovoc.Core.Sql; -using K4os.Xpovoc.Mongo; -using K4os.Xpovoc.MsSql; -using K4os.Xpovoc.MySql; -using K4os.Xpovoc.PgSql; -using K4os.Xpovoc.Redis; -using K4os.Xpovoc.SqLite; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using StackExchange.Redis; - -// ReSharper disable UnusedParameter.Local - -namespace Playground -{ - internal static class Program - { - private const int VLN = 1_000_000; - private static readonly int ProduceDelay = 0; - private static readonly int ConsumeDelay = 0; - private static readonly int ConsumeThreads = 4; - private static readonly bool EnablePruning = true; - - public static Task Compositions(string[] args) - { - var collection = new ServiceCollection(); - MySqlExamples.Configure(collection); - var provider = collection.BuildServiceProvider(); - MySqlExamples.Startup(provider); - return Task.CompletedTask; - } - - public static async Task Main(string[] args) - { - var loggerFactory = new LoggerFactory(); - loggerFactory.AddProvider(new ColorConsoleProvider()); - var serviceCollection = new ServiceCollection(); - serviceCollection.AddSingleton(loggerFactory); - - Configure(serviceCollection); - var serviceProvider = serviceCollection.BuildServiceProvider(); - - await Execute(loggerFactory, serviceProvider, args); - } - - private static void Configure(ServiceCollection serviceCollection) - { - var secrets = Secrets.Load("databases.xml").Required(); - - ConfigureMySql(serviceCollection, secrets); - ConfigurePgSql(serviceCollection, secrets); - ConfigureSqLite(serviceCollection, secrets); - ConfigureMsSql(serviceCollection, secrets); - ConfigureMongo(serviceCollection, secrets); - ConfigureRedis(serviceCollection, secrets); - - serviceCollection.AddSingleton( - new SchedulerConfig { - WorkerCount = ConsumeThreads, - KeepFinishedJobsPeriod = EnablePruning ? TimeSpan.Zero : TimeSpan.FromDays(90), - PruneInterval = EnablePruning ? TimeSpan.FromSeconds(1) : TimeSpan.FromDays(1), - }); - } - - private static void ConfigureMySql(ServiceCollection serviceCollection, XDocument secrets) - { - var connectionString = secrets.XPathSelectElement("/secrets/mysql")?.Value; - serviceCollection.AddSingleton( - new MySqlJobStorageConfig { - ConnectionString = connectionString.Required(), - Prefix = "xpovoc_", - }); - } - - private static void ConfigurePgSql(ServiceCollection serviceCollection, XDocument secrets) - { - var connectionString = secrets.XPathSelectElement("/secrets/pgsql")?.Value; - serviceCollection.AddSingleton( - new PgSqlJobStorageConfig { - ConnectionString = connectionString.Required(), - }); - } - - private static void ConfigureMsSql(ServiceCollection serviceCollection, XDocument secrets) - { - var connectionString = secrets.XPathSelectElement("/secrets/mssql")?.Value; - serviceCollection.AddSingleton( - new MsSqlJobStorageConfig { - ConnectionString = connectionString.Required(), - Schema = "xpovoc", - }); - } - - private static void ConfigureSqLite(ServiceCollection serviceCollection, XDocument secrets) - { - var connectionString = secrets.XPathSelectElement("/secrets/sqlite")?.Value; - serviceCollection.AddSingleton( - new SqLiteJobStorageConfig { - ConnectionString = connectionString.Required(), - Prefix = "xpovoc_", - PoolSize = 1, - }); - } - - private static void ConfigureMongo(ServiceCollection serviceCollection, XDocument secrets) - { - var connectionString = secrets.XPathSelectElement("/secrets/mongo")?.Value; - serviceCollection.AddSingleton( - p => new MongoClient(connectionString)); - serviceCollection.AddSingleton( - p => p.GetRequiredService().GetDatabase("test")); - } - - private static void ConfigureRedis(ServiceCollection serviceCollection, XDocument secrets) - { - var connectionString = secrets.XPathSelectElement("/secrets/redis")?.Value; - serviceCollection.AddSingleton( - p => ConnectionMultiplexer.Connect(connectionString).GetDatabase()); - serviceCollection.AddSingleton( - p => new RedisJobStorageConfig { Prefix = "xpovoc" }); - } - - private static async Task Execute( - ILoggerFactory loggerFactory, IServiceProvider serviceProvider, string[] args) - { - var cancel = new CancellationTokenSource(); - var token = cancel.Token; - var serializer = new DefaultJobSerializer(); - - var memStorage = new MemoryJobStorage(); - var mysqlStorage = new MySqlJobStorage( - serviceProvider.GetRequiredService(), serializer); - var postgresStorage = new PgSqlJobStorage( - serviceProvider.GetRequiredService(), serializer); - var sqliteStorage = new SqLiteJobStorage( - serviceProvider.GetRequiredService(), serializer); - var mssqlStorage = new MsSqlJobStorage( - serviceProvider.GetRequiredService(), serializer); - var mongoStorage = new MongoJobStorage( - serviceProvider.GetRequiredService, - "xpovoc_devel_jobs", - DefaultMongoJobSerializer.Instance); - var redisStorage = new RedisJobStorage( - serviceProvider.GetRequiredService(), - serviceProvider.GetRequiredService(), - serializer); - - var handler = new AdHocJobHandler(ConsumeOne); - var schedulerConfig = serviceProvider.GetRequiredService(); - // var scheduler = new DbJobScheduler(null, mysqlStorage, handler, schedulerConfig); - // var scheduler = new RxJobScheduler(loggerFactory, handler, Scheduler.Default); - - var scheduler = new DbJobScheduler(null, redisStorage, handler, schedulerConfig); - - // var producer = Task.CompletedTask; - // var producerSpeed = Task.CompletedTask; - var producer = Task.Run(() => Producer(token, scheduler), token); - var producerSpeed = Task.Run( - () => Measure( - token, - loggerFactory, - "Produced", - () => Volatile.Read(ref _producedCount))); - - var consumedSpeed = Task.Run( - () => Measure( - token, - loggerFactory, - "Consumed", - () => Volatile.Read(ref _consumedCount))); - - // ReSharper disable once MethodSupportsCancellation - await Task.Run(Console.ReadLine); - - cancel.Cancel(); - await Task.WhenAny(producer, producerSpeed, consumedSpeed); - } - - private static async Task Measure( - CancellationToken token, ILoggerFactory loggerFactory, string name, Func probe) - { - var logger = loggerFactory.CreateLogger(name); - - logger.LogInformation($"{name} started"); - - await Task.Delay(TimeSpan.FromSeconds(5), token); - - logger.LogInformation($"{name} active"); - - var stopwatch = Stopwatch.StartNew(); - - var counter = probe(); - var timestamp = stopwatch.Elapsed.TotalSeconds; - - while (!token.IsCancellationRequested) - { - await Task.Delay(TimeSpan.FromSeconds(3), token); - var delta = probe() - counter; - var interval = stopwatch.Elapsed.TotalSeconds - timestamp; - logger.LogDebug($"Rate({name}): {delta / interval:F1}/s ({counter})"); - counter += delta; - timestamp += interval; - } - } - - private static long _producedCount; - private static long _consumedCount; - - private static async Task Producer(CancellationToken token, IJobScheduler scheduler) - { - var random = new Random(); - while (!token.IsCancellationRequested) - { - if (ProduceDelay > 0) - await Task.Delay(ProduceDelay, token); - var delay = TimeSpan.FromSeconds(random.NextDouble() * 5); - var message = Guid.NewGuid(); - await scheduler.Schedule(DateTimeOffset.UtcNow.Add(delay), message); - Interlocked.Increment(ref _producedCount); - } - } - - private static readonly ConcurrentDictionary Guids = new(); - - private static void ConsumeOne(object payload) - { - if (ConsumeDelay > 0) - Thread.Sleep(ConsumeDelay); - var guid = (Guid)payload; - var result = Guids.TryAdd(guid, null); - if (!result) - { - Console.WriteLine("!!!"); - throw new ArgumentException("Job stealing!"); - } - - Interlocked.Increment(ref _consumedCount); - } - } -} +using System; +using Amazon.SQS; +using Playground; +using Serilog; +using Serilog.Extensions.Logging; + +Log.Logger = new LoggerConfiguration() + .MinimumLevel.Debug() + .WriteTo.Console() + .CreateLogger(); +var loggerFactory = new SerilogLoggerFactory(); +var sqsClient = new AmazonSQSClient(); + +// await InfiniteScheduling.Run(loggerFactory); + +var test = new TrueDeferredJobs(sqsClient, loggerFactory, TimeSpan.FromHours(1)); +await test.RunAsync(); diff --git a/src/Playground/SqsThroughput.cs b/src/Playground/SqsThroughput.cs new file mode 100644 index 0000000..7daafa9 --- /dev/null +++ b/src/Playground/SqsThroughput.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS; +using Amazon.SQS.Model; +using K4os.Xpovoc.Abstractions; +using K4os.Xpovoc.Core.Queue; +using K4os.Xpovoc.Core.Sql; +using K4os.Xpovoc.Sqs; +using K4os.Xpovoc.Sqs.Internal; +using Microsoft.Extensions.Logging; + +namespace Playground; + +public static class SqsThroughput +{ + + private const int SampleSize = 100_000; + private const string QueueName = "mk-test-move10k-adapter"; + + public static async Task Run(ILoggerFactory loggerFactory) + { + var log = loggerFactory.CreateLogger("SqsThroughputTest"); + + var client = + new AmazonSQSClient(new AmazonSQSConfig { ServiceURL = "http://localhost:9324" }); + await Flush(client); + + var adapter = new SqsJobQueueAdapter( + loggerFactory, + client, + new DefaultJobSerializer(), + new SqsJobQueueAdapterConfig { + QueueName = QueueName, + SqsConcurrency = 16, + JobConcurrency = 16, + }); + + var guids = Enumerable.Range(0, SampleSize).Select(_ => Guid.NewGuid()).ToList(); + + var push = SendAll(guids, adapter, log); + await push; + + var pull = PullAll(guids, adapter, log); + await pull; + + await Task.WhenAll(push, pull); + + log.LogInformation("Disposing..."); + + adapter.Dispose(); + + log.LogInformation("Done"); + } + + public static async Task SendAll(ICollection guids, IJobQueueAdapter queue, ILogger log) + { + var stopwatch = Stopwatch.StartNew(); + + var publishTasks = guids + .Select(g => new FakeJob(g)) + .Select(j => queue.Publish(TimeSpan.Zero, j, CancellationToken.None)); + await Task.WhenAll(publishTasks); + + var rate = guids.Count / stopwatch.Elapsed.TotalSeconds; + log.LogInformation("All messages sent {Rate:0.0}/s", rate); + } + + public static async Task PullAll(ICollection guids, IJobQueueAdapter queue, ILogger log) + { + var left = guids.ToHashSet(); + var done = new TaskCompletionSource(); + + Task HandleOne(IJob job) + { + lock (left) + { + left.Remove((Guid)job.Payload!); + if (left.Count <= 0) done.TrySetResult(true); + } + + return Task.CompletedTask; + } + + var stopwatch = Stopwatch.StartNew(); + + var subscription = queue.Subscribe((_, j) => HandleOne(j)); + + await done.Task; + + subscription.Dispose(); + + var rate = guids.Count / stopwatch.Elapsed.TotalSeconds; + log.LogInformation("All messages received {Rate:0.0}/s", rate); + } + + public static async Task Flush(IAmazonSQS amazonSqsClient) + { + var factory = new SqsQueueFactory(amazonSqsClient); + var queue = await factory.Create(QueueName, new SqsQueueSettings()); + + while (true) + { + var messages = await queue.Receive(CancellationToken.None); + if (messages.Count <= 0) break; + + var deletes = messages + .Select(m => new DeleteMessageBatchRequestEntry(m.MessageId, m.ReceiptHandle)) + .ToList(); + await queue.Delete(deletes, CancellationToken.None); + } + } + + internal class FakeJob: IJob + { + public Guid JobId { get; } + public DateTime UtcTime { get; } + public object? Payload { get; } + public object? Context { get; } + + public FakeJob(Guid guid) + { + JobId = guid; + UtcTime = DateTime.UtcNow; + Payload = guid; + Context = null; + } + } +} \ No newline at end of file diff --git a/src/Playground/TrueDeferredJobs.cs b/src/Playground/TrueDeferredJobs.cs new file mode 100644 index 0000000..ee1ea43 --- /dev/null +++ b/src/Playground/TrueDeferredJobs.cs @@ -0,0 +1,133 @@ +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS; +using K4os.Xpovoc.Abstractions; +using K4os.Xpovoc.Core.Queue; +using K4os.Xpovoc.Json; +using K4os.Xpovoc.Sqs; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace Playground; + +public class TrueDeferredJobs: IJobHandler +{ + protected readonly ILogger Log; + private readonly ILoggerFactory _loggerFactory; + private readonly IAmazonSQS _sqsClient; + private readonly ConcurrentDictionary _scheduled = new(); + private readonly TimeSpan _timeLimit; + + public TrueDeferredJobs(IAmazonSQS client, ILoggerFactory loggerFactory, TimeSpan timeLimit) + { + Log = loggerFactory.CreateLogger(); + _sqsClient = client; + _loggerFactory = loggerFactory; + _timeLimit = timeLimit; + } + + public async Task RunAsync() + { + var adapter = new SqsJobQueueAdapter( + _loggerFactory, + _sqsClient, + new JsonJobSerializer( + new JsonSerializerSettings { + TypeNameHandling = TypeNameHandling.All, + }), + new SqsJobQueueAdapterConfig { + QueueName = "xpovoc-TrueDeferredJobs", + JobConcurrency = 16, + }); + using var scheduler = new QueueJobScheduler(_loggerFactory, adapter, this); + + var limit = DateTime.UtcNow.Add(_timeLimit); + var next = DateTime.UtcNow; + + while (next < limit) + { + var @event = new Event { Id = Guid.NewGuid(), Time = next }; + _scheduled.TryAdd(@event.Id, @event); + next = next.AddSeconds(1); + } + + var actions = _scheduled.Values + .ToArray() + .OrderBy(x => x.Time) + .Select(e => scheduler.Schedule(e.Time, e)) + .ToArray(); + await Task.WhenAll(actions); + + while (true) + { + var now = DateTime.UtcNow; + if (now > limit) + { + if (_scheduled.IsEmpty) + { + Log.LogInformation("All events received"); + break; + } + + Log.LogWarning("Time limit reached"); + } + + if (now > limit.AddSeconds(10)) + { + if (!_scheduled.IsEmpty) + { + Log.LogError("Some events were not received: {Count}", _scheduled.Count); + } + + break; + } + + Log.LogInformation( + "Waiting for events... {Count} expected, time left {Left}", _scheduled.Count, + (limit - now).NotLessThan(TimeSpan.Zero)); + await Task.Delay(TimeSpan.FromSeconds(3)); + } + } + + public Task Handle(CancellationToken token, object payload) + { + var received = (Event)payload; + if (!_scheduled.TryRemove(received.Id, out _)) + { + Log.LogWarning("Received event {Id} which was not scheduled", received.Id); + return Task.CompletedTask; + } + + var now = DateTime.UtcNow; + var expected = received.Time; + + if (now < expected) + { + Log.LogError( + "Received event {Id} too early: {Received} vs {Now}", + received.Id, received.Time, now); + } + + if (now > expected.AddSeconds(3)) + { + Log.LogError( + "Received event {Id} too late: {Received} vs {Now}", + received.Id, received.Time, now); + } + + Log.LogInformation( + "Received event {Id} at {Now} (expected {Expected}, diff {Diff:0.00}s)", + received.Id, now, expected, (now - expected).TotalSeconds); + + return Task.CompletedTask; + } +} + +public class Event +{ + public Guid Id { get; set; } + public DateTime Time { get; set; } +} diff --git a/src/Playground/MySqlExamples.cs b/src/Playground/Utilities/MySqlExamples.cs similarity index 96% rename from src/Playground/MySqlExamples.cs rename to src/Playground/Utilities/MySqlExamples.cs index a4cbba5..937f392 100644 --- a/src/Playground/MySqlExamples.cs +++ b/src/Playground/Utilities/MySqlExamples.cs @@ -7,7 +7,7 @@ using Microsoft.Extensions.DependencyInjection; using Newtonsoft.Json; -namespace Playground; +namespace Playground.Utilities; internal class MySqlExamples { diff --git a/src/Playground/NullJobHandler.cs b/src/Playground/Utilities/NullJobHandler.cs similarity index 87% rename from src/Playground/NullJobHandler.cs rename to src/Playground/Utilities/NullJobHandler.cs index 19a6e54..be1c8a8 100644 --- a/src/Playground/NullJobHandler.cs +++ b/src/Playground/Utilities/NullJobHandler.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using K4os.Xpovoc.Abstractions; -namespace Playground; +namespace Playground.Utilities; internal class NullJobHandler: IJobHandler { diff --git a/src/Playground/PgSqlExamples.cs b/src/Playground/Utilities/PgSqlExamples.cs similarity index 96% rename from src/Playground/PgSqlExamples.cs rename to src/Playground/Utilities/PgSqlExamples.cs index 9d56336..5526eea 100644 --- a/src/Playground/PgSqlExamples.cs +++ b/src/Playground/Utilities/PgSqlExamples.cs @@ -7,7 +7,7 @@ using Microsoft.Extensions.DependencyInjection; using Newtonsoft.Json; -namespace Playground; +namespace Playground.Utilities; internal class PgSqlExamples { From e3b5c810276f5c8f8ed58b81acef446cc52490f7 Mon Sep 17 00:00:00 2001 From: Milosz Krajewski Date: Sat, 18 Nov 2023 15:14:51 +0000 Subject: [PATCH 3/5] many pollers --- Directory.Packages.props | 2 +- src/K4os.Shared/Extensions.cs | 4 +- src/K4os.Shared/K4os.Shared.csproj | 6 +++ src/K4os.Xpovoc.Core/K4os.Xpovoc.Core.csproj | 1 + .../K4os.Xpovoc.Mongo.csproj | 1 + .../Move10KUsingSqsAdapter.cs | 5 +- .../SqsQueueAdapterTests.cs | 18 +++---- .../Utilities/TestSqsQueue.cs | 2 +- src/K4os.Xpovoc.Sqs.Test/VeryLongRun.cs | 2 +- .../ISqsJobQueueAdapterSettings.cs | 28 +++++----- .../Internal/ISqsQueueSettings.cs | 8 +-- .../Internal/SqsQueueFactory.cs | 13 ++--- .../Internal/SqsQueueSettings.cs | 8 +-- src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs | 52 ++++++++++++------- src/Playground/InfiniteScheduling.cs | 15 +++--- src/Playground/Program.cs | 6 +-- src/Playground/SqsThroughput.cs | 5 +- src/Playground/TrueDeferredJobs.cs | 2 +- 18 files changed, 100 insertions(+), 78 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index e886c7c..ffc8934 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,7 +4,7 @@ --> - + diff --git a/src/K4os.Shared/Extensions.cs b/src/K4os.Shared/Extensions.cs index ffe4908..ce9ee3f 100644 --- a/src/K4os.Shared/Extensions.cs +++ b/src/K4os.Shared/Extensions.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading.Tasks; // ReSharper disable UnusedMember.Global @@ -13,7 +14,8 @@ namespace System; internal static class Extensions { public static T Required( - this T? subject, string? name = null) where T: class => + this T? subject, [CallerArgumentExpression("subject")] string? name = null) + where T: class => subject ?? throw new ArgumentNullException(name ?? ""); public static void TryDispose(this object subject) diff --git a/src/K4os.Shared/K4os.Shared.csproj b/src/K4os.Shared/K4os.Shared.csproj index 4641dbd..ae03c87 100644 --- a/src/K4os.Shared/K4os.Shared.csproj +++ b/src/K4os.Shared/K4os.Shared.csproj @@ -1,5 +1,11 @@ + netstandard2.0 + + + + + diff --git a/src/K4os.Xpovoc.Core/K4os.Xpovoc.Core.csproj b/src/K4os.Xpovoc.Core/K4os.Xpovoc.Core.csproj index 14df50a..c08c743 100644 --- a/src/K4os.Xpovoc.Core/K4os.Xpovoc.Core.csproj +++ b/src/K4os.Xpovoc.Core/K4os.Xpovoc.Core.csproj @@ -19,6 +19,7 @@ + \ No newline at end of file diff --git a/src/K4os.Xpovoc.Mongo/K4os.Xpovoc.Mongo.csproj b/src/K4os.Xpovoc.Mongo/K4os.Xpovoc.Mongo.csproj index 3783366..3af9b1c 100644 --- a/src/K4os.Xpovoc.Mongo/K4os.Xpovoc.Mongo.csproj +++ b/src/K4os.Xpovoc.Mongo/K4os.Xpovoc.Mongo.csproj @@ -15,6 +15,7 @@ + diff --git a/src/K4os.Xpovoc.Sqs.Test/Move10KUsingSqsAdapter.cs b/src/K4os.Xpovoc.Sqs.Test/Move10KUsingSqsAdapter.cs index 12a84a9..5673897 100644 --- a/src/K4os.Xpovoc.Sqs.Test/Move10KUsingSqsAdapter.cs +++ b/src/K4os.Xpovoc.Sqs.Test/Move10KUsingSqsAdapter.cs @@ -34,9 +34,10 @@ public async Task NoMessagesAreLost(int sqsConcurrency, int jobConcurrency) loggerFactory, queueFactory, new DefaultJobSerializer(), - new SqsJobQueueAdapterConfig { + new SqsJobQueueAdapterSettings { QueueName = "mk-test-move10k-adapter", - SqsConcurrency = sqsConcurrency, + PushConcurrency = sqsConcurrency, + PullConcurrency = sqsConcurrency, JobConcurrency = jobConcurrency, }); diff --git a/src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs b/src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs index 649e6df..17c4b57 100644 --- a/src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs +++ b/src/K4os.Xpovoc.Sqs.Test/SqsQueueAdapterTests.cs @@ -20,12 +20,12 @@ public class SqsJobQueueAdapterTests private static readonly JsonJobSerializer JsonJobSerializer = new( new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Auto }); - private static readonly SqsJobQueueAdapterConfig DefaultAdapterConfig = new() { + private static readonly SqsJobQueueAdapterSettings DefaultAdapterSettings = new() { QueueName = "mk-SqsJobQueueAdapter-tests", JobConcurrency = 1, }; - private static readonly TestConfig FastRoundtripConfig = new() { + private static readonly SqsJobQueueAdapterSettings FastRoundtripSettings = new() { QueueName = "mk-SqsJobQueueAdapter-5s-tests", JobConcurrency = 1, QueueSettings = new SqsQueueSettings { @@ -33,7 +33,7 @@ public class SqsJobQueueAdapterTests }, }; - private static readonly TestConfig BatchingConfig = new() { + private static readonly SqsJobQueueAdapterSettings BatchingSettings = new() { QueueName = "mk-SqsJobQueueAdapter-10s-tests", JobConcurrency = 16, QueueSettings = new SqsQueueSettings { @@ -54,7 +54,7 @@ public void AdapterCanBeInstantiatedAndDisposed() _loggerFactory, AmazonSqsClient, JsonJobSerializer, - DefaultAdapterConfig); + DefaultAdapterSettings); adapter.Dispose(); } @@ -65,7 +65,7 @@ public async Task CanPublishJob() _loggerFactory, AmazonSqsClient, JsonJobSerializer, - DefaultAdapterConfig); + DefaultAdapterSettings); await adapter.Publish( TimeSpan.Zero, @@ -82,7 +82,7 @@ public async Task CanSubscribe() _loggerFactory, AmazonSqsClient, JsonJobSerializer, - DefaultAdapterConfig); + DefaultAdapterSettings); var guid = Guid.NewGuid().ToString(); _output.WriteLine("Expecting: {0}", guid); @@ -112,7 +112,7 @@ public async Task CanDelayMessage() _loggerFactory, AmazonSqsClient, JsonJobSerializer, - DefaultAdapterConfig); + DefaultAdapterSettings); var guid = Guid.NewGuid().ToString(); _output.WriteLine("Expecting: {0}", guid); @@ -150,7 +150,7 @@ public async Task HandledMessageIsKeptInvisible() _loggerFactory, AmazonSqsClient, JsonJobSerializer, - FastRoundtripConfig); + FastRoundtripSettings); // make some artificial crowd for (var i = 0; i < 10; i++) @@ -210,7 +210,7 @@ public async Task InConcurrentSettingItUsesBatching() _loggerFactory, AmazonSqsClient, JsonJobSerializer, - BatchingConfig); + BatchingSettings); var expected = Enumerable .Range(0, 100) diff --git a/src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueue.cs b/src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueue.cs index d4f6cdd..138b33b 100644 --- a/src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueue.cs +++ b/src/K4os.Xpovoc.Sqs.Test/Utilities/TestSqsQueue.cs @@ -20,7 +20,7 @@ public TestSqsQueue( IScheduler scheduler) { _scheduler = scheduler; - _visibility = settings.VisibilityTimeout ?? SqsConstants.DefaultVisibilityTimeout; + _visibility = settings.VisibilityTimeout; } public int InQueue diff --git a/src/K4os.Xpovoc.Sqs.Test/VeryLongRun.cs b/src/K4os.Xpovoc.Sqs.Test/VeryLongRun.cs index 6b0a331..0db0283 100644 --- a/src/K4os.Xpovoc.Sqs.Test/VeryLongRun.cs +++ b/src/K4os.Xpovoc.Sqs.Test/VeryLongRun.cs @@ -37,7 +37,7 @@ public VeryLongRun(ITestOutputHelper output) loggerFactory, queueFactory, new DefaultJobSerializer(), - new SqsJobQueueAdapterConfig { QueueName = queueName, JobConcurrency = 1 }, + new SqsJobQueueAdapterSettings { QueueName = queueName, JobConcurrency = 1 }, testTimeSource); _jobScheduler = new QueueJobScheduler( loggerFactory, adapter, this, testTimeSource); diff --git a/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterSettings.cs b/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterSettings.cs index 502ad7c..c8ca6db 100644 --- a/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterSettings.cs +++ b/src/K4os.Xpovoc.Sqs/ISqsJobQueueAdapterSettings.cs @@ -3,24 +3,24 @@ namespace K4os.Xpovoc.Sqs; -public interface ISqsJobQueueAdapterConfig +public interface ISqsJobQueueAdapterSettings { public string QueueName { get; } - int? PushConcurrency { get; set; } - int? PullConcurrency { get; set; } - int? ExecConcurrency { get; set; } - TimeSpan? RetryInterval { get; set; } - int? RetryCount { get; set; } + int PushConcurrency { get; set; } + int PullConcurrency { get; set; } + int JobConcurrency { get; set; } + TimeSpan RetryInterval { get; set; } + int RetryCount { get; set; } ISqsQueueSettings? QueueSettings { get; set; } } -public class SqsJobQueueAdapterConfig: ISqsJobQueueAdapterConfig +public class SqsJobQueueAdapterSettings: ISqsJobQueueAdapterSettings { - public string QueueName { get; set; } = null!; - public int? PushConcurrency { get; set; } - public int? PullConcurrency { get; set; } - public int? ExecConcurrency { get; set; } - public TimeSpan? RetryInterval { get; set; } - public int? RetryCount { get; set; } - public ISqsQueueSettings? QueueSettings { get; set; } + public string QueueName { get; set; } = "xpovoc-scheduler"; + public int PushConcurrency { get; set; } = 4; + public int PullConcurrency { get; set; } = 1; + public int JobConcurrency { get; set; } = 4; + public TimeSpan RetryInterval { get; set; } = TimeSpan.FromSeconds(1); + public int RetryCount { get; set; } = 0; + public ISqsQueueSettings? QueueSettings { get; set; } = new SqsQueueSettings(); } diff --git a/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueSettings.cs b/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueSettings.cs index c165e46..ef50d63 100644 --- a/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueSettings.cs +++ b/src/K4os.Xpovoc.Sqs/Internal/ISqsQueueSettings.cs @@ -4,8 +4,8 @@ namespace K4os.Xpovoc.Sqs.Internal; public interface ISqsQueueSettings { - int? ReceiveCount { get; } - TimeSpan? RetentionPeriod { get; } - TimeSpan? VisibilityTimeout { get; } - TimeSpan? ReceiveMessageWait { get; } + int ReceiveCount { get; } + TimeSpan RetentionPeriod { get; } + TimeSpan VisibilityTimeout { get; } + TimeSpan ReceiveMessageWait { get; } } diff --git a/src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs b/src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs index fa19ba7..aa60aff 100644 --- a/src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs +++ b/src/K4os.Xpovoc.Sqs/Internal/SqsQueueFactory.cs @@ -47,27 +47,22 @@ await TryCreateQueue(queueName, deadLetterUrl, settings) ?? queueSettings.AddIfNotNull( QueueAttributeName.MessageRetentionPeriod, - ToQueueAttribute( - settings.RetentionPeriod ?? SqsConstants.MaximumRetentionPeriod)); + ToQueueAttribute(settings.RetentionPeriod)); queueSettings.AddIfNotNull( QueueAttributeName.VisibilityTimeout, - ToQueueAttribute( - settings.VisibilityTimeout ?? SqsConstants.DefaultVisibilityTimeout)); + ToQueueAttribute(settings.VisibilityTimeout)); queueSettings.AddIfNotNull( QueueAttributeName.ReceiveMessageWaitTimeSeconds, - ToQueueAttribute( - settings.ReceiveMessageWait ?? SqsConstants.DefaultReceiveMessageWait)); + ToQueueAttribute(settings.ReceiveMessageWait)); if (deadLetterUrl is not null) { var deadLetterArn = await GetQueueArn(deadLetterUrl); queueSettings.Add( QueueAttributeName.RedrivePolicy, - ToRedrivePolicy( - deadLetterArn, - settings.ReceiveCount)); + ToRedrivePolicy(deadLetterArn, settings.ReceiveCount)); } try diff --git a/src/K4os.Xpovoc.Sqs/Internal/SqsQueueSettings.cs b/src/K4os.Xpovoc.Sqs/Internal/SqsQueueSettings.cs index b5b873c..fb55d7b 100644 --- a/src/K4os.Xpovoc.Sqs/Internal/SqsQueueSettings.cs +++ b/src/K4os.Xpovoc.Sqs/Internal/SqsQueueSettings.cs @@ -4,8 +4,8 @@ namespace K4os.Xpovoc.Sqs.Internal; public class SqsQueueSettings: ISqsQueueSettings { - public int? ReceiveCount { get; set; } - public TimeSpan? RetentionPeriod { get; set; } - public TimeSpan? VisibilityTimeout { get; set; } - public TimeSpan? ReceiveMessageWait { get; set; } + public int ReceiveCount { get; set; } = SqsConstants.MaximumReceiveCount; + public TimeSpan RetentionPeriod { get; set; } = SqsConstants.MaximumRetentionPeriod; + public TimeSpan VisibilityTimeout { get; set; } = SqsConstants.DefaultVisibilityTimeout; + public TimeSpan ReceiveMessageWait { get; set; } = SqsConstants.DefaultReceiveMessageWait; } \ No newline at end of file diff --git a/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs b/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs index c20ad2d..89a4dfb 100644 --- a/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs +++ b/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs @@ -9,6 +9,7 @@ using K4os.Async.Toys; using K4os.Xpovoc.Abstractions; using K4os.Xpovoc.Core.Queue; +using K4os.Xpovoc.Core.Sql; using K4os.Xpovoc.Sqs.Internal; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -23,8 +24,9 @@ namespace K4os.Xpovoc.Sqs; public class SqsJobQueueAdapter: IJobQueueAdapter { - private const int DefaultSqsConcurrency = 16; + private const int DefaultSqsConcurrency = 1; private const int DefaultJobConcurrency = 4; + private const int MinimumBatchConcurrency = 16; private static readonly TimeSpan DefaultRetryInterval = TimeSpan.FromSeconds(1); private static readonly TimeSpan MinimumRetryInterval = TimeSpan.Zero; @@ -35,7 +37,7 @@ public class SqsJobQueueAdapter: IJobQueueAdapter private static readonly SqsQueueSettings DefaultSqsQueueSettings = new(); private readonly IJobSerializer _serializer; - private readonly ISqsJobQueueAdapterConfig _config; + private readonly ISqsJobQueueAdapterSettings _settings; private readonly ISqsQueueFactory _factory; private readonly Task _ready; @@ -53,25 +55,25 @@ public SqsJobQueueAdapter( ILoggerFactory? loggerFactory, IAmazonSQS client, IJobSerializer serializer, - ISqsJobQueueAdapterConfig config): + ISqsJobQueueAdapterSettings settings): this( loggerFactory, new SqsQueueFactory(client), serializer, - config) { } + settings) { } internal SqsJobQueueAdapter( ILoggerFactory? loggerFactory, ISqsQueueFactory queueFactory, - IJobSerializer serializer, - ISqsJobQueueAdapterConfig config, + IJobSerializer? serializer = null, + ISqsJobQueueAdapterSettings? settings = null, ITimeSource? timeSource = null) { var log = Log = loggerFactory?.CreateLogger(GetType()) ?? NullLogger.Instance; _factory = queueFactory; - _config = config; - _serializer = serializer; + _settings = Validate(settings ?? new SqsJobQueueAdapterSettings()); + _serializer = serializer ?? new DefaultJobSerializer(); _visibility = SqsConstants.DefaultVisibilityTimeout; // might not be true // those are set in Startup @@ -82,6 +84,17 @@ internal SqsJobQueueAdapter( _ready = Task.Run(() => Startup(log, timeSource)); } + private static SqsJobQueueAdapterSettings Validate(ISqsJobQueueAdapterSettings settings) => + new() { + QueueName = settings.QueueName.Required(), + JobConcurrency = settings.JobConcurrency.NotLessThan(1), + PullConcurrency = settings.PullConcurrency.NotLessThan(1), + PushConcurrency = settings.PushConcurrency.NotLessThan(1), + RetryInterval = settings.RetryInterval.NotLessThan(MinimumRetryInterval), + RetryCount = settings.RetryCount.NotLessThan(0), + QueueSettings = settings.QueueSettings ?? new SqsQueueSettings(), + }; + private async Task Startup(ILogger log, ITimeSource? timeSource) { var timeAdapter = timeSource is null ? null : new ToysTimeSourceAdapter(timeSource); @@ -94,14 +107,11 @@ private async Task Startup(ILogger log, ITimeSource? timeSource) } private Task CreateQueue() => - _factory.Create( - _config.QueueName, - // ReSharper disable once SuspiciousTypeConversion.Global - _config.QueueSettings ?? _config as ISqsQueueSettings ?? DefaultSqsQueueSettings); + _factory.Create(_settings.QueueName, _settings.QueueSettings ?? DefaultSqsQueueSettings); private SqsBatchPublisher CreatePublisher(SqsBatchAdapter adapter, ITimeAdapter? timeAdapter) { - var concurrency = (_config.SqsConcurrency ?? DefaultSqsConcurrency).NotLessThan(1); + var concurrency = _settings.PushConcurrency; var batchSenderSettings = new BatchBuilderSettings { BatchDelay = TimeSpan.Zero, @@ -125,11 +135,12 @@ private BatchSubscriber CreateSubscriber( SqsBatchAdapter adapter, Func handler) { - var retryInterval = (_config.RetryInterval ?? DefaultRetryInterval).NotLessThan(MinimumRetryInterval); - var retryCount = (_config.RetryCount ?? DefaultRetryCount).NotLessThan(0); - var sqsConcurrency = (_config.SqsConcurrency ?? DefaultSqsConcurrency).NotLessThan(1); - var jobConcurrency = (_config.JobConcurrency ?? DefaultJobConcurrency).NotLessThan(1); - + var retryInterval = _settings.RetryInterval; + var retryCount = _settings.RetryCount; + var pullConcurrency = _settings.PullConcurrency; + var jobConcurrency = _settings.JobConcurrency; + var batchConcurrency = (pullConcurrency * 4).NotLessThan(MinimumBatchConcurrency); + var touchInterval = CalculateTouchInterval(_visibility, retryInterval, retryCount); var touchDelay = CalculateTouchDelay(_visibility, touchInterval, retryInterval, retryCount); var subscriber = new BatchSubscriber( @@ -138,8 +149,10 @@ private BatchSubscriber CreateSubscriber( new BatchSubscriberSettings { AlternateBatches = true, AsynchronousDeletes = true, - BatchConcurrency = sqsConcurrency, + PollerCount = pullConcurrency, + InternalQueueSize = 1, HandlerCount = jobConcurrency, + BatchConcurrency = batchConcurrency, RetryInterval = retryInterval, RetryLimit = retryCount, TouchInterval = touchInterval, @@ -264,7 +277,6 @@ private async Task HandleMessage( } } - #warning double dispose is killing it (most likely BatchBuilder) public void Dispose() { _ready.Await(); // it is safe to finish async initialization diff --git a/src/Playground/InfiniteScheduling.cs b/src/Playground/InfiniteScheduling.cs index c20e057..b7c2b53 100644 --- a/src/Playground/InfiniteScheduling.cs +++ b/src/Playground/InfiniteScheduling.cs @@ -175,13 +175,16 @@ private static void ConfigureRedis(ServiceCollection serviceCollection, XDocumen private static void ConfigureSqs(ServiceCollection serviceCollection, XDocument secrets) { serviceCollection.AddSingleton( - p => new AmazonSQSClient( - new AmazonSQSConfig { ServiceURL = "http://localhost:9324", })); - serviceCollection.AddSingleton( - p => new SqsJobQueueAdapterConfig { + p => new AmazonSQSClient(new AmazonSQSConfig())); +// serviceCollection.AddSingleton( +// p => new AmazonSQSClient( +// new AmazonSQSConfig { ServiceURL = "http://localhost:9324", })); + serviceCollection.AddSingleton( + p => new SqsJobQueueAdapterSettings { QueueName = "xpovoc-playground", JobConcurrency = 16, - SqsConcurrency = 16, + PullConcurrency = 16, + PushConcurrency = 16, }); serviceCollection.AddSingleton( p => new SqsJobQueueAdapter( @@ -189,7 +192,7 @@ private static void ConfigureSqs(ServiceCollection serviceCollection, XDocument // p.GetRequiredService(), p.GetRequiredService(), p.GetRequiredService(), - p.GetRequiredService())); + p.GetRequiredService())); serviceCollection.AddSingleton( p => new QueueJobScheduler( NullLoggerFactory.Instance, diff --git a/src/Playground/Program.cs b/src/Playground/Program.cs index 13b1c19..f7d6ab7 100644 --- a/src/Playground/Program.cs +++ b/src/Playground/Program.cs @@ -11,7 +11,7 @@ var loggerFactory = new SerilogLoggerFactory(); var sqsClient = new AmazonSQSClient(); -// await InfiniteScheduling.Run(loggerFactory); +await InfiniteScheduling.Run(loggerFactory); -var test = new TrueDeferredJobs(sqsClient, loggerFactory, TimeSpan.FromHours(1)); -await test.RunAsync(); +// var test = new TrueDeferredJobs(sqsClient, loggerFactory, TimeSpan.FromHours(1)); +// await test.RunAsync(); diff --git a/src/Playground/SqsThroughput.cs b/src/Playground/SqsThroughput.cs index 7daafa9..0cd026e 100644 --- a/src/Playground/SqsThroughput.cs +++ b/src/Playground/SqsThroughput.cs @@ -33,9 +33,10 @@ public static async Task Run(ILoggerFactory loggerFactory) loggerFactory, client, new DefaultJobSerializer(), - new SqsJobQueueAdapterConfig { + new SqsJobQueueAdapterSettings { QueueName = QueueName, - SqsConcurrency = 16, + PushConcurrency = 16, + PullConcurrency = 4, JobConcurrency = 16, }); diff --git a/src/Playground/TrueDeferredJobs.cs b/src/Playground/TrueDeferredJobs.cs index ee1ea43..604437b 100644 --- a/src/Playground/TrueDeferredJobs.cs +++ b/src/Playground/TrueDeferredJobs.cs @@ -38,7 +38,7 @@ public async Task RunAsync() new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.All, }), - new SqsJobQueueAdapterConfig { + new SqsJobQueueAdapterSettings { QueueName = "xpovoc-TrueDeferredJobs", JobConcurrency = 16, }); From 4447fff95f9c1964bce10091ca64cf4d6669d6d7 Mon Sep 17 00:00:00 2001 From: Milosz Krajewski Date: Sat, 18 Nov 2023 16:30:17 +0000 Subject: [PATCH 4/5] confirage nmber of producers --- src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs | 5 ----- src/Playground/InfiniteScheduling.cs | 21 +++++++++++---------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs b/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs index 89a4dfb..756df85 100644 --- a/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs +++ b/src/K4os.Xpovoc.Sqs/SqsJobQueueAdapter.cs @@ -24,13 +24,8 @@ namespace K4os.Xpovoc.Sqs; public class SqsJobQueueAdapter: IJobQueueAdapter { - private const int DefaultSqsConcurrency = 1; - private const int DefaultJobConcurrency = 4; private const int MinimumBatchConcurrency = 16; - - private static readonly TimeSpan DefaultRetryInterval = TimeSpan.FromSeconds(1); private static readonly TimeSpan MinimumRetryInterval = TimeSpan.Zero; - private const int DefaultRetryCount = 0; protected ILogger Log { get; } diff --git a/src/Playground/InfiniteScheduling.cs b/src/Playground/InfiniteScheduling.cs index b7c2b53..6a2fc3d 100644 --- a/src/Playground/InfiniteScheduling.cs +++ b/src/Playground/InfiniteScheduling.cs @@ -174,17 +174,17 @@ private static void ConfigureRedis(ServiceCollection serviceCollection, XDocumen private static void ConfigureSqs(ServiceCollection serviceCollection, XDocument secrets) { - serviceCollection.AddSingleton( - p => new AmazonSQSClient(new AmazonSQSConfig())); // serviceCollection.AddSingleton( -// p => new AmazonSQSClient( -// new AmazonSQSConfig { ServiceURL = "http://localhost:9324", })); +// p => new AmazonSQSClient(new AmazonSQSConfig())); + serviceCollection.AddSingleton( + p => new AmazonSQSClient( + new AmazonSQSConfig { ServiceURL = "http://localhost:9324", })); serviceCollection.AddSingleton( p => new SqsJobQueueAdapterSettings { QueueName = "xpovoc-playground", JobConcurrency = 16, PullConcurrency = 16, - PushConcurrency = 16, + PushConcurrency = 64, }); serviceCollection.AddSingleton( p => new SqsJobQueueAdapter( @@ -215,11 +215,12 @@ private static async Task Execute( // var producer = Task.CompletedTask; // var producerSpeed = Task.CompletedTask; var producer = Task.WhenAll( - Task.Run(() => Producer(token, scheduler), token), - Task.Run(() => Producer(token, scheduler), token), - Task.Run(() => Producer(token, scheduler), token), - Task.Run(() => Producer(token, scheduler), token) - ); + Enumerable + .Range(0, 4) + .Select(_ => Task.Run(() => Producer(token, scheduler), token)) + .ToArray() + ); + var producerSpeed = Task.Run( () => Measure( token, From 7787d037a557137bad93f6250542006204f9ebeb Mon Sep 17 00:00:00 2001 From: Milosz Krajewski Date: Sat, 18 Nov 2023 21:51:51 +0000 Subject: [PATCH 5/5] tweaking playground --- CHANGES.md | 3 +++ src/Playground/InfiniteScheduling.cs | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a83bb23..bfc727b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,6 @@ +## 0.3.4 (2023/11/18) +* ADDED: SQS as scheduler + ## 0.3.3 (2023/01/10) * ADDED: Redis storage * YAK-SHAVED: build scripts diff --git a/src/Playground/InfiniteScheduling.cs b/src/Playground/InfiniteScheduling.cs index 6a2fc3d..a99e5d2 100644 --- a/src/Playground/InfiniteScheduling.cs +++ b/src/Playground/InfiniteScheduling.cs @@ -33,7 +33,7 @@ namespace Playground; internal static class InfiniteScheduling { private static readonly int ProduceDelay = 0; - private static readonly int ConsumeDelay = 0; + private static readonly int ConsumeDelay = 0; // 10*1000; private static readonly int ConsumeThreads = 4; private static readonly bool EnablePruning = true; @@ -183,8 +183,8 @@ private static void ConfigureSqs(ServiceCollection serviceCollection, XDocument p => new SqsJobQueueAdapterSettings { QueueName = "xpovoc-playground", JobConcurrency = 16, - PullConcurrency = 16, - PushConcurrency = 64, + PullConcurrency = 1, + PushConcurrency = 16, }); serviceCollection.AddSingleton( p => new SqsJobQueueAdapter(