From b690a1ea1cfb8c6a0214c9827e44da8d8e365697 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 12 Dec 2023 13:51:52 +0900 Subject: [PATCH 1/4] Make `AliasName` static --- osu.ElasticIndexer/AppSettings.cs | 2 ++ osu.ElasticIndexer/Commands/Index/CloseIndexCommand.cs | 3 ++- osu.ElasticIndexer/Commands/Index/DeleteIndexCommand.cs | 2 +- osu.ElasticIndexer/Commands/Index/ListIndicesCommand.cs | 4 ++-- osu.ElasticIndexer/Commands/Index/NukeAllIndicesCommand.cs | 2 +- osu.ElasticIndexer/Commands/Index/UpdateAliasCommand.cs | 2 +- osu.ElasticIndexer/Commands/Queue/WatchQueueCommand.cs | 2 +- osu.ElasticIndexer/OsuElasticClient.cs | 6 ++---- osu.ElasticIndexer/SoloScoreIndexer.cs | 4 ++-- 9 files changed, 14 insertions(+), 13 deletions(-) diff --git a/osu.ElasticIndexer/AppSettings.cs b/osu.ElasticIndexer/AppSettings.cs index f7c7d5f..bd7a5b9 100644 --- a/osu.ElasticIndexer/AppSettings.cs +++ b/osu.ElasticIndexer/AppSettings.cs @@ -40,5 +40,7 @@ static AppSettings() public static string RedisHost { get; } public static string Schema { get; } + + public static string AliasName { get; } = $"{AppSettings.Prefix}scores"; } } diff --git a/osu.ElasticIndexer/Commands/Index/CloseIndexCommand.cs b/osu.ElasticIndexer/Commands/Index/CloseIndexCommand.cs index e9c33f8..1720bb5 100644 --- a/osu.ElasticIndexer/Commands/Index/CloseIndexCommand.cs +++ b/osu.ElasticIndexer/Commands/Index/CloseIndexCommand.cs @@ -20,7 +20,8 @@ public override int OnExecute(CancellationToken token) if (base.OnExecute(token) != 0) return -1; - var indices = string.IsNullOrEmpty(Name) ? ElasticClient.GetIndices($"{ElasticClient.AliasName}_*") : ElasticClient.GetIndex(Name); + var indices = string.IsNullOrEmpty(Name) ? ElasticClient.GetIndices($"{AppSettings.AliasName}_*") : ElasticClient.GetIndex(Name); + var closeableIndices = indices.Where(entry => entry.Value.Aliases.Count == 0); if (!closeableIndices.Any()) diff --git a/osu.ElasticIndexer/Commands/Index/DeleteIndexCommand.cs b/osu.ElasticIndexer/Commands/Index/DeleteIndexCommand.cs index 2ad3b17..7d9504c 100644 --- a/osu.ElasticIndexer/Commands/Index/DeleteIndexCommand.cs +++ b/osu.ElasticIndexer/Commands/Index/DeleteIndexCommand.cs @@ -21,7 +21,7 @@ public override int OnExecute(CancellationToken token) if (base.OnExecute(token) != 0) return -1; - string? index = string.IsNullOrEmpty(Name) ? $"{elasticClient.AliasName}_*" : Name; + string? index = string.IsNullOrEmpty(Name) ? $"{AppSettings.AliasName}_*" : Name; var response = elasticClient.Cat.Indices(x => x.Index(index)); var closed = response.Records.Where(record => record.Status == "close"); diff --git a/osu.ElasticIndexer/Commands/Index/ListIndicesCommand.cs b/osu.ElasticIndexer/Commands/Index/ListIndicesCommand.cs index f75ddc3..205e761 100644 --- a/osu.ElasticIndexer/Commands/Index/ListIndicesCommand.cs +++ b/osu.ElasticIndexer/Commands/Index/ListIndicesCommand.cs @@ -30,7 +30,7 @@ public virtual int OnExecute(CancellationToken token) Console.WriteLine($"Current schema: {currentSchema}"); Console.WriteLine(); - var indices = ElasticClient.GetIndices($"{ElasticClient.AliasName}_*"); + var indices = ElasticClient.GetIndices($"{AppSettings.AliasName}_*"); var records = ElasticClient.Cat.Indices(descriptor => descriptor.Index(Indices.All)).Records; Console.WriteLine($"# Elasticsearch indices ({indices.Count})"); @@ -65,7 +65,7 @@ public virtual int OnExecute(CancellationToken token) return -1; } - if (!currentIndex.Aliases.ContainsKey(ElasticClient.AliasName)) + if (!currentIndex.Aliases.ContainsKey(AppSettings.AliasName)) { Console.WriteLine(ConsoleColor.Red, "ERROR: Current schema is not aliased correctly"); return -1; diff --git a/osu.ElasticIndexer/Commands/Index/NukeAllIndicesCommand.cs b/osu.ElasticIndexer/Commands/Index/NukeAllIndicesCommand.cs index 98650db..2316038 100644 --- a/osu.ElasticIndexer/Commands/Index/NukeAllIndicesCommand.cs +++ b/osu.ElasticIndexer/Commands/Index/NukeAllIndicesCommand.cs @@ -12,7 +12,7 @@ public class NukeAllIndicesCommand : ListIndicesCommand { public override int OnExecute(CancellationToken token) { - string prefix = $"{ElasticClient.AliasName}_*"; + string prefix = $"{AppSettings.AliasName}_*"; // Don't exit on error as we're likely trying to fix an error. base.OnExecute(token); diff --git a/osu.ElasticIndexer/Commands/Index/UpdateAliasCommand.cs b/osu.ElasticIndexer/Commands/Index/UpdateAliasCommand.cs index d380962..ec55960 100644 --- a/osu.ElasticIndexer/Commands/Index/UpdateAliasCommand.cs +++ b/osu.ElasticIndexer/Commands/Index/UpdateAliasCommand.cs @@ -40,7 +40,7 @@ public override int OnExecute(CancellationToken token) // TODO: should check if completed? string? indexName = index.Value.Key.Name; - ElasticClient.UpdateAlias(ElasticClient.AliasName, indexName, Close); + ElasticClient.UpdateAlias(AppSettings.AliasName, indexName, Close); Redis.SetCurrentSchema(indexName); return 0; diff --git a/osu.ElasticIndexer/Commands/Queue/WatchQueueCommand.cs b/osu.ElasticIndexer/Commands/Queue/WatchQueueCommand.cs index 94eae1f..4e57582 100644 --- a/osu.ElasticIndexer/Commands/Queue/WatchQueueCommand.cs +++ b/osu.ElasticIndexer/Commands/Queue/WatchQueueCommand.cs @@ -14,7 +14,7 @@ public class WatchQueueCommand public int OnExecute(CancellationToken token) { string currentIndex = RedisAccess.GetConnection().GetCurrentSchema(); - string proposedIndex = $"{new OsuElasticClient().AliasName}_{AppSettings.Schema}"; + string proposedIndex = $"{AppSettings.AliasName}_{AppSettings.Schema}"; if (string.IsNullOrEmpty(currentIndex)) Console.WriteLine(ConsoleColor.Yellow, "WARNING: No current schema set, will set new schema as current"); diff --git a/osu.ElasticIndexer/OsuElasticClient.cs b/osu.ElasticIndexer/OsuElasticClient.cs index f7b9686..7b9d07f 100644 --- a/osu.ElasticIndexer/OsuElasticClient.cs +++ b/osu.ElasticIndexer/OsuElasticClient.cs @@ -13,8 +13,6 @@ namespace osu.ElasticIndexer { public class OsuElasticClient : ElasticClient { - public readonly string AliasName = $"{AppSettings.Prefix}scores"; - public OsuElasticClient(bool throwsExceptions = true) : base(new ConnectionSettings(new Uri(AppSettings.ElasticsearchHost)) .EnableApiVersioningHeader() @@ -55,7 +53,7 @@ public IReadOnlyDictionary GetIndices(string name, Expand public KeyValuePair? GetIndexForSchema(string schema) { - KeyValuePair? index = GetIndices($"{AliasName}_{schema}") + KeyValuePair? index = GetIndices($"{AppSettings.AliasName}_{schema}") .SingleOrDefault(); if (string.IsNullOrEmpty(index?.Key?.Name)) @@ -90,7 +88,7 @@ public void UpdateAlias(string alias, string index, bool close = true) private string createIndex(string schema) { - string name = $"{AliasName}_{schema}"; + string name = $"{AppSettings.AliasName}_{schema}"; Console.WriteLine(ConsoleColor.Cyan, $"Creating new index `{name}`."); diff --git a/osu.ElasticIndexer/SoloScoreIndexer.cs b/osu.ElasticIndexer/SoloScoreIndexer.cs index 6f4f3f2..da6f035 100644 --- a/osu.ElasticIndexer/SoloScoreIndexer.cs +++ b/osu.ElasticIndexer/SoloScoreIndexer.cs @@ -61,11 +61,11 @@ private void checkSchema() return; // schema has changed to the current one - if (previousSchema != schema && schema == $"{elasticClient.AliasName}_{AppSettings.Schema}") + if (previousSchema != schema && schema == $"{AppSettings.AliasName}_{AppSettings.Schema}") { Console.WriteLine(ConsoleColor.Yellow, $"Schema switched to current: {schema}"); previousSchema = schema; - elasticClient.UpdateAlias(elasticClient.AliasName, indexName); + elasticClient.UpdateAlias(AppSettings.AliasName, indexName); return; } From ebd44ac2f73341a2be798665e0d839f3e1f910ee Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 12 Dec 2023 13:59:43 +0900 Subject: [PATCH 2/4] Use direct queue name --- osu.ElasticIndexer/IndexQueueProcessor.cs | 4 +--- osu.ElasticIndexer/UnrunnableProcessor.cs | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/osu.ElasticIndexer/IndexQueueProcessor.cs b/osu.ElasticIndexer/IndexQueueProcessor.cs index b2a4df4..3df0d52 100644 --- a/osu.ElasticIndexer/IndexQueueProcessor.cs +++ b/osu.ElasticIndexer/IndexQueueProcessor.cs @@ -12,8 +12,6 @@ namespace osu.ElasticIndexer { public class IndexQueueProcessor : QueueProcessor { - private static readonly string queue_name = $"score-index-{AppSettings.Schema}"; - private readonly OsuElasticClient elasticClient; private readonly string index; @@ -24,7 +22,7 @@ public class IndexQueueProcessor : QueueProcessor internal IndexQueueProcessor(string index, OsuElasticClient elasticClient, Action stopCallback) : base(new QueueConfiguration { - InputQueueName = queue_name, + InputQueueName = $"{AppSettings.AliasName}_{AppSettings.Schema}", BatchSize = AppSettings.BatchSize, ErrorThreshold = AppSettings.BatchSize * 2, // needs to be larger than BatchSize to handle ES busy errors. MaxInFlightItems = AppSettings.BatchSize * AppSettings.BufferSize diff --git a/osu.ElasticIndexer/UnrunnableProcessor.cs b/osu.ElasticIndexer/UnrunnableProcessor.cs index c460985..892e2e9 100644 --- a/osu.ElasticIndexer/UnrunnableProcessor.cs +++ b/osu.ElasticIndexer/UnrunnableProcessor.cs @@ -8,10 +8,8 @@ namespace osu.ElasticIndexer { public class UnrunnableProcessor : QueueProcessor { - private static readonly string queue_name = $"score-index-{AppSettings.Schema}"; - internal UnrunnableProcessor() - : base(new QueueConfiguration { InputQueueName = queue_name }) + : base(new QueueConfiguration { InputQueueName = $"{AppSettings.AliasName}_{AppSettings.Schema}" }) { if (string.IsNullOrEmpty(AppSettings.Schema)) throw new MissingSchemaException(); From c4ffc232fc02657ee8fc8c81528b1a963b1ca1a9 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 12 Dec 2023 23:39:22 +0900 Subject: [PATCH 3/4] Ensure prefix is initialised before forming `AliasName` --- osu.ElasticIndexer/AppSettings.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/osu.ElasticIndexer/AppSettings.cs b/osu.ElasticIndexer/AppSettings.cs index bd7a5b9..00ce94b 100644 --- a/osu.ElasticIndexer/AppSettings.cs +++ b/osu.ElasticIndexer/AppSettings.cs @@ -21,6 +21,7 @@ static AppSettings() Prefix = Environment.GetEnvironmentVariable("ES_INDEX_PREFIX") ?? string.Empty; ElasticsearchHost = Environment.GetEnvironmentVariable("ES_HOST") ?? "http://localhost:9200"; RedisHost = Environment.GetEnvironmentVariable("REDIS_HOST") ?? "localhost"; + AliasName = $"{Prefix}scores"; if (string.IsNullOrEmpty(Schema)) Console.WriteLine(ConsoleColor.Yellow, "WARNING: Running without SCHEMA envvar specification"); if (!string.IsNullOrEmpty(Prefix)) Console.WriteLine(ConsoleColor.Yellow, $"WARNING: Running with PREFIX envvar specification ({Prefix})"); @@ -41,6 +42,6 @@ static AppSettings() public static string Schema { get; } - public static string AliasName { get; } = $"{AppSettings.Prefix}scores"; + public static string AliasName { get; } } } From 9aa157cb3886d241c50b74af7309c0720a393ba6 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 12 Dec 2023 23:40:16 +0900 Subject: [PATCH 4/4] Remove unused `RedisHost` setting --- osu.ElasticIndexer/AppSettings.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/osu.ElasticIndexer/AppSettings.cs b/osu.ElasticIndexer/AppSettings.cs index 00ce94b..e85785b 100644 --- a/osu.ElasticIndexer/AppSettings.cs +++ b/osu.ElasticIndexer/AppSettings.cs @@ -20,7 +20,6 @@ static AppSettings() Schema = Environment.GetEnvironmentVariable("SCHEMA") ?? string.Empty; Prefix = Environment.GetEnvironmentVariable("ES_INDEX_PREFIX") ?? string.Empty; ElasticsearchHost = Environment.GetEnvironmentVariable("ES_HOST") ?? "http://localhost:9200"; - RedisHost = Environment.GetEnvironmentVariable("REDIS_HOST") ?? "localhost"; AliasName = $"{Prefix}scores"; if (string.IsNullOrEmpty(Schema)) Console.WriteLine(ConsoleColor.Yellow, "WARNING: Running without SCHEMA envvar specification"); @@ -38,8 +37,6 @@ static AppSettings() public static string Prefix { get; } - public static string RedisHost { get; } - public static string Schema { get; } public static string AliasName { get; }