Skip to content

Commit 32b75e3

Browse files
ReubenBondReubenBond
authored
Fix load score calculation in ResourceOptimizedPlacementDirector (#9611)
Fix environment statistics normalization in ResourceOptimizedPlacementDirector
1 parent 65199ad commit 32b75e3

File tree

14 files changed

+621
-217
lines changed

14 files changed

+621
-217
lines changed

‎Orleans.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Orleans.CodeGenerator.Tests
270270
EndProject
271271
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ActivationSheddingToy", "playground\ActivationSheddingToy\ActivationSheddingToy.csproj", "{67697CD2-ABA6-4DC8-BCDA-CE31632CA31E}"
272272
EndProject
273+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ActivationRebalancing.AppHost", "playground\ActivationRebalancing\ActivationRebalancing.AppHost\ActivationRebalancing.AppHost.csproj", "{7D8A2688-7117-492B-8357-0DFC0EDCF012}"
274+
EndProject
273275
Global
274276
GlobalSection(SolutionConfigurationPlatforms) = preSolution
275277
Debug|Any CPU = Debug|Any CPU
@@ -708,6 +710,10 @@ Global
708710
{67697CD2-ABA6-4DC8-BCDA-CE31632CA31E}.Debug|Any CPU.Build.0 = Debug|Any CPU
709711
{67697CD2-ABA6-4DC8-BCDA-CE31632CA31E}.Release|Any CPU.ActiveCfg = Release|Any CPU
710712
{67697CD2-ABA6-4DC8-BCDA-CE31632CA31E}.Release|Any CPU.Build.0 = Release|Any CPU
713+
{7D8A2688-7117-492B-8357-0DFC0EDCF012}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
714+
{7D8A2688-7117-492B-8357-0DFC0EDCF012}.Debug|Any CPU.Build.0 = Debug|Any CPU
715+
{7D8A2688-7117-492B-8357-0DFC0EDCF012}.Release|Any CPU.ActiveCfg = Release|Any CPU
716+
{7D8A2688-7117-492B-8357-0DFC0EDCF012}.Release|Any CPU.Build.0 = Release|Any CPU
711717
EndGlobalSection
712718
GlobalSection(SolutionProperties) = preSolution
713719
HideSolutionNode = FALSE
@@ -838,6 +844,7 @@ Global
838844
{E613A10D-757D-44BA-97C1-3D06C22BDB2E} = {4C5D66BF-EE1C-4DD8-8551-D1B7F3768A34}
839845
{AE0BB9EA-FC9C-428A-B8F1-501798CA4B50} = {A6573187-FD0D-4DF7-91D1-03E07E470C0A}
840846
{67697CD2-ABA6-4DC8-BCDA-CE31632CA31E} = {A41DE3D1-F8AA-4234-BE6F-3C9646A1507A}
847+
{7D8A2688-7117-492B-8357-0DFC0EDCF012} = {B0DC8B8D-29CD-4CA3-A874-471F75595829}
841848
EndGlobalSection
842849
GlobalSection(ExtensibilityGlobals) = postSolution
843850
SolutionGuid = {7BFB3429-B5BB-4DB1-95B4-67D77A864952}

‎playground/ActivationRebalancing/ActivationRebalancing.AppHost/ActivationRebalancing.AppHost.csproj‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<Sdk Name="Aspire.AppHost.Sdk" Version="9.0.0" />
4+
5+
<PropertyGroup>
6+
<OutputType>Exe</OutputType>
7+
<TargetFramework>net8.0</TargetFramework>
8+
<ImplicitUsings>enable</ImplicitUsings>
9+
<Nullable>enable</Nullable>
10+
<IsAspireHost>true</IsAspireHost>
11+
<UserSecretsId>6e52d7b6-4f1f-4e1c-abef-df2fe839e7f2</UserSecretsId>
12+
</PropertyGroup>
13+
14+
<ItemGroup>
15+
<PackageReference Include="Aspire.Hosting.AppHost" />
16+
<PackageReference Include="Aspire.Hosting.Orleans" />
17+
<PackageReference Include="Aspire.Hosting.Redis" />
18+
</ItemGroup>
19+
20+
<ItemGroup>
21+
<ProjectReference Include="..\ActivationRebalancing.Cluster\ActivationRebalancing.Cluster.csproj" />
22+
<ProjectReference Include="..\ActivationRebalancing.Frontend\ActivationRebalancing.Frontend.csproj" />
23+
</ItemGroup>
24+
25+
</Project>

‎playground/ActivationRebalancing/ActivationRebalancing.AppHost/Program.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
var builder = DistributedApplication.CreateBuilder(args);
2+
3+
var redis = builder.AddRedis("orleans-redis");
4+
5+
var orleans = builder.AddOrleans("cluster")
6+
.WithClustering(redis);
7+
8+
var backend = builder.AddProject<Projects.ActivationRebalancing_Cluster>("backend")
9+
.WithReference(orleans)
10+
.WaitFor(redis)
11+
.WithReplicas(5);
12+
13+
builder.AddProject<Projects.ActivationRebalancing_Frontend>("frontend")
14+
.WithReference(orleans.AsClient())
15+
.WaitFor(backend)
16+
.WithReplicas(1);
17+
18+
builder.Build().Run();

‎playground/ActivationRebalancing/ActivationRebalancing.AppHost/Properties/launchSettings.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"$schema": "https://json.schemastore.org/launchsettings.json",
3+
"profiles": {
4+
"https": {
5+
"commandName": "Project",
6+
"dotnetRunMessages": true,
7+
"launchBrowser": true,
8+
"applicationUrl": "https://localhost:17267;http://localhost:15094",
9+
"environmentVariables": {
10+
"ASPNETCORE_ENVIRONMENT": "Development",
11+
"DOTNET_ENVIRONMENT": "Development",
12+
"DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21125",
13+
"DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22078"
14+
}
15+
},
16+
"http": {
17+
"commandName": "Project",
18+
"dotnetRunMessages": true,
19+
"launchBrowser": true,
20+
"applicationUrl": "http://localhost:15094",
21+
"environmentVariables": {
22+
"ASPNETCORE_ENVIRONMENT": "Development",
23+
"DOTNET_ENVIRONMENT": "Development",
24+
"DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19108",
25+
"DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20187"
26+
}
27+
}
28+
}
29+
}

‎playground/ActivationRebalancing/ActivationRebalancing.AppHost/appsettings.Development.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"Logging": {
3+
"LogLevel": {
4+
"Default": "Information",
5+
"Microsoft.AspNetCore": "Warning"
6+
}
7+
}
8+
}

‎playground/ActivationRebalancing/ActivationRebalancing.AppHost/appsettings.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"Logging": {
3+
"LogLevel": {
4+
"Default": "Information",
5+
"Microsoft.AspNetCore": "Warning",
6+
"Aspire.Hosting.Dcp": "Warning"
7+
}
8+
}
9+
}

‎playground/ActivationRebalancing/ActivationRebalancing.Cluster/ActivationRebalancing.Cluster.csproj‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,13 @@
77
<OrleansBuildTimeCodeGen>true</OrleansBuildTimeCodeGen>
88
</PropertyGroup>
99

10+
<ItemGroup>
11+
<PackageReference Include="Aspire.StackExchange.Redis" />
12+
</ItemGroup>
13+
1014
<ItemGroup>
1115
<ProjectReference Include="..\..\..\src\Orleans.Server\Orleans.Server.csproj" />
16+
<ProjectReference Include="..\..\..\src\Redis\Orleans.Clustering.Redis\Orleans.Clustering.Redis.csproj" />
1217
</ItemGroup>
1318

1419
</Project>

‎playground/ActivationRebalancing/ActivationRebalancing.Cluster/Program.cs‎

Lines changed: 35 additions & 145 deletions
Original file line numberDiff line numberDiff line change
@@ -1,170 +1,60 @@
1-
using System.Diagnostics;
2-
using System.Net;
31
using Microsoft.Extensions.DependencyInjection;
42
using Microsoft.Extensions.Hosting;
53
using Microsoft.Extensions.Logging;
64
using Orleans.Configuration;
7-
using Orleans.Runtime.Placement;
85

9-
#nullable enable
10-
11-
// Ledjon: The silos will run in the same process so they will have the same memory usage.
12-
// I previously had 4 console apps to run the example, but didn't want to add so many proj into the solution.
13-
// I am sure with something like Aspire that would be easier, but for now I'll leave them like this.
14-
// You (the reader) feel free to run this in different processes for a more realistic example.
15-
16-
var host0 = await StartSiloHost(0);
17-
var host1 = await StartSiloHost(1);
18-
var host2 = await StartSiloHost(2);
19-
var host3 = await StartSiloHost(3);
20-
IHost? host5 = null;
21-
22-
Console.WriteLine("All silos have started.");
23-
24-
var grainFactory = host0.Services.GetRequiredService<IGrainFactory>();
25-
var mgmtGrain = grainFactory.GetGrain<IManagementGrain>(0);
26-
27-
var silos = await mgmtGrain.GetHosts(onlyActive: true);
28-
Debug.Assert(silos.Count == 4);
29-
var addresses = silos.Select(x => x.Key).ToArray();
30-
31-
var tasks = new List<Task>();
32-
RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[0]);
33-
for (var i = 0; i < 300; i++)
34-
{
35-
tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
36-
}
37-
38-
RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[1]);
39-
for (var i = 0; i < 30; i++)
40-
{
41-
tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
42-
}
43-
44-
RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[2]);
45-
for (var i = 0; i < 410; i++)
46-
{
47-
tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
48-
}
49-
50-
RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[3]);
51-
for (var i = 0; i < 120; i++)
52-
{
53-
tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
54-
}
55-
56-
var sessionCount = 0;
57-
while (true)
58-
{
59-
if (sessionCount == 25)
6+
var builder = Host.CreateApplicationBuilder(args);
7+
builder.AddKeyedRedisClient("orleans-redis");
8+
builder.Logging.AddFilter("Orleans.Runtime.Placement.Rebalancing", LogLevel.Trace);
9+
#pragma warning disable ORLEANSEXP002
10+
builder.UseOrleans(builder => builder
11+
.Configure<GrainCollectionOptions>(o =>
6012
{
61-
RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[0]);
62-
for (var i = 0; i < 50; i++)
63-
{
64-
tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
65-
}
66-
67-
RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[1]);
68-
for (var i = 0; i < 50; i++)
69-
{
70-
tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
71-
}
72-
}
73-
74-
if (sessionCount == 35)
13+
o.CollectionQuantum = TimeSpan.FromSeconds(15);
14+
})
15+
.Configure<ResourceOptimizedPlacementOptions>(o =>
7516
{
76-
RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[1]);
77-
for (var i = 0; i < 50; i++)
78-
{
79-
tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
80-
}
17+
o.LocalSiloPreferenceMargin = 0;
18+
})
19+
.Configure<ActivationRebalancerOptions>(o =>
20+
{
21+
o.RebalancerDueTime = TimeSpan.FromSeconds(5);
22+
o.SessionCyclePeriod = TimeSpan.FromSeconds(5);
23+
// uncomment these below, if you want higher migration rate
24+
//o.CycleNumberWeight = 1;
25+
//o.SiloNumberWeight = 0;
26+
})
27+
.AddActivationRebalancer());
28+
#pragma warning restore ORLEANSEXP002
8129

82-
RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[2]);
83-
for (var i = 0; i < 50; i++)
84-
{
85-
tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
86-
}
87-
}
30+
builder.Services.AddHostedService<LoadDriverBackgroundService>();
31+
var app = builder.Build();
8832

89-
if (sessionCount == 40)
90-
{
91-
host5 = await StartSiloHost(4);
92-
}
33+
await app.RunAsync();
9334

94-
if (sessionCount == 45)
35+
internal class LoadDriverBackgroundService(IGrainFactory client) : BackgroundService
36+
{
37+
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
9538
{
96-
RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[2]);
97-
for (var i = 0; i < 50; i++)
39+
while (!stoppingToken.IsCancellationRequested)
9840
{
99-
tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
100-
}
41+
for (var i = 0; i < 5 * Random.Shared.Next(1, 1000); i++)
42+
{
43+
await client.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping();
44+
}
10145

102-
RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[3]);
103-
for (var i = 0; i < 50; i++)
104-
{
105-
tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
46+
await Task.Delay(Random.Shared.Next(500, 1_000), stoppingToken);
10647
}
10748
}
108-
109-
await Task.Delay(5000); // session duration
110-
sessionCount++;
111-
112-
if (sessionCount > 55)
113-
{
114-
break;
115-
}
116-
}
117-
118-
Console.WriteLine("Simulation has finished. Press Enter to terminate...");
119-
Console.ReadLine();
120-
121-
await host0.StopAsync();
122-
await host1.StopAsync();
123-
await host2.StopAsync();
124-
await host3.StopAsync();
125-
126-
if (host5 != null)
127-
{
128-
await host5.StopAsync();
129-
}
130-
131-
static async Task<IHost> StartSiloHost(int num)
132-
{
133-
#pragma warning disable ORLEANSEXP002
134-
var host = Host.CreateDefaultBuilder()
135-
.ConfigureLogging(builder => builder
136-
.AddFilter("", LogLevel.Error)
137-
.AddFilter("Orleans.Runtime.Placement.Rebalancing", LogLevel.Trace)
138-
.AddConsole())
139-
.UseOrleans(builder => builder
140-
.Configure<ActivationRebalancerOptions>(o =>
141-
{
142-
o.RebalancerDueTime = TimeSpan.FromSeconds(5);
143-
o.SessionCyclePeriod = TimeSpan.FromSeconds(5);
144-
// uncomment these below, if you want higher migration rate
145-
//o.CycleNumberWeight = 1;
146-
//o.SiloNumberWeight = 0;
147-
})
148-
.UseLocalhostClustering(
149-
siloPort: EndpointOptions.DEFAULT_SILO_PORT + num,
150-
gatewayPort: EndpointOptions.DEFAULT_GATEWAY_PORT + num,
151-
primarySiloEndpoint: new IPEndPoint(IPAddress.Loopback, EndpointOptions.DEFAULT_SILO_PORT))
152-
.AddActivationRebalancer())
153-
.Build();
154-
#pragma warning restore ORLEANSEXP002
155-
156-
await host.StartAsync();
157-
Console.WriteLine($"Silo{num} started.");
158-
159-
return host;
16049
}
16150

16251
public interface IRebalancingTestGrain : IGrainWithGuidKey
16352
{
16453
Task Ping();
16554
}
16655

56+
[CollectionAgeLimit(Minutes = 0.5)]
16757
public class RebalancingTestGrain : Grain, IRebalancingTestGrain
16858
{
16959
public Task Ping() => Task.CompletedTask;
170-
}
60+
}

‎playground/ActivationRebalancing/ActivationRebalancing.Frontend/ActivationRebalancing.Frontend.csproj‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,17 @@
44
<TargetFramework>net8.0</TargetFramework>
55
<Nullable>enable</Nullable>
66
<ImplicitUsings>enable</ImplicitUsings>
7+
<OrleansBuildTimeCodeGen>true</OrleansBuildTimeCodeGen>
78
</PropertyGroup>
89

10+
<ItemGroup>
11+
<PackageReference Include="Aspire.StackExchange.Redis" />
12+
</ItemGroup>
13+
914
<ItemGroup>
1015
<ProjectReference Include="..\..\..\src\Orleans.Server\Orleans.Server.csproj" />
1116
<ProjectReference Include="..\ActivationRebalancing.Cluster\ActivationRebalancing.Cluster.csproj" />
17+
<ProjectReference Include="..\..\..\src\Redis\Orleans.Clustering.Redis\Orleans.Clustering.Redis.csproj" />
1218
</ItemGroup>
1319

1420
</Project>

‎playground/ActivationRebalancing/ActivationRebalancing.Frontend/Program.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22

33
var builder = WebApplication.CreateBuilder(args);
44

5-
builder.UseOrleansClient(clientBuilder => clientBuilder.UseLocalhostClustering());
5+
builder.AddKeyedRedisClient("orleans-redis");
6+
builder.UseOrleansClient();
67
builder.Services.AddControllers();
78

89
var app = builder.Build();

0 commit comments

Comments
 (0)