Skip to content

Commit

Permalink
merge v0.3.2
Browse files Browse the repository at this point in the history
  • Loading branch information
@tmm360
tmm360 committed Nov 19, 2024
2 parents a970bb8 + 4be9637 commit c775e60
Show file tree
Hide file tree
Showing 14 changed files with 334 additions and 519 deletions.
3 changes: 3 additions & 0 deletions EthernaGatewayCli.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,16 @@ EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug-DevEnv|Any CPU = Debug-DevEnv|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{F348B863-6892-45AD-AC33-3AD833A14BBB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F348B863-6892-45AD-AC33-3AD833A14BBB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F348B863-6892-45AD-AC33-3AD833A14BBB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F348B863-6892-45AD-AC33-3AD833A14BBB}.Release|Any CPU.Build.0 = Release|Any CPU
{F348B863-6892-45AD-AC33-3AD833A14BBB}.Debug-DevEnv|Any CPU.ActiveCfg = Debug-DevEnv|Any CPU
{F348B863-6892-45AD-AC33-3AD833A14BBB}.Debug-DevEnv|Any CPU.Build.0 = Debug-DevEnv|Any CPU
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{6DD03B58-3404-41AD-B84D-E43E600C54F3} = {D3B7E18B-46F8-4072-9D14-801FDB15868C}
Expand Down
123 changes: 82 additions & 41 deletions src/EthernaGatewayCli/Commands/Etherna/Chunk/UploadCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,38 +16,34 @@
using Etherna.CliHelper.Models.Commands;
using Etherna.CliHelper.Services;
using Etherna.GatewayCli.Services;
using Etherna.Sdk.Users.Gateway.Services;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.WebSockets;
using System.Reflection;
using System.Threading;
using System.Text;
using System.Threading.Tasks;

namespace Etherna.GatewayCli.Commands.Etherna.Chunk
{
public class UploadCommand : CommandBase<UploadCommandOptions>
public class UploadCommand(
Assembly assembly,
IAuthenticationService authService,
IGatewayService gatewayService,
IIoService ioService,
IPostageBatchService postageBatchService,
IServiceProvider serviceProvider)
: CommandBase<UploadCommandOptions>(assembly, ioService, serviceProvider)
{
// Consts.
private ushort ChunkBatchMaxSize = 500;
private const int UploadMaxRetry = 10;
private readonly TimeSpan UploadRetryTimeSpan = TimeSpan.FromSeconds(5);

// Fields.
private readonly IAuthenticationService authService;
private readonly IGatewayService gatewayService;

// Constructor.
public UploadCommand(
Assembly assembly,
IAuthenticationService authService,
IGatewayService gatewayService,
IIoService ioService,
IServiceProvider serviceProvider)
: base(assembly, ioService, serviceProvider)
{
this.authService = authService;
this.gatewayService = gatewayService;
}

// Properties.
public override string CommandArgsHelpString => "CHUNK_DIR";
public override string Description => "Upload chunks from a directory";
Expand Down Expand Up @@ -77,53 +73,98 @@ protected override async Task ExecuteAsync(string[] commandArgs)
var batchDepth = postageBuckets.RequiredPostageBatchDepth;

// Identify postage batch and tag to use.
var postageBatchId = await gatewayService.GetUsablePostageBatchIdAsync(
var batchId = await postageBatchService.GetUsablePostageBatchAsync(
batchDepth,
Options.UsePostageBatchId is null ? (PostageBatchId?)null : new PostageBatchId(Options.UsePostageBatchId),
Options.NewPostageTtl,
Options.NewPostageAutoPurchase,
Options.UsePostageBatchId is null ? (PostageBatchId?)null : new PostageBatchId(Options.UsePostageBatchId),
Options.NewPostageLabel);
var tagInfo = await gatewayService.CreateTagAsync(postageBatchId); //necessary to not bypass bee local storage

// Upload with websocket.

IoService.WriteLine($"Start uploading {chunkFiles.Length} chunks...");

int totalUploaded = 0;
for (int i = 0; i < UploadMaxRetry && totalUploaded < chunkFiles.Length; i++)
var uploadStartDateTime = DateTime.UtcNow;
for (int retry = 0; retry < UploadMaxRetry && totalUploaded < chunkFiles.Length; retry++)
{
// Create websocket.
using var chunkUploaderWs = await gatewayService.GetChunkUploaderWebSocketAsync(postageBatchId, tagInfo.Id);

try
{
for (int j = totalUploaded; j < chunkFiles.Length; j++)
var lastUpdateDateTime = DateTime.UtcNow;

// Iterate on chunk batches.
while(totalUploaded < chunkFiles.Length)
{
var chunkPath = chunkFiles[j];
var chunk = SwarmChunk.BuildFromSpanAndData(
Path.GetFileNameWithoutExtension(chunkPath),
await File.ReadAllBytesAsync(chunkPath));
var now = DateTime.UtcNow;
if (now - lastUpdateDateTime > TimeSpan.FromSeconds(1))
{
PrintProgressLine(
"Uploading chunks",
totalUploaded,
chunkFiles.Length,
uploadStartDateTime);
lastUpdateDateTime = now;
}

var chunkBatchFiles = chunkFiles.Skip(totalUploaded).Take(ChunkBatchMaxSize).ToArray();

List<SwarmChunk> chunkBatch = [];
foreach (var chunkFile in chunkBatchFiles)
chunkBatch.Add(SwarmChunk.BuildFromSpanAndData(
Path.GetFileNameWithoutExtension(chunkFile),
await File.ReadAllBytesAsync(chunkFile)));

await chunkUploaderWs.SendChunkAsync(chunk, CancellationToken.None);
await gatewayService.ChunksBulkUploadAsync(
chunkBatch.ToArray(),
batchId);
retry = 0;

totalUploaded++;
totalUploaded += chunkBatchFiles.Length;
}
IoService.WriteLine();
}
catch (Exception e) when (e is WebSocketException or OperationCanceledException)
catch (Exception e) when (
e is HttpRequestException
or InvalidOperationException
or WebSocketException
or OperationCanceledException)
{
IoService.WriteErrorLine($"Error uploading chunks");
IoService.WriteLine(e.ToString());

if (i + 1 < UploadMaxRetry)
if (retry + 1 < UploadMaxRetry)
{
Console.WriteLine("Retry...");
await Task.Delay(UploadRetryTimeSpan);
}
}
finally
{
await chunkUploaderWs.CloseAsync();
}
}

IoService.WriteLine($"Uploaded {totalUploaded} chunks successfully.");
}

private void PrintProgressLine(string message, long uploadedChunks, long totalChunks, DateTime startDateTime)
{
// Calculate ETA.
var elapsedTime = DateTime.UtcNow - startDateTime;
TimeSpan? eta = null;
var progressStatus = (double)uploadedChunks / totalChunks;
if (progressStatus != 0)
{
var totalRequiredTime = TimeSpan.FromSeconds(elapsedTime.TotalSeconds / progressStatus);
eta = totalRequiredTime - elapsedTime;
}

// Print update.
var strBuilder = new StringBuilder();

strBuilder.Append(CultureInfo.InvariantCulture,
$"{message} ({(progressStatus * 100):N2}%) {uploadedChunks} chunks of {totalChunks}.");

if (eta is not null)
strBuilder.Append(CultureInfo.InvariantCulture, $" ETA: {eta:hh\\:mm\\:ss}");

strBuilder.Append('\r');

IoService.Write(strBuilder.ToString());
}
}
}
1 change: 1 addition & 0 deletions src/EthernaGatewayCli/Commands/Etherna/Chunk/UploadCommandOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
// You should have received a copy of the GNU Affero General Public License along with Etherna Gateway CLI.
// If not, see <https://www.gnu.org/licenses/>.

using Etherna.BeeNet.Models;
using Etherna.CliHelper.Models.Commands;
using Etherna.CliHelper.Models.Commands.OptionRequirements;
using System;
Expand Down
35 changes: 16 additions & 19 deletions src/EthernaGatewayCli/Commands/Etherna/Postage/CreateCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,31 +16,21 @@
using Etherna.CliHelper.Models.Commands;
using Etherna.CliHelper.Services;
using Etherna.GatewayCli.Services;
using Etherna.Sdk.Users.Gateway.Services;
using System;
using System.Reflection;
using System.Threading.Tasks;

namespace Etherna.GatewayCli.Commands.Etherna.Postage
{
public class CreateCommand : CommandBase<CreateCommandOptions>
public class CreateCommand(
Assembly assembly,
IAuthenticationService authService,
IGatewayService gatewayService,
IIoService ioService,
IServiceProvider serviceProvider)
: CommandBase<CreateCommandOptions>(assembly, ioService, serviceProvider)
{
// Fields.
private readonly IAuthenticationService authService;
private readonly IGatewayService gatewayService;

// Constructor.
public CreateCommand(
Assembly assembly,
IAuthenticationService authService,
IGatewayService gatewayService,
IIoService ioService,
IServiceProvider serviceProvider)
: base(assembly, ioService, serviceProvider)
{
this.authService = authService;
this.gatewayService = gatewayService;
}

// Properties.
public override string Description => "Create a new postage batch";

Expand All @@ -66,7 +56,14 @@ protected override async Task ExecuteAsync(string[] commandArgs)
}
else throw new InvalidOperationException("Amount or TTL are required");

var batchId = await gatewayService.CreatePostageBatchAsync(amount, Options.Depth, Options.Label);
var batchId = await gatewayService.CreatePostageBatchAsync(
amount,
Options.Depth,
Options.Label,
onWaitingBatchCreation: () => IoService.Write("Waiting for batch created... (it may take a while)"),
onBatchCreated: _ => IoService.WriteLine(". Done"),
onWaitingBatchUsable: () => IoService.Write("Waiting for batch being usable... (it may take a while)"),
onBatchUsable: () => IoService.WriteLine(". Done"));

// Print result.
IoService.WriteLine();
Expand Down
26 changes: 8 additions & 18 deletions src/EthernaGatewayCli/Commands/Etherna/Postage/InfoCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,21 @@
using Etherna.CliHelper.Services;
using Etherna.GatewayCli.Services;
using Etherna.Sdk.Gateway.GenClients;
using Etherna.Sdk.Users.Gateway.Services;
using System;
using System.Reflection;
using System.Text.Json;
using System.Threading.Tasks;

namespace Etherna.GatewayCli.Commands.Etherna.Postage
{
public class InfoCommand : CommandBase
public class InfoCommand(
Assembly assembly,
IAuthenticationService authService,
IGatewayService gatewayService,
IIoService ioService,
IServiceProvider serviceProvider)
: CommandBase(assembly, ioService, serviceProvider)
{
// Consts.
private static readonly JsonSerializerOptions SerializerOptions = new()
Expand All @@ -38,23 +45,6 @@ public class InfoCommand : CommandBase
},
WriteIndented = true
};

// Fields.
private readonly IAuthenticationService authService;
private readonly IGatewayService gatewayService;

// Constructor.
public InfoCommand(
Assembly assembly,
IAuthenticationService authService,
IGatewayService gatewayService,
IIoService ioService,
IServiceProvider serviceProvider)
: base(assembly, ioService, serviceProvider)
{
this.authService = authService;
this.gatewayService = gatewayService;
}

// Properties.
public override string Description => "Get info about a postage batch";
Expand Down
26 changes: 8 additions & 18 deletions src/EthernaGatewayCli/Commands/Etherna/Resource/FundCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,31 +15,21 @@
using Etherna.CliHelper.Models.Commands;
using Etherna.CliHelper.Services;
using Etherna.GatewayCli.Services;
using Etherna.Sdk.Users.Gateway.Services;
using System;
using System.Reflection;
using System.Threading.Tasks;

namespace Etherna.GatewayCli.Commands.Etherna.Resource
{
public class FundCommand : CommandBase<FundCommandOptions>
public class FundCommand(
Assembly assembly,
IAuthenticationService authService,
IGatewayService gatewayService,
IIoService ioService,
IServiceProvider serviceProvider)
: CommandBase<FundCommandOptions>(assembly, ioService, serviceProvider)
{
// Fields.
private readonly IAuthenticationService authService;
private readonly IGatewayService gatewayService;

// Constructor.
public FundCommand(
Assembly assembly,
IAuthenticationService authService,
IGatewayService gatewayService,
IIoService ioService,
IServiceProvider serviceProvider)
: base(assembly, ioService, serviceProvider)
{
this.authService = authService;
this.gatewayService = gatewayService;
}

// Properties.
public override string CommandArgsHelpString => "RESOURCE_ID";
public override string Description => "Fund resource budget";
Expand Down
Loading

0 comments on commit c775e60

Please sign in to comment.