Skip to content

Commit

Permalink
feat: add Outputs extraction to taskDumper
Browse files Browse the repository at this point in the history
  • Loading branch information
@ereali-aneo
ereali-aneo committed Sep 2, 2024
1 parent 53977af commit 4837ad6
Show file tree
Hide file tree
Showing 5 changed files with 174 additions and 30 deletions.
2 changes: 1 addition & 1 deletion src/TaskDumper/ArmoniK.TaskDumper.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="ArmoniK.Api.Client" Version="3.19.0" />
<PackageReference Include="ArmoniK.Api.Client" Version="3.20.0-edge.20.f49315d" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
Expand Down
134 changes: 134 additions & 0 deletions src/TaskDumper/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
using System.CommandLine;
using System.IO;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;

Expand All @@ -26,11 +27,16 @@
using ArmoniK.Api.Client.Submitter;
using ArmoniK.Api.gRPC.V1;
using ArmoniK.Api.gRPC.V1.Results;
using ArmoniK.Api.gRPC.V1.SortDirection;
using ArmoniK.Api.gRPC.V1.Tasks;
using ArmoniK.Api.gRPC.V1.Worker;

using Microsoft.Extensions.Configuration;

using FilterField = ArmoniK.Api.gRPC.V1.Tasks.FilterField;
using Filters = ArmoniK.Api.gRPC.V1.Tasks.Filters;
using FiltersAnd = ArmoniK.Api.gRPC.V1.Tasks.FiltersAnd;

namespace ArmoniK.TaskDumper;

internal static class Program
Expand Down Expand Up @@ -143,6 +149,134 @@ await resultClient.DownloadResultData(taskResponse.Task.SessionId,
taskResponse.Task.PayloadId,
CancellationToken.None) ?? Encoding.ASCII.GetBytes(""));
}

// Found all results createdBy taskId
var taskCreated = await taskClient.ListTasksAsync(new ListTasksRequest
{
Filters = new Filters
{
Or =
{
new FiltersAnd
{
And =
{
new FilterField
{
FilterString = new FilterString
{
Operator = FilterStringOperator.Equal,
Value = taskId,
},
Field = new TaskField
{
TaskSummaryField = new TaskSummaryField
{
Field = TaskSummaryEnumField.CreatedBy,
},
},
},
},
},
},
},
Sort = new ListTasksRequest.Types.Sort
{
Direction = SortDirection.Asc,
Field = new TaskField
{
TaskSummaryField = new TaskSummaryField
{
Field = TaskSummaryEnumField.TaskId,
},
},
},
PageSize = 1,
Page = 0,
});

if (!Directory.Exists(folder + Path.DirectorySeparatorChar + "Outputs" + Path.DirectorySeparatorChar + "Results"))
{
Directory.CreateDirectory(folder + Path.DirectorySeparatorChar + "Outputs" + Path.DirectorySeparatorChar + "Results");
}

var outputs = new OutputsData();
foreach (var task in taskCreated.Tasks)
{
outputs.Tasks[task.Id] = task;
using (var tw = new StreamWriter(folder + Path.DirectorySeparatorChar + "Outputs" + Path.DirectorySeparatorChar + task.Id + ".json",
false))
{
await tw.WriteLineAsync(task.ToString());
}
}

var resultsCreated = await resultClient.ListResultsAsync(new ListResultsRequest
{
Filters = new Api.gRPC.V1.Results.Filters
{
Or =
{
new Api.gRPC.V1.Results.FiltersAnd
{
And =
{
new Api.gRPC.V1.Results.FilterField
{
FilterString = new FilterString
{
Operator = FilterStringOperator.Equal,
Value = taskId,
},
Field = new ResultField
{
ResultRawField = new ResultRawField
{
Field = ResultRawEnumField.CreatedBy,
},
},
},
},
},
},
},
Sort = new ListResultsRequest.Types.Sort
{
Direction = SortDirection.Asc,
Field = new ResultField
{
ResultRawField = new ResultRawField
{
Field = ResultRawEnumField.ResultId,
},
},
},
Page = 0,
PageSize = 1,
});


foreach (var result in resultsCreated.Results)
{
outputs.Results[result.ResultId] = result;
await File.WriteAllBytesAsync(Path.Combine(folder + Path.DirectorySeparatorChar + "Outputs" + Path.DirectorySeparatorChar + "Results",
result.ResultId),
await resultClient.DownloadResultData(taskResponse.Task.SessionId,
result.ResultId,
CancellationToken.None) ?? Encoding.ASCII.GetBytes(""));
}

using (var tw = new StreamWriter(folder + Path.DirectorySeparatorChar + "Outputs" + Path.DirectorySeparatorChar + "Results.json",
false))
{
await tw.WriteLineAsync(JsonSerializer.Serialize(outputs.Results));
}

using (var tw = new StreamWriter(folder + Path.DirectorySeparatorChar + "Outputs" + Path.DirectorySeparatorChar + "Tasks.json",
false))
{
await tw.WriteLineAsync(JsonSerializer.Serialize(outputs.Tasks));
}
}

public static async Task<int> Main(string[] args)
Expand Down
2 changes: 1 addition & 1 deletion src/TaskReRunner/ArmoniK.TaskReRunner.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="ArmoniK.Api.Core" Version="3.18.1" />
<PackageReference Include="ArmoniK.Api.Core" Version="3.20.0-edge.20.f49315d" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
<PackageReference Include="Serilog.Settings.Configuration" Version="8.0.1" />
Expand Down
10 changes: 10 additions & 0 deletions src/TaskReRunner/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,16 @@ public static void Run(string path)
logger_.LogInformation("Submitted Task Data : {task}",
task.Value);
}

// recover results create a folder taskrerunner
// comparer nombres de results
// comparer name si name ou id identique comparer binaire de maniere intelligente genre diff
// comparer nombre de task creer
// comparer payload des task si meme id
// mettre dans bon dossier
// question est ce que le result Id et task iD peut etre controler via option ?
// creer deuxieme json with all get result , get task de la tache rejouer ?
// package nuget don't support caps ?
}

public static async Task<int> Main(string[] args)
Expand Down
56 changes: 28 additions & 28 deletions test/ReRunnerAgentTest/ReRunnerAgentTest.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,28 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
<PackageReference Include="NUnit" Version="3.14.0" />
<PackageReference Include="NUnit.Analyzers" Version="3.9.0" />
<PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\src\TaskReRunner\ArmoniK.TaskReRunner.csproj" />
</ItemGroup>

<ItemGroup>
<Using Include="NUnit.Framework" />
</ItemGroup>

</Project>
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
<PackageReference Include="NUnit" Version="3.14.0" />
<PackageReference Include="NUnit.Analyzers" Version="3.9.0" />
<PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\src\TaskReRunner\ArmoniK.TaskReRunner.csproj" />
</ItemGroup>

<ItemGroup>
<Using Include="NUnit.Framework" />
</ItemGroup>

</Project>

0 comments on commit 4837ad6

Please sign in to comment.