Skip to content

Commit

Permalink
feat: read CSV and map it to an strongly typed object (#43)
Browse files Browse the repository at this point in the history 
* feat: read CSV and map it to an strongly typed object
* ci: include download of SDK 9.0
  • Loading branch information
@Seddryck
Seddryck authored Nov 30, 2024
1 parent dd38624 commit 8ba1859
Show file tree
Hide file tree
Showing 19 changed files with 2,376 additions and 240 deletions.
110 changes: 55 additions & 55 deletions Directory.Build.props
View file
Original file line number Diff line number Diff line change
@@ -1,75 +1,75 @@
<Project>
<PropertyGroup>
<!-- By default every projects are packable except Testing, Benchmark, Profiler projects-->
<IsPackable>true</IsPackable>
<IsPackable Condition="$(MSBuildProjectName.EndsWith('Testing'))">false</IsPackable>
<IsPackable Condition="$(MSBuildProjectName.EndsWith('Benchmark'))">false</IsPackable>
<IsPackable Condition="$(MSBuildProjectName.EndsWith('Profiler'))">false</IsPackable>
<DebugType>portable</DebugType>
</PropertyGroup>
<PropertyGroup>
<!-- By default every projects are packable except Testing, Benchmark, Profiler projects-->
<IsPackable>true</IsPackable>
<IsPackable Condition="$(MSBuildProjectName.EndsWith('Testing'))">false</IsPackable>
<IsPackable Condition="$(MSBuildProjectName.EndsWith('Benchmark'))">false</IsPackable>
<IsPackable Condition="$(MSBuildProjectName.EndsWith('Profiler'))">false</IsPackable>
<DebugType>portable</DebugType>
</PropertyGroup>

<PropertyGroup>
<TargetFrameworks>net6.0;net7.0;net8.0</TargetFrameworks>
<Platform>AnyCPU</Platform>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>12.0</LangVersion>
</PropertyGroup>
<PropertyGroup>
<TargetFrameworks>net8.0;net9.0</TargetFrameworks>
<Platform>AnyCPU</Platform>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>13.0</LangVersion>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="DotNet.ReproducibleBuilds" Version="1.1.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="DotNet.ReproducibleBuilds" Version="1.1.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference>
</ItemGroup>


<PropertyGroup>
<RestoreProjectStyle>PackageReference</RestoreProjectStyle>
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
</PropertyGroup>

<PropertyGroup>
<Version>0.0.0</Version>
<Authors>Cédric L. Charlier</Authors>
<Owners>Seddryck</Owners>
<Company>nbiguity</Company>
<PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
<PackageProjectUrl>https://github.com/Seddrycl/PocketCsvReader</PackageProjectUrl>
<RepositoryType>git</RepositoryType>
<RequireLicenseAcceptance>false</RequireLicenseAcceptance>
<PackageIcon>icon\pocket-csv-reader.png</PackageIcon>
<SymbolPackageFormat Condition=" '$(DebugType)' != 'embedded' ">snupkg</SymbolPackageFormat>
<PackageReadmeFile>README.md</PackageReadmeFile>
</PropertyGroup>
<PropertyGroup>
<Version>0.0.0</Version>
<Authors>Cédric L. Charlier</Authors>
<Owners>Seddryck</Owners>
<Company>nbiguity</Company>
<PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
<PackageProjectUrl>https://github.com/Seddrycl/PocketCsvReader</PackageProjectUrl>
<RepositoryType>git</RepositoryType>
<RequireLicenseAcceptance>false</RequireLicenseAcceptance>
<PackageIcon>icon\pocket-csv-reader.png</PackageIcon>
<SymbolPackageFormat Condition=" '$(DebugType)' != 'embedded' ">snupkg</SymbolPackageFormat>
<PackageReadmeFile>README.md</PackageReadmeFile>
</PropertyGroup>

<ItemGroup>
<None Include="..\pocket-csv-reader.png" Pack="true" PackagePath="icon\" />
<None Include="..\README.md" Pack="true" PackagePath="\" />
</ItemGroup>
<ItemGroup>
<None Include="..\pocket-csv-reader.png" Pack="true" PackagePath="icon\" />
<None Include="..\README.md" Pack="true" PackagePath="\" />
</ItemGroup>

<PropertyGroup>
<GenerateAssemblyInfo>true</GenerateAssemblyInfo>
</PropertyGroup>

<PropertyGroup>
<!-- disable warning when XML comments are missing -->
<NoWarn>$(NoWarn);CS1591</NoWarn>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)'=='Release' AND $(IsPackable) == true">
<GeneratePackageOnBuild>false</GeneratePackageOnBuild>
<Deterministic>true</Deterministic>
</PropertyGroup>
<PropertyGroup>
<!-- disable warning when XML comments are missing -->
<NoWarn>$(NoWarn);CS1591</NoWarn>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)'=='Release' AND $(IsPackable) == true">
<GeneratePackageOnBuild>false</GeneratePackageOnBuild>
<Deterministic>true</Deterministic>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)|$(TargetFramework)|$(Platform)'=='Debug|net6.0|AnyCPU'">
<WarningLevel>5</WarningLevel>
<LangVersion>preview</LangVersion>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(TargetFramework)|$(Platform)'=='Debug|net6.0|AnyCPU'">
<WarningLevel>5</WarningLevel>
<LangVersion>preview</LangVersion>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)|$(TargetFramework)|$(Platform)'=='Release|net6.0|AnyCPU'">
<WarningLevel>5</WarningLevel>
<LangVersion>preview</LangVersion>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(TargetFramework)|$(Platform)'=='Release|net6.0|AnyCPU'">
<WarningLevel>5</WarningLevel>
<LangVersion>preview</LangVersion>
</PropertyGroup>

</Project>
2 changes: 1 addition & 1 deletion PocketCsvReader.Benchmark/ToDataReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ public CustomConfig()

// Create a specific job for each version
AddJob(Job.Default
.WithRuntime(CoreRuntime.Core80)
.WithRuntime(CoreRuntime.Core90)
.WithWarmupCount(1) // 1 warm-up iteration
.WithIterationCount(5) // 5 actual iterations
); // Identify the job by version
Expand Down
120 changes: 120 additions & 0 deletions PocketCsvReader.Testing/CsvObjectReaderTest.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
using PocketCsvReader;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Text;
using System.Reflection;
using System.Security.Cryptography;
using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces;

namespace PocketCsvReader.Testing;

[TestFixture]
public class CsvObjectReaderTest
{
private static MemoryStream CreateStream(string content)
{
byte[] byteArray = Encoding.UTF8.GetBytes(content);
MemoryStream stream = new MemoryStream(byteArray);
stream.Position = 0;
return stream;
}

private record struct Human(string Name, bool IsAdult);
[Test]
public void GetString_SingleFieldAttemptForSecond_Throws()
{
var spanMapper = new SpanMapper<Human>((span, fieldSpans) =>
{
return new Human(
span.Slice(fieldSpans.First().Start, fieldSpans.First().Length).ToString(),
int.Parse(span.Slice(fieldSpans.Last().Start, fieldSpans.Last().Length).ToString()) > 18);
});

var profile = new CsvProfile(',', '\"', "\r\n", false);
using var stream = CreateStream("foo,16\r\nbar,21");
using var dataReader = new CsvObjectReader<Human>(stream, profile, spanMapper);

var humans = dataReader.Read().ToArray();
Assert.That(humans, Has.Length.EqualTo(2));
Assert.That(humans[0].Name, Is.EqualTo("foo"));
Assert.That(humans[0].IsAdult, Is.False);
Assert.That(humans[1].Name, Is.EqualTo("bar"));
Assert.That(humans[1].IsAdult, Is.True);
}

private record struct Financial(
int Year, int Month, int Day, DateTime DateTime,
string ResolutionCode, string Status, string AreaCode, string AreaTypeCode, string AreaName, string MapCode,
decimal Expenses, decimal Income, string Currency, DateTime UpdateTime);
[Test]
[TestCase("Ansi")]
[TestCase("Utf16-BE")]
[TestCase("Utf16-LE")]
[TestCase("Utf8-BOM")]
[TestCase("Utf8")]
public void Read_FinancialWithCompleteParsers_CorrectRowsColumns(string filename)
{
var profile = new CsvProfile('\t', '\"', "\r\n", true);

using (var stream =
Assembly.GetExecutingAssembly()
.GetManifestResourceStream($"{Assembly.GetExecutingAssembly().GetName().Name}.Resources.{filename}.csv")
?? throw new FileNotFoundException()
)
{
var spanMapper = new SpanMapper<Financial>((span, fieldSpans) =>
{
return new Financial(
int.Parse(span.Slice(fieldSpans.ElementAt(0).Start, fieldSpans.ElementAt(0).Length)),
int.Parse(span.Slice(fieldSpans.ElementAt(1).Start, fieldSpans.ElementAt(1).Length)),
int.Parse(span.Slice(fieldSpans.ElementAt(2).Start, fieldSpans.ElementAt(2).Length)),
DateTime.Parse(span.Slice(fieldSpans.ElementAt(3).Start, fieldSpans.ElementAt(3).Length)),
span.Slice(fieldSpans.ElementAt(4).Start, fieldSpans.ElementAt(4).Length).ToString(),
span.Slice(fieldSpans.ElementAt(5).Start, fieldSpans.ElementAt(5).Length).ToString(),
span.Slice(fieldSpans.ElementAt(6).Start, fieldSpans.ElementAt(6).Length).ToString(),
span.Slice(fieldSpans.ElementAt(7).Start, fieldSpans.ElementAt(7).Length).ToString(),
span.Slice(fieldSpans.ElementAt(8).Start, fieldSpans.ElementAt(8).Length).ToString(),
span.Slice(fieldSpans.ElementAt(9).Start, fieldSpans.ElementAt(9).Length).ToString(),
decimal.Parse(span.Slice(fieldSpans.ElementAt(10).Start, fieldSpans.ElementAt(10).Length)),
decimal.Parse(span.Slice(fieldSpans.ElementAt(11).Start, fieldSpans.ElementAt(11).Length)),
span.Slice(fieldSpans.ElementAt(12).Start, fieldSpans.ElementAt(12).Length).ToString(),
DateTime.Parse(span.Slice(fieldSpans.ElementAt(13).Start, fieldSpans.ElementAt(13).Length)));
});
var rowCount = 0;
using var dataReader = new CsvObjectReader<Financial>(stream, profile, spanMapper);
foreach(var human in dataReader.Read())
{ Console.WriteLine($"{rowCount++}: {human.AreaCode}"); }
Assert.That(rowCount, Is.EqualTo(21));
}
}

[Test]
[TestCase("Ansi")]
[TestCase("Utf16-BE")]
[TestCase("Utf16-LE")]
[TestCase("Utf8-BOM")]
[TestCase("Utf8")]
public void Read_FinancialWithSpanObjectBuilder_CorrectRowsColumns(string filename)
{
var profile = new CsvProfile('\t', '\"', "\r\n", true);

using (var stream =
Assembly.GetExecutingAssembly()
.GetManifestResourceStream($"{Assembly.GetExecutingAssembly().GetName().Name}.Resources.{filename}.csv")
?? throw new FileNotFoundException()
)
{
var objBuilder = new SpanObjectBuilder<Financial>();
var spanMapper = new SpanMapper<Financial>(objBuilder.Instantiate);
var rowCount = 0;
using var dataReader = new CsvObjectReader<Financial>(stream, profile, spanMapper);
foreach (var human in dataReader.Read())
{ rowCount++; }
Assert.That(rowCount, Is.EqualTo(21));
}
}
}
35 changes: 35 additions & 0 deletions PocketCsvReader.Testing/CsvReaderTest.cs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.VisualBasic;
using NUnit.Framework;

namespace PocketCsvReader.Testing;
Expand Down Expand Up @@ -78,4 +80,37 @@ public void ToArrayOfString_PackageAssetStream_Successful(string filename)
var arrays = new CsvReader(profile).ToArrayString(stream);
Assert.That(arrays.Count, Is.EqualTo(1695));
}

private record struct PackageAsset(
string Guid, DateTimeOffset Created, string Name, string Version,
DateTimeOffset Updated, string Description, string Runtime,
string Field1, string Field2, string Field3, string Field4, string Field5,
string Field6, string Field7, string Field8, string Field9, string Field10,
string Field11, string Field12, string Field13, string Field14, string Field15,
string Field16, string Field17, string Field18
);

[Test]
[TestCase(@"Resources\PackageAssets.csv")]
public void ToObjectWithSpanMapper_PackageAssetStream_Successful(string filename)
{
var objBuilder = new SpanObjectBuilder<PackageAsset>();
objBuilder.SetParser<DateTimeOffset>(s => DateTimeOffset.Parse(s, CultureInfo.InvariantCulture));
var spanMapper = new SpanMapper<PackageAsset>(objBuilder.Instantiate);

using var stream = File.OpenRead(filename);
var profile = new CsvProfile(',', '\"', Environment.NewLine, false);
var arrays = new CsvReader(profile).To(stream, spanMapper);
Assert.That(arrays.Count, Is.EqualTo(1695));
}

[Test]
[TestCase(@"Resources\PackageAssets.csv")]
public void ToObject_PackageAssetStream_Successful(string filename)
{
using var stream = File.OpenRead(filename);
var profile = new CsvProfile(',', '\"', Environment.NewLine, false);
var arrays = new CsvReader(profile).To<PackageAsset>(stream);
Assert.That(arrays.Count, Is.EqualTo(1695));
}
}
Loading

0 comments on commit 8ba1859

Please sign in to comment.