From 8ba1859b3a6302a039d6e49ecba1699ba4138ac7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20L=2E=20Charlier?= Date: Sat, 30 Nov 2024 17:21:08 +0100 Subject: [PATCH] feat: read CSV and map it to an strongly typed object (#43) * feat: read CSV and map it to an strongly typed object * ci: include download of SDK 9.0 --- Directory.Build.props | 110 +- PocketCsvReader.Benchmark/ToDataReader.cs | 2 +- .../CsvObjectReaderTest.cs | 120 ++ PocketCsvReader.Testing/CsvReaderTest.cs | 35 + PocketCsvReader.Testing/FieldParserTest.cs | 56 +- PocketCsvReader.Testing/RecordParserTTest.cs | 98 + PocketCsvReader.Testing/RecordParserTest.cs | 36 +- .../SpanObjectBuilderTests.cs | 63 + ...{FieldParser.cs => ArrayOfStringMapper.cs} | 30 +- PocketCsvReader/CsvArrayString.cs | 6 +- PocketCsvReader/CsvObjectReader.cs | 84 + PocketCsvReader/CsvReader.cs | 30 + PocketCsvReader/FieldSpan.cs | 14 + PocketCsvReader/RecordParser.T.cs | 36 + PocketCsvReader/RecordParser.cs | 207 +-- PocketCsvReader/SpanMapper.cs | 8 + PocketCsvReader/SpanObjectBuilder.cs | 67 + appveyor.yml | 11 +- dotnet-install.ps1 | 1603 +++++++++++++++++ 19 files changed, 2376 insertions(+), 240 deletions(-) create mode 100644 PocketCsvReader.Testing/CsvObjectReaderTest.cs create mode 100644 PocketCsvReader.Testing/RecordParserTTest.cs create mode 100644 PocketCsvReader.Testing/SpanObjectBuilderTests.cs rename PocketCsvReader/{FieldParser.cs => ArrayOfStringMapper.cs} (72%) create mode 100644 PocketCsvReader/CsvObjectReader.cs create mode 100644 PocketCsvReader/FieldSpan.cs create mode 100644 PocketCsvReader/RecordParser.T.cs create mode 100644 PocketCsvReader/SpanMapper.cs create mode 100644 PocketCsvReader/SpanObjectBuilder.cs create mode 100644 dotnet-install.ps1 diff --git a/Directory.Build.props b/Directory.Build.props index 25d7ada..a062c54 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,27 +1,27 @@ - - - true - false - false - false - portable - + + + true + false + false + false + portable + - - net6.0;net7.0;net8.0 - AnyCPU - enable - enable - 12.0 - + + net8.0;net9.0 + AnyCPU + enable + enable + 13.0 + - - - all - runtime; build; native; contentfiles; analyzers - - + + + all + runtime; build; native; contentfiles; analyzers + + @@ -29,47 +29,47 @@ true - - 0.0.0 - Cédric L. Charlier - Seddryck - nbiguity - Apache-2.0 - https://github.com/Seddrycl/PocketCsvReader - git - false - icon\pocket-csv-reader.png - snupkg - README.md - + + 0.0.0 + Cédric L. Charlier + Seddryck + nbiguity + Apache-2.0 + https://github.com/Seddrycl/PocketCsvReader + git + false + icon\pocket-csv-reader.png + snupkg + README.md + - - - - + + + + true - - - - $(NoWarn);CS1591 - - - false - true - + + + $(NoWarn);CS1591 + + + + false + true + - - 5 - preview - + + 5 + preview + - - 5 - preview - + + 5 + preview + diff --git a/PocketCsvReader.Benchmark/ToDataReader.cs b/PocketCsvReader.Benchmark/ToDataReader.cs index 4feb04d..e1b1e2b 100644 --- a/PocketCsvReader.Benchmark/ToDataReader.cs +++ b/PocketCsvReader.Benchmark/ToDataReader.cs @@ -146,7 +146,7 @@ public CustomConfig() // Create a specific job for each version AddJob(Job.Default - .WithRuntime(CoreRuntime.Core80) + .WithRuntime(CoreRuntime.Core90) .WithWarmupCount(1) // 1 warm-up iteration .WithIterationCount(5) // 5 actual iterations ); // Identify the job by version diff --git a/PocketCsvReader.Testing/CsvObjectReaderTest.cs b/PocketCsvReader.Testing/CsvObjectReaderTest.cs new file mode 100644 index 0000000..899e05b --- /dev/null +++ b/PocketCsvReader.Testing/CsvObjectReaderTest.cs @@ -0,0 +1,120 @@ +using PocketCsvReader; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using System.Text; +using System.Reflection; +using System.Security.Cryptography; +using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; + +namespace PocketCsvReader.Testing; + +[TestFixture] +public class CsvObjectReaderTest +{ + private static MemoryStream CreateStream(string content) + { + byte[] byteArray = Encoding.UTF8.GetBytes(content); + MemoryStream stream = new MemoryStream(byteArray); + stream.Position = 0; + return stream; + } + + private record struct Human(string Name, bool IsAdult); + [Test] + public void GetString_SingleFieldAttemptForSecond_Throws() + { + var spanMapper = new SpanMapper((span, fieldSpans) => + { + return new Human( + span.Slice(fieldSpans.First().Start, fieldSpans.First().Length).ToString(), + int.Parse(span.Slice(fieldSpans.Last().Start, fieldSpans.Last().Length).ToString()) > 18); + }); + + var profile = new CsvProfile(',', '\"', "\r\n", false); + using var stream = CreateStream("foo,16\r\nbar,21"); + using var dataReader = new CsvObjectReader(stream, profile, spanMapper); + + var humans = dataReader.Read().ToArray(); + Assert.That(humans, Has.Length.EqualTo(2)); + Assert.That(humans[0].Name, Is.EqualTo("foo")); + Assert.That(humans[0].IsAdult, Is.False); + Assert.That(humans[1].Name, Is.EqualTo("bar")); + Assert.That(humans[1].IsAdult, Is.True); + } + + private record struct Financial( + int Year, int Month, int Day, DateTime DateTime, + string ResolutionCode, string Status, string AreaCode, string AreaTypeCode, string AreaName, string MapCode, + decimal Expenses, decimal Income, string Currency, DateTime UpdateTime); + [Test] + [TestCase("Ansi")] + [TestCase("Utf16-BE")] + [TestCase("Utf16-LE")] + [TestCase("Utf8-BOM")] + [TestCase("Utf8")] + public void Read_FinancialWithCompleteParsers_CorrectRowsColumns(string filename) + { + var profile = new CsvProfile('\t', '\"', "\r\n", true); + + using (var stream = + Assembly.GetExecutingAssembly() + .GetManifestResourceStream($"{Assembly.GetExecutingAssembly().GetName().Name}.Resources.{filename}.csv") + ?? throw new FileNotFoundException() + ) + { + var spanMapper = new SpanMapper((span, fieldSpans) => + { + return new Financial( + int.Parse(span.Slice(fieldSpans.ElementAt(0).Start, fieldSpans.ElementAt(0).Length)), + int.Parse(span.Slice(fieldSpans.ElementAt(1).Start, fieldSpans.ElementAt(1).Length)), + int.Parse(span.Slice(fieldSpans.ElementAt(2).Start, fieldSpans.ElementAt(2).Length)), + DateTime.Parse(span.Slice(fieldSpans.ElementAt(3).Start, fieldSpans.ElementAt(3).Length)), + span.Slice(fieldSpans.ElementAt(4).Start, fieldSpans.ElementAt(4).Length).ToString(), + span.Slice(fieldSpans.ElementAt(5).Start, fieldSpans.ElementAt(5).Length).ToString(), + span.Slice(fieldSpans.ElementAt(6).Start, fieldSpans.ElementAt(6).Length).ToString(), + span.Slice(fieldSpans.ElementAt(7).Start, fieldSpans.ElementAt(7).Length).ToString(), + span.Slice(fieldSpans.ElementAt(8).Start, fieldSpans.ElementAt(8).Length).ToString(), + span.Slice(fieldSpans.ElementAt(9).Start, fieldSpans.ElementAt(9).Length).ToString(), + decimal.Parse(span.Slice(fieldSpans.ElementAt(10).Start, fieldSpans.ElementAt(10).Length)), + decimal.Parse(span.Slice(fieldSpans.ElementAt(11).Start, fieldSpans.ElementAt(11).Length)), + span.Slice(fieldSpans.ElementAt(12).Start, fieldSpans.ElementAt(12).Length).ToString(), + DateTime.Parse(span.Slice(fieldSpans.ElementAt(13).Start, fieldSpans.ElementAt(13).Length))); + }); + var rowCount = 0; + using var dataReader = new CsvObjectReader(stream, profile, spanMapper); + foreach(var human in dataReader.Read()) + { Console.WriteLine($"{rowCount++}: {human.AreaCode}"); } + Assert.That(rowCount, Is.EqualTo(21)); + } + } + + [Test] + [TestCase("Ansi")] + [TestCase("Utf16-BE")] + [TestCase("Utf16-LE")] + [TestCase("Utf8-BOM")] + [TestCase("Utf8")] + public void Read_FinancialWithSpanObjectBuilder_CorrectRowsColumns(string filename) + { + var profile = new CsvProfile('\t', '\"', "\r\n", true); + + using (var stream = + Assembly.GetExecutingAssembly() + .GetManifestResourceStream($"{Assembly.GetExecutingAssembly().GetName().Name}.Resources.{filename}.csv") + ?? throw new FileNotFoundException() + ) + { + var objBuilder = new SpanObjectBuilder(); + var spanMapper = new SpanMapper(objBuilder.Instantiate); + var rowCount = 0; + using var dataReader = new CsvObjectReader(stream, profile, spanMapper); + foreach (var human in dataReader.Read()) + { rowCount++; } + Assert.That(rowCount, Is.EqualTo(21)); + } + } +} diff --git a/PocketCsvReader.Testing/CsvReaderTest.cs b/PocketCsvReader.Testing/CsvReaderTest.cs index 1096220..efd51ea 100644 --- a/PocketCsvReader.Testing/CsvReaderTest.cs +++ b/PocketCsvReader.Testing/CsvReaderTest.cs @@ -1,8 +1,10 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Text; using System.Threading.Tasks; +using Microsoft.VisualBasic; using NUnit.Framework; namespace PocketCsvReader.Testing; @@ -78,4 +80,37 @@ public void ToArrayOfString_PackageAssetStream_Successful(string filename) var arrays = new CsvReader(profile).ToArrayString(stream); Assert.That(arrays.Count, Is.EqualTo(1695)); } + + private record struct PackageAsset( + string Guid, DateTimeOffset Created, string Name, string Version, + DateTimeOffset Updated, string Description, string Runtime, + string Field1, string Field2, string Field3, string Field4, string Field5, + string Field6, string Field7, string Field8, string Field9, string Field10, + string Field11, string Field12, string Field13, string Field14, string Field15, + string Field16, string Field17, string Field18 + ); + + [Test] + [TestCase(@"Resources\PackageAssets.csv")] + public void ToObjectWithSpanMapper_PackageAssetStream_Successful(string filename) + { + var objBuilder = new SpanObjectBuilder(); + objBuilder.SetParser(s => DateTimeOffset.Parse(s, CultureInfo.InvariantCulture)); + var spanMapper = new SpanMapper(objBuilder.Instantiate); + + using var stream = File.OpenRead(filename); + var profile = new CsvProfile(',', '\"', Environment.NewLine, false); + var arrays = new CsvReader(profile).To(stream, spanMapper); + Assert.That(arrays.Count, Is.EqualTo(1695)); + } + + [Test] + [TestCase(@"Resources\PackageAssets.csv")] + public void ToObject_PackageAssetStream_Successful(string filename) + { + using var stream = File.OpenRead(filename); + var profile = new CsvProfile(',', '\"', Environment.NewLine, false); + var arrays = new CsvReader(profile).To(stream); + Assert.That(arrays.Count, Is.EqualTo(1695)); + } } diff --git a/PocketCsvReader.Testing/FieldParserTest.cs b/PocketCsvReader.Testing/FieldParserTest.cs index 5212d66..b6c8e1f 100644 --- a/PocketCsvReader.Testing/FieldParserTest.cs +++ b/PocketCsvReader.Testing/FieldParserTest.cs @@ -16,8 +16,8 @@ public void ReadField_NotQualified_CorrectString(string item, string result) item.AsSpan().CopyTo(buffer); var profile = new CsvProfile(';', '\'', '`', "\r\n", false, false, 4096, "(empty)", "(null)"); - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 0, item.Length, false, false); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(0, item.Length, false, false)); Assert.That(value, Is.EqualTo(result)); } @@ -29,8 +29,8 @@ public void ReadField_Empty_CorrectString(string item, string result) item.AsSpan().CopyTo(buffer); var profile = new CsvProfile(';', '\'', '`', "\r\n", false, false, 4096, "?", "(null)"); - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 0, 0, false, false); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(0, 0, false, false)); Assert.That(value, Is.EqualTo(result)); } @@ -42,8 +42,8 @@ public void ReadField_Null_CorrectString(string item, string result) item.AsSpan().CopyTo(buffer); var profile = new CsvProfile(new CsvDialectDescriptor { NullSequence = "(null)" }); - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 0, item.Length, false, false); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(0, item.Length, false, false)); Assert.That(value, Is.EqualTo(result)); } @@ -55,8 +55,8 @@ public void ReadField_NullButQuoted_CorrectString(string item, string result) item.AsSpan().CopyTo(buffer); var profile = new CsvProfile(new CsvDialectDescriptor { NullSequence = "(null)" }); - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 1, item.Length - 2, false, true); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(1, item.Length - 2, false, true)); Assert.That(value, Is.EqualTo(result)); } @@ -70,8 +70,8 @@ public void ReadField_Qualified_CorrectString(string item, string result) item.AsSpan().CopyTo(buffer); var profile = new CsvProfile(';', '`', '\\', "\r\n", false, false, 4096, "?", "(null)"); - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 1, item.Length - 2, false, false); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(1, item.Length - 2, false, false)); Assert.That(value, Is.EqualTo(result)); } @@ -86,8 +86,8 @@ public void ReadFieldWithoutHandleSpecialValues_Qualified_CorrectString(string i { ParserOptimizations = new ParserOptimizationOptions { HandleSpecialValues = false } }; - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 1, item.Length - 2, false, false); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(1, item.Length - 2, false, false)); Assert.That(value, Is.EqualTo(result)); } @@ -100,8 +100,8 @@ public void ReadField_EscapedWithOtherChar_CorrectString(string item, string res item.AsSpan().CopyTo(buffer); var profile = new CsvProfile(';', '\'', '`', "\r\n", false, false, 4096, "(empty)", "(null)"); - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 1, item.Length - 2, true, true); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(1, item.Length - 2, true, true)); Assert.That(value, Is.EqualTo(result)); } @@ -115,8 +115,8 @@ public void ReadFieldWithoutUnescapeChars_EscapedWithOtherChar_CorrectString(str var profile = new CsvProfile(';', '\'', '`', "\r\n", false, false, 4096, "(empty)", "(null)"); profile.ParserOptimizations = new ParserOptimizationOptions { UnescapeChars = false }; - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 1, item.Length - 2, true, true); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(1, item.Length - 2, true, true)); Assert.That(value, Is.EqualTo(result)); } @@ -130,8 +130,8 @@ public void ReadField_EscapedWithDoubleChar_CorrectString(string item, string re item.AsSpan().CopyTo(buffer); var profile = new CsvProfile(';', '\"', '\"', "\r\n", false, false, 4096, "(empty)", "(null)"); - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 1, item.Length - 2, true, true); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(1, item.Length - 2, true, true)); Assert.That(value, Is.EqualTo(result)); } @@ -147,8 +147,8 @@ public void ReadField_StringPool_CorrectString() { ParserOptimizations = new ParserOptimizationOptions { PoolString = stringPool.GetOrAdd } }; - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 0, "foo".Length, false, false); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(0, "foo".Length, false, false)); Assert.That(value, Is.EqualTo("foo")); } @@ -170,8 +170,8 @@ public void ReadField_StringPool_Called() { ParserOptimizations = new ParserOptimizationOptions { PoolString = poolString } }; - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 0, "foo".Length, false, false); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(0, "foo".Length, false, false)); Assert.That(value, Is.EqualTo("foo")); Assert.That(count, Is.EqualTo(1)); } @@ -186,8 +186,8 @@ public void ReadField_NullSequence_NullValue(string field, string NullSequence) var profile = new CsvProfile(new CsvDialectDescriptor { NullSequence = NullSequence }); - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 0, field.Length, false, false); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(0, field.Length, false, false)); Assert.That(value, Is.Null); } @@ -201,8 +201,8 @@ public void ReadField_WrongNullSequence_NotNullValue(string field, string NullSe var profile = new CsvProfile(new CsvDialectDescriptor { NullSequence = NullSequence }); - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 0, field.Length, false, false); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(0, field.Length, false, false)); Assert.That(value, Is.Not.Null); Assert.That(value, Is.EqualTo(field)); } @@ -217,8 +217,8 @@ public void ReadField_AnySequence_MappedValue(string field, string sequence, str var profile = CsvProfile.CommaDoubleQuote; profile.Sequences.Add(sequence, map); - var reader = new FieldParser(profile); - var value = reader.ReadField(buffer, 0, field.Length, false, false); + var reader = new ArrayOfStringMapper(profile); + var value = reader.Map(buffer, new FieldSpan(0, field.Length, false, false)); Assert.That(value, Is.Not.Null); Assert.That(value, Is.EqualTo(map)); } diff --git a/PocketCsvReader.Testing/RecordParserTTest.cs b/PocketCsvReader.Testing/RecordParserTTest.cs new file mode 100644 index 0000000..792d25b --- /dev/null +++ b/PocketCsvReader.Testing/RecordParserTTest.cs @@ -0,0 +1,98 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; + +namespace PocketCsvReader.Testing; +public class RecordParserTTest +{ + [Test] + [TestCase("foo")] + [TestCase("'foo'")] + [TestCase("foo\r\n")] + [TestCase("'foo'\r\n")] + public void ReadNextRecord_SingleField_CorrectParsing(string record) + { + var buffer = new MemoryStream(Encoding.UTF8.GetBytes(record)); + + var profile = new CsvProfile(';', '\'', '\'', "\r\n", false, true, 4096, string.Empty, string.Empty); + var spanMapper = new SpanMapper((span, fieldSpans) => span.Slice(fieldSpans.First().Start, fieldSpans.First().Length).ToString()); + using var reader = new RecordParser(new StreamReader(buffer), profile, spanMapper, ArrayPool.Create(256, 5)); + var eof = reader.ReadNextRecord(out var value); + Assert.That(value, Is.EqualTo("foo")); + } + + private record struct Employee(string Name, int Age); + + [TestCase("foo;16\r\n", "foo", 16)] + [TestCase("'foo';16\r\n", "foo", 16)] + public void ReadNextRecord_RecordWithLineTerminator_CorrectParsing(string record, string name, int age) + { + var buffer = new MemoryStream(Encoding.UTF8.GetBytes(record)); + + var profile = new CsvProfile(';', '\'', '\'', "\r\n", false, false, 4096, "(empty)", "(null)"); + var spanMapper = new SpanMapper((span, fieldSpans) => + { + return new Employee( + span.Slice(fieldSpans.First().Start, fieldSpans.First().Length).ToString(), + int.Parse(span.Slice(fieldSpans.Last().Start, fieldSpans.Last().Length).ToString())); + }); + using var reader = new RecordParser(new StreamReader(buffer), profile, spanMapper, ArrayPool.Create(256, 5)); + reader.ReadNextRecord(out var value); + Assert.That(value, Is.TypeOf()); + Assert.That(value.Name, Is.EqualTo(name)); + Assert.That(value.Age, Is.EqualTo(age)); + } + + [TestCase("foo;16\r\nbar;18")] + [TestCase("'foo';16\r\nbar;'18'")] + public void ReadNextRecord_TwoRecords_CorrectParsing(string record) + { + var buffer = new MemoryStream(Encoding.UTF8.GetBytes(record)); + + var profile = new CsvProfile(';', '\'', '\'', "\r\n", false, false, 4096, "(empty)", "(null)"); + var spanMapper = new SpanMapper((span, fieldSpans) => + { + return new Employee( + span.Slice(fieldSpans.First().Start, fieldSpans.First().Length).ToString(), + int.Parse(span.Slice(fieldSpans.Last().Start, fieldSpans.Last().Length).ToString())); + }); + using var reader = new RecordParser(new StreamReader(buffer), profile, spanMapper, ArrayPool.Create(256, 5)); + reader.ReadNextRecord(out var value); + Assert.That(value, Is.TypeOf()); + Assert.That(value.Name, Is.EqualTo("foo")); + Assert.That(value.Age, Is.EqualTo(16)); + reader.ReadNextRecord(out value); + Assert.That(value, Is.TypeOf()); + Assert.That(value.Name, Is.EqualTo("bar")); + Assert.That(value.Age, Is.EqualTo(18)); + } + + private record struct Human(string Name, bool IsAdult); + + [TestCase("foo;22\r\nbar;26")] + public void ReadNextRecord_LogicBasedRecord_CorrectParsing(string record) + { + var buffer = new MemoryStream(Encoding.UTF8.GetBytes(record)); + + var profile = new CsvProfile(';', '\'', '\'', "\r\n", false, false, 4096, "(empty)", "(null)"); + var spanMapper = new SpanMapper((span, fieldSpans) => + { + return new Human( + span.Slice(fieldSpans.First().Start, fieldSpans.First().Length).ToString(), + int.Parse(span.Slice(fieldSpans.Last().Start, fieldSpans.Last().Length).ToString())>18); + }); + using var reader = new RecordParser(new StreamReader(buffer), profile, spanMapper, ArrayPool.Create(256, 5)); + reader.ReadNextRecord(out var value); + Assert.That(value, Is.TypeOf()); + Assert.That(value.Name, Is.EqualTo("foo")); + Assert.That(value.IsAdult, Is.True); + reader.ReadNextRecord(out value); + Assert.That(value, Is.TypeOf()); + Assert.That(value.Name, Is.EqualTo("bar")); + Assert.That(value.IsAdult, Is.True); + } +} diff --git a/PocketCsvReader.Testing/RecordParserTest.cs b/PocketCsvReader.Testing/RecordParserTest.cs index 0866194..ece1213 100644 --- a/PocketCsvReader.Testing/RecordParserTest.cs +++ b/PocketCsvReader.Testing/RecordParserTest.cs @@ -22,7 +22,7 @@ public void ReadNextRecord_SingleField_CorrectParsing(string record) using var reader = new RecordParser(new StreamReader(buffer), profile, ArrayPool.Create(256, 5)); var eof = reader.ReadNextRecord(out var values); Assert.That(values, Has.Length.EqualTo(1)); - Assert.That(values.First(), Is.EqualTo("foo")); + Assert.That(values!.First(), Is.EqualTo("foo")); } [TestCase("foo\r\n", "foo")] @@ -37,7 +37,7 @@ public void ReadNextRecord_RecordWithLineTerminator_CorrectParsing(string record reader.ReadNextRecord(out var values); Assert.That(values, Has.Length.EqualTo(tokens.Length)); for (int i = 0; i < tokens.Length; i++) - Assert.That(values[i], Is.EqualTo(tokens[i])); + Assert.That(values![i], Is.EqualTo(tokens[i])); } [TestCase("foo", "foo")] @@ -52,7 +52,7 @@ public void ReadNextRecord_RecordWithoutLineTerminator_CorrectParsing(string rec reader.ReadNextRecord(out var values); Assert.That(values, Has.Length.EqualTo(tokens.Length)); for (int i = 0; i < tokens.Length; i++) - Assert.That(values[i], Is.EqualTo(tokens[i])); + Assert.That(values![i], Is.EqualTo(tokens[i])); } [Test] @@ -92,8 +92,8 @@ public void ReadNextRecord_RecordWithTwoFields_CorrectParsing(string record, str new CsvDialectDescriptor() { Delimiter=';', QuoteChar='\'', DoubleQuote=true }); using var reader = new RecordParser(new StreamReader(buffer), profile, ArrayPool.Create(256, 5)); reader.ReadNextRecord(out var values); - Assert.That(values[0], Is.EqualTo(firstToken)); - Assert.That(values[1], Is.EqualTo("xyz")); + Assert.That(values![0], Is.EqualTo(firstToken)); + Assert.That(values![1], Is.EqualTo("xyz")); } [Test] @@ -108,7 +108,7 @@ public void ReadNextRecord_SingleFieldWithTextQualifier_CorrectParsing(string re using var reader = new RecordParser(new StreamReader(buffer), profile, ArrayPool.Create(256, 5)); reader.ReadNextRecord(out var values); Assert.That(values, Has.Length.EqualTo(1)); - Assert.That(values.First(), Is.EqualTo(expected)); + Assert.That(values!.First(), Is.EqualTo(expected)); } [Test] @@ -123,7 +123,7 @@ public void ReadNextRecord_SingleFieldWithTextEscaper_CorrectParsing(string reco using var reader = new RecordParser(new StreamReader(buffer), profile, ArrayPool.Create(256, 5)); reader.ReadNextRecord(out var values); Assert.That(values, Has.Length.EqualTo(1)); - Assert.That(values.First(), Is.EqualTo("fo'o")); + Assert.That(values!.First(), Is.EqualTo("fo'o")); } [Test] @@ -138,7 +138,7 @@ public void ReadNextRecord_SingleFieldWithDoubleQuote_CorrectParsing(string reco using var reader = new RecordParser(new StreamReader(buffer), profile, ArrayPool.Create(256, 5)); reader.ReadNextRecord(out var values); Assert.That(values, Has.Length.EqualTo(1)); - Assert.That(values.First(), Is.EqualTo("fo'o")); + Assert.That(values!.First(), Is.EqualTo("fo'o")); } [Test] @@ -159,7 +159,7 @@ public void ReadNextRecord_RecordWithThreeFields_CorrectParsing(string record, s using var reader = new RecordParser(new StreamReader(buffer), profile, ArrayPool.Create(256, 5)); reader.ReadNextRecord(out var values); Assert.That(values, Has.Length.EqualTo(3)); - Assert.That(values[2], Is.EqualTo(thirdToken)); + Assert.That(values![2], Is.EqualTo(thirdToken)); } [Test] @@ -172,7 +172,7 @@ public void ReadNextRecord_NullField_NullValue() var eof = reader.ReadNextRecord(out var values); Assert.That(eof, Is.True); Assert.That(values, Has.Length.EqualTo(2)); - Assert.That(values[1], Is.Null); + Assert.That(values![1], Is.Null); } [Test] @@ -206,7 +206,7 @@ public void ReadNextRecord_Csv_CorrectResults(string text, string recordSeparato { reader.ReadNextRecord(out var values); Assert.That(values, Has.Length.GreaterThan(0)); - foreach (var value in values) + foreach (var value in values!) Assert.That(value, Is.EqualTo("abc")); } writer.Dispose(); @@ -229,7 +229,7 @@ public void ReadNextRecord_SkipInitialWhitespace_CorrectResults(string record) using var streamReader = new StreamReader(stream); reader.ReadNextRecord(out var values); Assert.That(values, Has.Length.EqualTo(2)); - Assert.That(values[0], Is.EqualTo("foo")); + Assert.That(values![0], Is.EqualTo("foo")); Assert.That(values[1], Is.EqualTo("bar")); } @@ -345,9 +345,9 @@ public void GetFirstRecord_Csv_CorrectResult(string text, string recordSeparator using (var streamReader = new StreamReader(stream, Encoding.UTF8, true)) { - var reader = new RecordParser(streamReader, CsvProfile.SemiColumnDoubleQuote, ArrayPool.Create(256, 5)); - var value = reader.GetFirstRecord(streamReader, recordSeparator, bufferSize); - Assert.That(value, Is.EqualTo("abc" + recordSeparator).Or.EqualTo("abc")); + var reader = new RecordParser(streamReader, new CsvProfile(',', '\\', recordSeparator, false, false, bufferSize), ArrayPool.Create(256, 5)); + var value = reader.GetFirstRecord(); + Assert.That(value, Is.EqualTo("abc")); } writer.Dispose(); } @@ -368,9 +368,9 @@ public void GetFirstRecord_CsvWithSemiSeparator_CorrectResult(string text, strin using (var streamReader = new StreamReader(stream, Encoding.UTF8, true)) { - var reader = new RecordParser(streamReader, CsvProfile.SemiColumnDoubleQuote); - var value = reader.GetFirstRecord(streamReader, recordSeparator, bufferSize); - Assert.That(value, Is.EqualTo("abc+abc" + recordSeparator).Or.EqualTo("abc+abc")); + var reader = new RecordParser(streamReader, new CsvProfile(',', '\\', recordSeparator, false, false, bufferSize)); + var value = reader.GetFirstRecord(); + Assert.That(value, Is.EqualTo("abc+abc")); } writer.Dispose(); } diff --git a/PocketCsvReader.Testing/SpanObjectBuilderTests.cs b/PocketCsvReader.Testing/SpanObjectBuilderTests.cs new file mode 100644 index 0000000..4caaa46 --- /dev/null +++ b/PocketCsvReader.Testing/SpanObjectBuilderTests.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestPlatform.Utilities; +using NUnit.Framework; + +namespace PocketCsvReader.Testing; +public class SpanObjectBuilderTests +{ + + private record struct StringBox(string value); + private record struct IntBox(int value); + private record struct FloatBox(float value); + private record struct DateTimeBox(DateTime value); + private record struct DateOnlyBox(DateOnly value); + private record struct TimeOnlyBox(TimeOnly value); + + public class SpanObjectBuilderProxy : SpanObjectBuilder + { + public SpanObjectBuilderProxy() : base() { } + public T Instantiate(char[] array, FieldSpan[] fieldSpans) + => base.Instantiate(array, fieldSpans); + } + + [Test] + [TestCase("foo", "foo", typeof(StringBox))] + [TestCase("10", 10, typeof(IntBox))] + [TestCase("10.15", 10.15f, typeof(FloatBox))] + public void Instantiate_SingleField_Valid(string input, object output, Type type) + { + var builderType = typeof(SpanObjectBuilderProxy<>).MakeGenericType(type); + var builder = Activator.CreateInstance(builderType); + var instantiateMethod = builderType.GetMethod("Instantiate", [typeof(char[]), typeof(FieldSpan[])])!; + var fieldSpans = new[] { new FieldSpan(0, input.Length, false, false) }; + var result = instantiateMethod.Invoke(builder, [input.ToCharArray(), fieldSpans ])!; + Assert.That(type.GetProperty("value")!.GetValue(result), Is.EqualTo(output)); + } + + + [TestCase("2024-12-12T16:12:13", typeof(DateTime), typeof(DateTimeBox))] + [TestCase("2024-12-12", typeof(DateOnly), typeof(DateOnlyBox))] + [TestCase("16:12:13", typeof(TimeOnly), typeof(TimeOnlyBox))] + public void Instantiate_SingleFieldFromText_Valid(string input, Type fromText, Type type) + { + var output = fromText.GetMethod("Parse", [typeof(string)])!.Invoke(null, [input])!; + Instantiate_SingleField_Valid(input, output, type); + } + + [Test] + public void Instantiate_SingleFieldButWrongType_FormatException() + { + var builder = new SpanObjectBuilder(); + var fieldSpans = new[] { new FieldSpan(0, "2024-12-12".Length, false, false) }; + var ex = Assert.Catch(() => builder.Instantiate(new ReadOnlySpan("2024-12-12".ToCharArray()), fieldSpans)); + Assert.That(ex!.Message, Does.Contain("Int32")); + Assert.That(ex.Message, Does.Contain("0")); + Assert.That(ex.Message, Does.Contain("2024-12-12")); + } +} diff --git a/PocketCsvReader/FieldParser.cs b/PocketCsvReader/ArrayOfStringMapper.cs similarity index 72% rename from PocketCsvReader/FieldParser.cs rename to PocketCsvReader/ArrayOfStringMapper.cs index 3f410a5..daa0fb4 100644 --- a/PocketCsvReader/FieldParser.cs +++ b/PocketCsvReader/ArrayOfStringMapper.cs @@ -1,5 +1,6 @@ using System; using System.Buffers; +using System.Collections; using System.Collections.Generic; using System.Data; using System.Linq; @@ -9,7 +10,7 @@ namespace PocketCsvReader; public delegate string PoolString(ReadOnlySpan memory); -public class FieldParser +public class ArrayOfStringMapper { protected internal CsvProfile Profile { get; private set; } protected ArrayPool? Pool { get; } @@ -20,30 +21,27 @@ public class FieldParser private static readonly PoolString defaultPoolString = (ReadOnlySpan span) => span.ToString(); - public FieldParser(CsvProfile profile) + public ArrayOfStringMapper(CsvProfile profile) : this(profile, ArrayPool.Shared) { } - public FieldParser(CsvProfile profile, ArrayPool? pool, PoolString? fetchString = null) + public ArrayOfStringMapper(CsvProfile profile, ArrayPool? pool) => (Profile, Pool, FetchString, HandlesSpecialValues, UnescapesChars) = (profile, pool, profile.ParserOptimizations.PoolString ?? defaultPoolString , profile.ParserOptimizations.HandleSpecialValues, profile.ParserOptimizations.UnescapeChars); - public string? ReadField(ReadOnlySpan buffer, int start, int length, bool isEscapedField, bool wasQuotedField) - => ReadField(Span.Empty, buffer, start, length, isEscapedField, wasQuotedField); - - public string? ReadField(ReadOnlySpan longSpan, ReadOnlySpan buffer, int start, int length, bool isEscapedField, bool wasQuotedField) + public string?[] Map(ReadOnlySpan span, IEnumerable fieldSpans) { - ReadOnlySpan fieldSpan; - if (longSpan.Length > 0 && length>=0) - fieldSpan = longSpan.Concat(buffer.Slice(start, length)); - else if (longSpan.Length > 0 && length < 0) - fieldSpan = longSpan.Slice(0, longSpan.Length + length); - else - fieldSpan = buffer.Slice(start, length); - return ExtractField(fieldSpan, isEscapedField, wasQuotedField); + var fields = new string?[fieldSpans.Count()]; + var index = 0; + foreach (var fieldSpan in fieldSpans) + fields[index++] = Map(span, fieldSpan); + return fields; } - public string? ExtractField(ReadOnlySpan buffer, bool isEscapedField, bool wasQuotedField) + public string? Map(ReadOnlySpan span, FieldSpan fieldSpan) + => ExtractField(span.Slice(fieldSpan.Start, fieldSpan.Length), fieldSpan.IsEscaped, fieldSpan.WasQuoted); + + protected internal string? ExtractField(ReadOnlySpan buffer, bool isEscapedField, bool wasQuotedField) { if (HandlesSpecialValues) { diff --git a/PocketCsvReader/CsvArrayString.cs b/PocketCsvReader/CsvArrayString.cs index aacc1ca..c0427e6 100644 --- a/PocketCsvReader/CsvArrayString.cs +++ b/PocketCsvReader/CsvArrayString.cs @@ -68,7 +68,7 @@ public void Initialize() IsEof = RecordParser!.ReadNextRecord(out var values); - if (IsEof && values.Length == 0) + if (IsEof && (values is null || values.Length == 0)) { values = null; Extra = null; @@ -80,12 +80,12 @@ public void Initialize() int unnamedFieldIndex = 0; if (RecordParser.Profile.Descriptor.Header) { - Fields = values.Select(value => value ?? $"field_{unnamedFieldIndex++}").ToArray(); + Fields = values!.Select(value => value ?? $"field_{unnamedFieldIndex++}").ToArray(); return ReadNextRecord(); // Skip header and read next record } else { - Fields = values.Select(_ => $"field_{unnamedFieldIndex++}").ToArray(); + Fields = values!.Select(_ => $"field_{unnamedFieldIndex++}").ToArray(); } } else diff --git a/PocketCsvReader/CsvObjectReader.cs b/PocketCsvReader/CsvObjectReader.cs new file mode 100644 index 0000000..e87d9b8 --- /dev/null +++ b/PocketCsvReader/CsvObjectReader.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Reflection.PortableExecutable; +using System.Text; +using System.Threading.Tasks; + +namespace PocketCsvReader; +public class CsvObjectReader : IDisposable +{ + protected RecordParser? RecordParser { get; set; } + protected CsvProfile Profile { get; } + protected Stream Stream { get; } + protected StreamReader? StreamReader { get; private set; } + protected Memory buffer; + public int RowCount { get; private set; } = 0; + + protected EncodingInfo? FileEncoding { get; private set; } + + protected bool IsEof { get; private set; } = false; + protected int BufferSize { get; private set; } = 64 * 1024; + + protected SpanMapper SpanMapper { get; } + + public CsvObjectReader(Stream stream, CsvProfile profile, SpanMapper? spanMapper = null) + { + Stream = stream; + buffer = new Memory(new char[BufferSize]); + Profile = profile; + SpanMapper = spanMapper ?? new SpanMapper(new SpanObjectBuilder().Instantiate); + } + + public void Initialize() + { + FileEncoding ??= new EncodingDetector().GetStreamEncoding(Stream); + StreamReader = new StreamReader(Stream, FileEncoding!.Encoding, false); + var bufferBOM = new char[1]; + StreamReader.Read(bufferBOM, 0, bufferBOM.Length); + StreamReader.Rewind(); + + if (FileEncoding!.BomBytesCount > 0) + StreamReader.BaseStream.Position = FileEncoding!.BomBytesCount; + + IsEof = false; + RecordParser = new RecordParser(StreamReader, Profile, SpanMapper); + } + + public IEnumerable Read() + { + if (FileEncoding is null) + Initialize(); + if (IsEof) + yield break; + + while (!IsEof) + { + if (RowCount == 0 && Profile.Descriptor.Header) + { + var _ = RecordParser!.ReadHeaders(); + } + IsEof = RecordParser!.ReadNextRecord(out var value); + if (IsEof && EqualityComparer.Default.Equals(value, default)) + yield break; + RowCount++; + yield return value; + } + yield break; + } + + public void Dispose() + { + StreamReader?.Dispose(); + Stream?.Dispose(); + RecordParser?.Dispose(); + GC.SuppressFinalize(this); // Prevents finalizer from running + } + + ~CsvObjectReader() + { + Dispose(); + } +} diff --git a/PocketCsvReader/CsvReader.cs b/PocketCsvReader/CsvReader.cs index f22dd6e..2e244e7 100644 --- a/PocketCsvReader/CsvReader.cs +++ b/PocketCsvReader/CsvReader.cs @@ -122,6 +122,36 @@ public IDataReader ToDataReader(Stream stream) return new CsvArrayString(stream, Profile).Read(); } + /// + /// Reads the specified CSV file and returns an for iterating over its records and fields. + /// + /// The name or full path of the CSV file to read. + /// An instance for sequentially reading each record and field in the CSV file. + /// + /// This method provides an for efficient, read-only, forward-only access to CSV data, + /// suitable for large files or cases where full file loading into memory is unnecessary. + /// + public IEnumerable To(string filename, SpanMapper? spanMapper = null) + { + CheckFileExists(filename); + var stream = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, Profile.ParserOptimizations.BufferSize); + return new CsvObjectReader(stream, Profile, spanMapper).Read(); + } + + /// + /// Reads the CSV data from the provided stream and returns an for efficient object-by-object access. + /// + /// The containing CSV data, positioned at the beginning of the content. + /// An that allows sequential access to each record and field in the CSV file. + /// + /// This method processes the CSV data from the stream and provides an for forward-only, read-only access, + /// ideal for handling large datasets without loading the entire file into memory at once. + /// + public IEnumerable To(Stream stream, SpanMapper? spanMapper = null) + { + return new CsvObjectReader(stream, Profile, spanMapper).Read(); + } + protected virtual void CheckFileExists(string filename) { if (!File.Exists(filename)) diff --git a/PocketCsvReader/FieldSpan.cs b/PocketCsvReader/FieldSpan.cs new file mode 100644 index 0000000..ab1abcf --- /dev/null +++ b/PocketCsvReader/FieldSpan.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace PocketCsvReader; +public record struct FieldSpan +( + int Start, + int Length, + bool WasQuoted, + bool IsEscaped +); diff --git a/PocketCsvReader/RecordParser.T.cs b/PocketCsvReader/RecordParser.T.cs new file mode 100644 index 0000000..1eff35e --- /dev/null +++ b/PocketCsvReader/RecordParser.T.cs @@ -0,0 +1,36 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Reflection.PortableExecutable; +using System.Text; + +namespace PocketCsvReader; +public class RecordParser : RecordParser +{ + protected SpanMapper SpanMapper { get; } + + public RecordParser(StreamReader reader, CsvProfile profile, SpanMapper spanMapper) + : this(reader, profile, spanMapper, ArrayPool.Shared) + { } + + public RecordParser(StreamReader reader, CsvProfile profile, SpanMapper spanMapper, ArrayPool? pool) + : this(profile, profile.ParserOptimizations.ReadAhead + ? new DoubleBuffer(reader, profile.ParserOptimizations.BufferSize, pool) + : new SingleBuffer(reader, profile.ParserOptimizations.BufferSize, pool) + , spanMapper, pool + ) + { } + + protected RecordParser(CsvProfile profile, IBufferReader buffer, SpanMapper spanMapper, ArrayPool? pool) + : base(profile, buffer, pool) + { + SpanMapper = spanMapper; + } + + public virtual bool ReadNextRecord(out T value) + => base.ReadNextRecord(SpanMapper, out value); + + +} diff --git a/PocketCsvReader/RecordParser.cs b/PocketCsvReader/RecordParser.cs index d7727c6..f2c3626 100644 --- a/PocketCsvReader/RecordParser.cs +++ b/PocketCsvReader/RecordParser.cs @@ -10,14 +10,13 @@ namespace PocketCsvReader; public class RecordParser : IDisposable { public CsvProfile Profile { get; } - protected FieldParser FieldParser { get; } protected CharParser CharParser { get; } protected IBufferReader Reader { get; } protected ReadOnlyMemory Buffer { get; private set; } protected ArrayPool? Pool { get; } + private SpanMapper SpanMapper { get; } private int? FieldsCount { get; set; } - public RecordParser(StreamReader reader, CsvProfile profile) : this(reader, profile, ArrayPool.Shared) { } @@ -26,18 +25,20 @@ public RecordParser(StreamReader reader, CsvProfile profile, ArrayPool? po : this(profile, profile.ParserOptimizations.ReadAhead ? new DoubleBuffer(reader, profile.ParserOptimizations.BufferSize, pool) : new SingleBuffer(reader, profile.ParserOptimizations.BufferSize, pool) - , pool - ) + , pool) { } protected RecordParser(CsvProfile profile, IBufferReader buffer, ArrayPool? pool) - => (Profile, Reader, FieldParser, CharParser) = (profile, buffer, new FieldParser(profile, pool ?? ArrayPool.Shared), new(profile)); + => (Profile, Reader, SpanMapper, CharParser) = (profile, buffer, new ArrayOfStringMapper(profile, pool ?? ArrayPool.Shared).Map, new(profile)); + + public virtual bool ReadNextRecord(out string?[]? value) + => ReadNextRecord(SpanMapper, out value); - public virtual bool ReadNextRecord(out string?[] fields) + protected virtual bool ReadNextRecord(SpanMapper spanMapper, out T value) { var index = 0; var eof = false; - var listFields = new List(FieldsCount ?? 20); + var fieldSpans = new List(FieldsCount ?? 20); var longSpan = Span.Empty; if (Buffer.Length == 0) @@ -56,27 +57,25 @@ public virtual bool ReadNextRecord(out string?[] fields) var state = CharParser.Parse(c); if (state == ParserState.Field || state == ParserState.Record) { - // InternalParse field and reset longSpan - listFields.Add(FieldParser.ReadField(longSpan, span, CharParser.FieldStart, CharParser.FieldLength, CharParser.IsEscapedField, CharParser.IsQuotedField)); - longSpan = Span.Empty; + fieldSpans.Add(new FieldSpan(CharParser.FieldStart, CharParser.FieldLength, CharParser.IsEscapedField, CharParser.IsQuotedField)); if (state == ParserState.Record) { CharParser.Reset(); Buffer = Buffer.Slice(index + 1); - FieldsCount ??= listFields.Count; - fields = [.. listFields]; + FieldsCount ??= fieldSpans.Count; + value = spanMapper.Invoke(longSpan.Length > 0 ? (ReadOnlySpan)(longSpan.Concat(span)) : span, fieldSpans); return false; } } else if (state == ParserState.Error) throw new InvalidDataException($"Invalid character '{c}' at position {index}."); - // Handle continuation for fields spanning multiple buffers + // Handle continuation for value spanning multiple buffers if (++index == bufferSize) { - if (state == ParserState.Continue) - longSpan = longSpan.Concat(span.Slice(CharParser.FieldStart, bufferSize - CharParser.FieldStart), Pool); + if (state == ParserState.Continue || state == ParserState.Field) + longSpan = longSpan.Concat(span, Pool); if (!Reader.IsEof) { @@ -85,8 +84,6 @@ public virtual bool ReadNextRecord(out string?[] fields) span = Buffer.Span; eof = bufferSize == 0; index = 0; - if (!eof) - CharParser.Reset(); } else { @@ -96,15 +93,14 @@ public virtual bool ReadNextRecord(out string?[] fields) } } - CharParser.Reset(); switch (CharParser.ParseEof()) { case ParserState.Record: - listFields.Add(FieldParser.ReadField(longSpan, 0, longSpan.Length + CharParser.FieldLength, CharParser.IsEscapedField, CharParser.IsQuotedField)); - fields = [.. listFields]; + fieldSpans.Add(new FieldSpan(CharParser.FieldStart, CharParser.FieldLength, CharParser.IsEscapedField, CharParser.IsQuotedField)); + value = spanMapper.Invoke(longSpan.Length > 0 ? (ReadOnlySpan)longSpan.Concat(span) : span, fieldSpans); return true; case ParserState.Eof: - fields = []; + value = (typeof(T).IsArray ? (T?)(object)Array.CreateInstance(typeof(T).GetElementType()!, 0) : default)!; return true; case ParserState.Error: throw new InvalidDataException($"Invalid character End-of-File."); @@ -113,6 +109,28 @@ public virtual bool ReadNextRecord(out string?[] fields) } } + public virtual string[] ReadHeaders() + { + var headerMapper = new SpanMapper((span, fieldSpans) => + { + var headers = new string[fieldSpans.Count()]; + var index = 0; + foreach (var fieldSpan in fieldSpans) + headers[index++] = span.Slice(fieldSpan.Start, fieldSpan.Length).ToString(); + return headers; + }); + + var unnamedFieldIndex = -1; + ReadNextRecord(headerMapper, out var fields); + return fields.Select(value => + { + unnamedFieldIndex++; + return string.IsNullOrWhiteSpace(value) || !Profile.Descriptor.Header + ? $"field_{unnamedFieldIndex}" + : value!; + }).ToArray(); + } + public int? CountRecords() { if (!Profile.ParserOptimizations.RowCountAtStart) @@ -121,127 +139,82 @@ public virtual bool ReadNextRecord(out string?[] fields) var count = CountRecordSeparators(); count -= Convert.ToInt16(Profile.Descriptor.Header); + CharParser.Reset(); Reader.Reset(); return count; } protected virtual int CountRecordSeparators() { - int i = 0; - int n = 0; - int j = 0; - bool separatorAtEnd = false; - bool isCommentLine = false; - bool isFirstCharOfLine = true; - - do - { - var span = Reader.Read().Span; - n = span.Length; - if (n > 0 && i == 0) - i = 1; + var span = ReadOnlySpan.Empty; + var index = 0; + var bufferSize = 0; + var count = 0; - foreach (var c in span) + while (true) + { + if (index == bufferSize) { - if (c != '\0') - { - if (c == Profile.Descriptor.CommentChar && isFirstCharOfLine) - isCommentLine = true; - isFirstCharOfLine = false; - - separatorAtEnd = false; - if (c == Profile.Descriptor.LineTerminator[j]) - { - j++; - if (j == Profile.Descriptor.LineTerminator.Length) - { - if (!isCommentLine) - i++; - j = 0; - separatorAtEnd = true; - isCommentLine = false; - isFirstCharOfLine = true; - } - } - else - j = 0; - } + if (Reader.IsEof) + break; + var buffer = Reader.Read(); + index = 0; + span = buffer.Span; + bufferSize = span.Length; } - } while (!Reader.IsEof); - if (separatorAtEnd) - i -= 1; + if (bufferSize == 0) + break; - if (isCommentLine) - i -= 1; + if (CharParser.Parse(span[index]) == ParserState.Record) + count++; + index++; + } + if (CharParser.ParseEof() == ParserState.Record) + count++; - return i; + return count; } - public string GetFirstRecord(StreamReader reader, string recordSeparator, int bufferSize) + public virtual string GetFirstRecord() { - int i = 0; - int j = 0; - Span longRecord = stackalloc char[0]; + var longSpan = Span.Empty; + var span = ReadOnlySpan.Empty; + var index = 0; + var bufferSize = 0; - var found = false; - var array = Pool?.Rent(Profile.ParserOptimizations.BufferSize) ?? new char[Profile.ParserOptimizations.BufferSize]; - while (!found) + while (true) { - var buffer = new Span(array); - var n = reader.ReadBlock(buffer); - buffer = buffer.Slice(0, n); - if (n == 0) - found = true; - - foreach (var c in buffer) + if (index == bufferSize) { - i++; - if (c == '\0') - found = true; - else if (c == recordSeparator[j]) - { - j++; - if (j == recordSeparator.Length) - found = true; - } - else - j = 0; - - if (found) + if (Reader.IsEof) break; + if (bufferSize > 0) + longSpan = longSpan.Concat(span, Pool); + var buffer = Reader.Read(); + index = 0; + span = buffer.Span; + bufferSize = span.Length; } - if (longRecord.Length == 0) - longRecord = buffer.Slice(0, i); - else + if (bufferSize == 0) + break; + + if (CharParser.Parse(span[index]) == ParserState.Record) { - var newArray = Pool?.Rent(longRecord.Length + i) ?? new char[longRecord.Length + i]; - var newSpan = newArray.AsSpan().Slice(0, longRecord.Length + i); - longRecord.CopyTo(newSpan); - buffer.CopyTo(newSpan.Slice(longRecord.Length)); - longRecord = newSpan; - Pool?.Return(newArray); + index -= Profile.Descriptor.LineTerminator.Length - 1; + break; } - i = 0; + + index++; } + CharParser.Reset(); - if (array is not null) - Pool?.Return(array); - return longRecord.ToString(); - } - - public virtual string[] ReadHeaders() - { - var unnamedFieldIndex = -1; - ReadNextRecord(out var fields); - return fields.Select(value => - { - unnamedFieldIndex++; - return string.IsNullOrWhiteSpace(value) || !Profile.Descriptor.Header - ? $"field_{unnamedFieldIndex}" - : value!; - }).ToArray(); + if (longSpan.Length == 0) + return span.Slice(0, index).ToString(); + if (index >= 0) + return (longSpan.Concat(span.Slice(0, index), Pool)).ToString(); + return longSpan.Slice(0, longSpan.Length + index).ToString(); } public void Dispose() diff --git a/PocketCsvReader/SpanMapper.cs b/PocketCsvReader/SpanMapper.cs new file mode 100644 index 0000000..c284903 --- /dev/null +++ b/PocketCsvReader/SpanMapper.cs @@ -0,0 +1,8 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace PocketCsvReader; +public delegate T SpanMapper(ReadOnlySpan span, IEnumerable fieldSpans); diff --git a/PocketCsvReader/SpanObjectBuilder.cs b/PocketCsvReader/SpanObjectBuilder.cs new file mode 100644 index 0000000..1101fd8 --- /dev/null +++ b/PocketCsvReader/SpanObjectBuilder.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.Design; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace PocketCsvReader; + +public delegate object? Parse(ReadOnlySpan span); + +public class SpanObjectBuilder +{ + private Dictionary ParserMapping { get; } = new(); + + public SpanObjectBuilder() + { + var culture = CultureInfo.InvariantCulture; + ParserMapping.Add(typeof(string), s => s.ToString()); + ParserMapping.Add(typeof(int), s => int.Parse(s, culture)); + ParserMapping.Add(typeof(long), s => long.Parse(s, culture)); + ParserMapping.Add(typeof(short), s => short.Parse(s, culture)); + ParserMapping.Add(typeof(byte), s => byte.Parse(s, culture)); + ParserMapping.Add(typeof(float), s => float.Parse(s, culture)); + ParserMapping.Add(typeof(double), s => double.Parse(s, culture)); + ParserMapping.Add(typeof(decimal), s => decimal.Parse(s, culture)); + ParserMapping.Add(typeof(bool), s => bool.Parse(s)); + ParserMapping.Add(typeof(DateTime), s => DateTime.Parse(s)); + ParserMapping.Add(typeof(DateOnly), s => DateOnly.Parse(s)); + ParserMapping.Add(typeof(TimeOnly), s => TimeOnly.Parse(s)); + ParserMapping.Add(typeof(DateTimeOffset), s => DateTimeOffset.Parse(s)); + ParserMapping.Add(typeof(char), s => s[0]); + } + + public void SetParser(Parse parse) + { + if (ParserMapping.ContainsKey(typeof(TField))) + ParserMapping[typeof(TField)] = parse; + else + ParserMapping.Add(typeof(TField), parse); + } + + public T Instantiate(ReadOnlySpan span, IEnumerable fieldSpans) + { + var ctors = typeof(T).GetConstructors(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); + var ctor = ctors.First(c => c.GetParameters().Length == fieldSpans.Count()); + var index = 0; + var fields = new object?[fieldSpans.Count()]; + foreach (var fieldSpan in fieldSpans) + { + var type = ctor.GetParameters()[index].ParameterType; + if(!ParserMapping.TryGetValue(type, out var parse)) + throw new Exception($"No parser found for type {type}."); + try + { + var field = parse(span.Slice(fieldSpan.Start, fieldSpan.Length)); + fields[index++] = field; + } + catch (Exception ex) + { + throw new FormatException($"Error parsing field {index} of type {type} for value {span.Slice(fieldSpan.Start, fieldSpan.Length).ToString()}", ex); + } + } + return (T)ctor.Invoke(fields); + } +} diff --git a/appveyor.yml b/appveyor.yml index df0b3e3..9b6760c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -21,11 +21,18 @@ environment: init: - cmd: git config --global core.autocrlf true -- cmd: setx IGNORE_NORMALISATION_GIT_HEAD_MOVE "1" -- cmd: setx DOTNET_NO_WORKLOAD_UPDATE_NOTIFICATION "true" +- cmd: setx IGNORE_NORMALISATION_GIT_HEAD_MOVE 1 +- cmd: setx DOTNET_NO_WORKLOAD_UPDATE_NOTIFICATION 1 +- cmd: setx DOTNET_CLI_TELEMETRY_OPTOUT 1 +- cmd: setx DOTNET_NOLOGO 1 - cmd: RefreshEnv.cmd - pwsh: Write-Host "Target branch is '$($env:APPVEYOR_REPO_BRANCH)'" +install: +- ps: | + Invoke-WebRequest -Uri 'https://dot.net/v1/dotnet-install.ps1' -UseBasicParsing -OutFile "$env:temp\dotnet-install.ps1" + & $env:temp\dotnet-install.ps1 -Architecture x64 -Version '9.0.100' -InstallDir "$env:ProgramFiles\dotnet" + before_build: - cmd: gitversion /output buildserver /verbosity Minimal - pwsh: Write-Host "Building PocketCsvReader version $($env:GitVersion_SemVer)" diff --git a/dotnet-install.ps1 b/dotnet-install.ps1 new file mode 100644 index 0000000..176a40b --- /dev/null +++ b/dotnet-install.ps1 @@ -0,0 +1,1603 @@ +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +<# +.SYNOPSIS + Installs dotnet cli +.DESCRIPTION + Installs dotnet cli. If dotnet installation already exists in the given directory + it will update it only if the requested version differs from the one already installed. + + Note that the intended use of this script is for Continuous Integration (CI) scenarios, where: + - The SDK needs to be installed without user interaction and without admin rights. + - The SDK installation doesn't need to persist across multiple CI runs. + To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer. + +.PARAMETER Channel + Default: LTS + Download from the Channel specified. Possible values: + - STS - the most recent Standard Term Support release + - LTS - the most recent Long Term Support release + - 2-part version in a format A.B - represents a specific release + examples: 2.0, 1.0 + - 3-part version in a format A.B.Cxx - represents a specific SDK release + examples: 5.0.1xx, 5.0.2xx + Supported since 5.0 release + Warning: Value "Current" is deprecated for the Channel parameter. Use "STS" instead. + Note: The version parameter overrides the channel parameter when any version other than 'latest' is used. +.PARAMETER Quality + Download the latest build of specified quality in the channel. The possible values are: daily, signed, validated, preview, GA. + Works only in combination with channel. Not applicable for STS and LTS channels and will be ignored if those channels are used. + For SDK use channel in A.B.Cxx format: using quality together with channel in A.B format is not supported. + Supported since 5.0 release. + Note: The version parameter overrides the channel parameter when any version other than 'latest' is used, and therefore overrides the quality. +.PARAMETER Version + Default: latest + Represents a build version on specific channel. Possible values: + - latest - the latest build on specific channel + - 3-part version in a format A.B.C - represents specific version of build + examples: 2.0.0-preview2-006120, 1.1.0 +.PARAMETER Internal + Download internal builds. Requires providing credentials via -FeedCredential parameter. +.PARAMETER FeedCredential + Token to access Azure feed. Used as a query string to append to the Azure feed. + This parameter typically is not specified. +.PARAMETER InstallDir + Default: %LocalAppData%\Microsoft\dotnet + Path to where to install dotnet. Note that binaries will be placed directly in a given directory. +.PARAMETER Architecture + Default: - this value represents currently running OS architecture + Architecture of dotnet binaries to be installed. + Possible values are: , amd64, x64, x86, arm64, arm +.PARAMETER SharedRuntime + This parameter is obsolete and may be removed in a future version of this script. + The recommended alternative is '-Runtime dotnet'. + Installs just the shared runtime bits, not the entire SDK. +.PARAMETER Runtime + Installs just a shared runtime, not the entire SDK. + Possible values: + - dotnet - the Microsoft.NETCore.App shared runtime + - aspnetcore - the Microsoft.AspNetCore.App shared runtime + - windowsdesktop - the Microsoft.WindowsDesktop.App shared runtime +.PARAMETER DryRun + If set it will not perform installation but instead display what command line to use to consistently install + currently requested version of dotnet cli. In example if you specify version 'latest' it will display a link + with specific version so that this command can be used deterministicly in a build script. + It also displays binaries location if you prefer to install or download it yourself. +.PARAMETER NoPath + By default this script will set environment variable PATH for the current process to the binaries folder inside installation folder. + If set it will display binaries location but not set any environment variable. +.PARAMETER Verbose + Displays diagnostics information. +.PARAMETER AzureFeed + Default: https://dotnetcli.azureedge.net/dotnet + For internal use only. + Allows using a different storage to download SDK archives from. + This parameter is only used if $NoCdn is false. +.PARAMETER UncachedFeed + For internal use only. + Allows using a different storage to download SDK archives from. + This parameter is only used if $NoCdn is true. +.PARAMETER ProxyAddress + If set, the installer will use the proxy when making web requests +.PARAMETER ProxyUseDefaultCredentials + Default: false + Use default credentials, when using proxy address. +.PARAMETER ProxyBypassList + If set with ProxyAddress, will provide the list of comma separated urls that will bypass the proxy +.PARAMETER SkipNonVersionedFiles + Default: false + Skips installing non-versioned files if they already exist, such as dotnet.exe. +.PARAMETER NoCdn + Disable downloading from the Azure CDN, and use the uncached feed directly. +.PARAMETER JSonFile + Determines the SDK version from a user specified global.json file + Note: global.json must have a value for 'SDK:Version' +.PARAMETER DownloadTimeout + Determines timeout duration in seconds for dowloading of the SDK file + Default: 1200 seconds (20 minutes) +.PARAMETER KeepZip + If set, downloaded file is kept +.PARAMETER ZipPath + Use that path to store installer, generated by default +.EXAMPLE + dotnet-install.ps1 -Version 7.0.401 + Installs the .NET SDK version 7.0.401 +.EXAMPLE + dotnet-install.ps1 -Channel 8.0 -Quality GA + Installs the latest GA (general availability) version of the .NET 8.0 SDK +#> +[cmdletbinding()] +param( + [string]$Channel="LTS", + [string]$Quality, + [string]$Version="Latest", + [switch]$Internal, + [string]$JSonFile, + [Alias('i')][string]$InstallDir="", + [string]$Architecture="", + [string]$Runtime, + [Obsolete("This parameter may be removed in a future version of this script. The recommended alternative is '-Runtime dotnet'.")] + [switch]$SharedRuntime, + [switch]$DryRun, + [switch]$NoPath, + [string]$AzureFeed, + [string]$UncachedFeed, + [string]$FeedCredential, + [string]$ProxyAddress, + [switch]$ProxyUseDefaultCredentials, + [string[]]$ProxyBypassList=@(), + [switch]$SkipNonVersionedFiles, + [switch]$NoCdn, + [int]$DownloadTimeout=1200, + [switch]$KeepZip, + [string]$ZipPath=[System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()), + [switch]$Help +) + +Set-StrictMode -Version Latest +$ErrorActionPreference="Stop" +$ProgressPreference="SilentlyContinue" + +function Say($str) { + try { + Write-Host "dotnet-install: $str" + } + catch { + # Some platforms cannot utilize Write-Host (Azure Functions, for instance). Fall back to Write-Output + Write-Output "dotnet-install: $str" + } +} + +function Say-Warning($str) { + try { + Write-Warning "dotnet-install: $str" + } + catch { + # Some platforms cannot utilize Write-Warning (Azure Functions, for instance). Fall back to Write-Output + Write-Output "dotnet-install: Warning: $str" + } +} + +# Writes a line with error style settings. +# Use this function to show a human-readable comment along with an exception. +function Say-Error($str) { + try { + # Write-Error is quite oververbose for the purpose of the function, let's write one line with error style settings. + $Host.UI.WriteErrorLine("dotnet-install: $str") + } + catch { + Write-Output "dotnet-install: Error: $str" + } +} + +function Say-Verbose($str) { + try { + Write-Verbose "dotnet-install: $str" + } + catch { + # Some platforms cannot utilize Write-Verbose (Azure Functions, for instance). Fall back to Write-Output + Write-Output "dotnet-install: $str" + } +} + +function Measure-Action($name, $block) { + $time = Measure-Command $block + $totalSeconds = $time.TotalSeconds + Say-Verbose "Action '$name' took $totalSeconds seconds" +} + +function Get-Remote-File-Size($zipUri) { + try { + $response = Invoke-WebRequest -Uri $zipUri -Method Head + $fileSize = $response.Headers["Content-Length"] + if ((![string]::IsNullOrEmpty($fileSize))) { + Say "Remote file $zipUri size is $fileSize bytes." + + return $fileSize + } + } + catch { + Say-Verbose "Content-Length header was not extracted for $zipUri." + } + + return $null +} + +function Say-Invocation($Invocation) { + $command = $Invocation.MyCommand; + $args = (($Invocation.BoundParameters.Keys | foreach { "-$_ `"$($Invocation.BoundParameters[$_])`"" }) -join " ") + Say-Verbose "$command $args" +} + +function Invoke-With-Retry([ScriptBlock]$ScriptBlock, [System.Threading.CancellationToken]$cancellationToken = [System.Threading.CancellationToken]::None, [int]$MaxAttempts = 3, [int]$SecondsBetweenAttempts = 1) { + $Attempts = 0 + $local:startTime = $(get-date) + + while ($true) { + try { + return & $ScriptBlock + } + catch { + $Attempts++ + if (($Attempts -lt $MaxAttempts) -and -not $cancellationToken.IsCancellationRequested) { + Start-Sleep $SecondsBetweenAttempts + } + else { + $local:elapsedTime = $(get-date) - $local:startTime + if (($local:elapsedTime.TotalSeconds - $DownloadTimeout) -gt 0 -and -not $cancellationToken.IsCancellationRequested) { + throw New-Object System.TimeoutException("Failed to reach the server: connection timeout: default timeout is $DownloadTimeout second(s)"); + } + throw; + } + } + } +} + +function Get-Machine-Architecture() { + Say-Invocation $MyInvocation + + # On PS x86, PROCESSOR_ARCHITECTURE reports x86 even on x64 systems. + # To get the correct architecture, we need to use PROCESSOR_ARCHITEW6432. + # PS x64 doesn't define this, so we fall back to PROCESSOR_ARCHITECTURE. + # Possible values: amd64, x64, x86, arm64, arm + if( $ENV:PROCESSOR_ARCHITEW6432 -ne $null ) { + return $ENV:PROCESSOR_ARCHITEW6432 + } + + try { + if( ((Get-CimInstance -ClassName CIM_OperatingSystem).OSArchitecture) -like "ARM*") { + if( [Environment]::Is64BitOperatingSystem ) + { + return "arm64" + } + return "arm" + } + } + catch { + # Machine doesn't support Get-CimInstance + } + + return $ENV:PROCESSOR_ARCHITECTURE +} + +function Get-CLIArchitecture-From-Architecture([string]$Architecture) { + Say-Invocation $MyInvocation + + if ($Architecture -eq "") { + $Architecture = Get-Machine-Architecture + } + + switch ($Architecture.ToLowerInvariant()) { + { ($_ -eq "amd64") -or ($_ -eq "x64") } { return "x64" } + { $_ -eq "x86" } { return "x86" } + { $_ -eq "arm" } { return "arm" } + { $_ -eq "arm64" } { return "arm64" } + default { throw "Architecture '$Architecture' not supported. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues" } + } +} + +function ValidateFeedCredential([string] $FeedCredential) +{ + if ($Internal -and [string]::IsNullOrWhitespace($FeedCredential)) { + $message = "Provide credentials via -FeedCredential parameter." + if ($DryRun) { + Say-Warning "$message" + } else { + throw "$message" + } + } + + #FeedCredential should start with "?", for it to be added to the end of the link. + #adding "?" at the beginning of the FeedCredential if needed. + if ((![string]::IsNullOrWhitespace($FeedCredential)) -and ($FeedCredential[0] -ne '?')) { + $FeedCredential = "?" + $FeedCredential + } + + return $FeedCredential +} +function Get-NormalizedQuality([string]$Quality) { + Say-Invocation $MyInvocation + + if ([string]::IsNullOrEmpty($Quality)) { + return "" + } + + switch ($Quality) { + { @("daily", "signed", "validated", "preview") -contains $_ } { return $Quality.ToLowerInvariant() } + #ga quality is available without specifying quality, so normalizing it to empty + { $_ -eq "ga" } { return "" } + default { throw "'$Quality' is not a supported value for -Quality option. Supported values are: daily, signed, validated, preview, ga. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues." } + } +} + +function Get-NormalizedChannel([string]$Channel) { + Say-Invocation $MyInvocation + + if ([string]::IsNullOrEmpty($Channel)) { + return "" + } + + if ($Channel.Contains("Current")) { + Say-Warning 'Value "Current" is deprecated for -Channel option. Use "STS" instead.' + } + + if ($Channel.StartsWith('release/')) { + Say-Warning 'Using branch name with -Channel option is no longer supported with newer releases. Use -Quality option with a channel in X.Y format instead, such as "-Channel 5.0 -Quality Daily."' + } + + switch ($Channel) { + { $_ -eq "lts" } { return "LTS" } + { $_ -eq "sts" } { return "STS" } + { $_ -eq "current" } { return "STS" } + default { return $Channel.ToLowerInvariant() } + } +} + +function Get-NormalizedProduct([string]$Runtime) { + Say-Invocation $MyInvocation + + switch ($Runtime) { + { $_ -eq "dotnet" } { return "dotnet-runtime" } + { $_ -eq "aspnetcore" } { return "aspnetcore-runtime" } + { $_ -eq "windowsdesktop" } { return "windowsdesktop-runtime" } + { [string]::IsNullOrEmpty($_) } { return "dotnet-sdk" } + default { throw "'$Runtime' is not a supported value for -Runtime option, supported values are: dotnet, aspnetcore, windowsdesktop. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues." } + } +} + + +# The version text returned from the feeds is a 1-line or 2-line string: +# For the SDK and the dotnet runtime (2 lines): +# Line 1: # commit_hash +# Line 2: # 4-part version +# For the aspnetcore runtime (1 line): +# Line 1: # 4-part version +function Get-Version-From-LatestVersion-File-Content([string]$VersionText) { + Say-Invocation $MyInvocation + + $Data = -split $VersionText + + $VersionInfo = @{ + CommitHash = $(if ($Data.Count -gt 1) { $Data[0] }) + Version = $Data[-1] # last line is always the version number. + } + return $VersionInfo +} + +function Load-Assembly([string] $Assembly) { + try { + Add-Type -Assembly $Assembly | Out-Null + } + catch { + # On Nano Server, Powershell Core Edition is used. Add-Type is unable to resolve base class assemblies because they are not GAC'd. + # Loading the base class assemblies is not unnecessary as the types will automatically get resolved. + } +} + +function GetHTTPResponse([Uri] $Uri, [bool]$HeaderOnly, [bool]$DisableRedirect, [bool]$DisableFeedCredential) +{ + $cts = New-Object System.Threading.CancellationTokenSource + + $downloadScript = { + + $HttpClient = $null + + try { + # HttpClient is used vs Invoke-WebRequest in order to support Nano Server which doesn't support the Invoke-WebRequest cmdlet. + Load-Assembly -Assembly System.Net.Http + + if(-not $ProxyAddress) { + try { + # Despite no proxy being explicitly specified, we may still be behind a default proxy + $DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy; + if($DefaultProxy -and (-not $DefaultProxy.IsBypassed($Uri))) { + if ($null -ne $DefaultProxy.GetProxy($Uri)) { + $ProxyAddress = $DefaultProxy.GetProxy($Uri).OriginalString + } else { + $ProxyAddress = $null + } + $ProxyUseDefaultCredentials = $true + } + } catch { + # Eat the exception and move forward as the above code is an attempt + # at resolving the DefaultProxy that may not have been a problem. + $ProxyAddress = $null + Say-Verbose("Exception ignored: $_.Exception.Message - moving forward...") + } + } + + $HttpClientHandler = New-Object System.Net.Http.HttpClientHandler + if($ProxyAddress) { + $HttpClientHandler.Proxy = New-Object System.Net.WebProxy -Property @{ + Address=$ProxyAddress; + UseDefaultCredentials=$ProxyUseDefaultCredentials; + BypassList = $ProxyBypassList; + } + } + if ($DisableRedirect) + { + $HttpClientHandler.AllowAutoRedirect = $false + } + $HttpClient = New-Object System.Net.Http.HttpClient -ArgumentList $HttpClientHandler + + # Default timeout for HttpClient is 100s. For a 50 MB download this assumes 500 KB/s average, any less will time out + # Defaulting to 20 minutes allows it to work over much slower connections. + $HttpClient.Timeout = New-TimeSpan -Seconds $DownloadTimeout + + if ($HeaderOnly){ + $completionOption = [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead + } + else { + $completionOption = [System.Net.Http.HttpCompletionOption]::ResponseContentRead + } + + if ($DisableFeedCredential) { + $UriWithCredential = $Uri + } + else { + $UriWithCredential = "${Uri}${FeedCredential}" + } + + $Task = $HttpClient.GetAsync("$UriWithCredential", $completionOption).ConfigureAwait("false"); + $Response = $Task.GetAwaiter().GetResult(); + + if (($null -eq $Response) -or ((-not $HeaderOnly) -and (-not ($Response.IsSuccessStatusCode)))) { + # The feed credential is potentially sensitive info. Do not log FeedCredential to console output. + $DownloadException = [System.Exception] "Unable to download $Uri." + + if ($null -ne $Response) { + $DownloadException.Data["StatusCode"] = [int] $Response.StatusCode + $DownloadException.Data["ErrorMessage"] = "Unable to download $Uri. Returned HTTP status code: " + $DownloadException.Data["StatusCode"] + + if (404 -eq [int] $Response.StatusCode) + { + $cts.Cancel() + } + } + + throw $DownloadException + } + + return $Response + } + catch [System.Net.Http.HttpRequestException] { + $DownloadException = [System.Exception] "Unable to download $Uri." + + # Pick up the exception message and inner exceptions' messages if they exist + $CurrentException = $PSItem.Exception + $ErrorMsg = $CurrentException.Message + "`r`n" + while ($CurrentException.InnerException) { + $CurrentException = $CurrentException.InnerException + $ErrorMsg += $CurrentException.Message + "`r`n" + } + + # Check if there is an issue concerning TLS. + if ($ErrorMsg -like "*SSL/TLS*") { + $ErrorMsg += "Ensure that TLS 1.2 or higher is enabled to use this script.`r`n" + } + + $DownloadException.Data["ErrorMessage"] = $ErrorMsg + throw $DownloadException + } + finally { + if ($null -ne $HttpClient) { + $HttpClient.Dispose() + } + } + } + + try { + return Invoke-With-Retry $downloadScript $cts.Token + } + finally + { + if ($null -ne $cts) + { + $cts.Dispose() + } + } +} + +function Get-Version-From-LatestVersion-File([string]$AzureFeed, [string]$Channel) { + Say-Invocation $MyInvocation + + $VersionFileUrl = $null + if ($Runtime -eq "dotnet") { + $VersionFileUrl = "$AzureFeed/Runtime/$Channel/latest.version" + } + elseif ($Runtime -eq "aspnetcore") { + $VersionFileUrl = "$AzureFeed/aspnetcore/Runtime/$Channel/latest.version" + } + elseif ($Runtime -eq "windowsdesktop") { + $VersionFileUrl = "$AzureFeed/WindowsDesktop/$Channel/latest.version" + } + elseif (-not $Runtime) { + $VersionFileUrl = "$AzureFeed/Sdk/$Channel/latest.version" + } + else { + throw "Invalid value for `$Runtime" + } + + Say-Verbose "Constructed latest.version URL: $VersionFileUrl" + + try { + $Response = GetHTTPResponse -Uri $VersionFileUrl + } + catch { + Say-Verbose "Failed to download latest.version file." + throw + } + $StringContent = $Response.Content.ReadAsStringAsync().Result + + switch ($Response.Content.Headers.ContentType) { + { ($_ -eq "application/octet-stream") } { $VersionText = $StringContent } + { ($_ -eq "text/plain") } { $VersionText = $StringContent } + { ($_ -eq "text/plain; charset=UTF-8") } { $VersionText = $StringContent } + default { throw "``$Response.Content.Headers.ContentType`` is an unknown .version file content type." } + } + + $VersionInfo = Get-Version-From-LatestVersion-File-Content $VersionText + + return $VersionInfo +} + +function Parse-Jsonfile-For-Version([string]$JSonFile) { + Say-Invocation $MyInvocation + + If (-Not (Test-Path $JSonFile)) { + throw "Unable to find '$JSonFile'" + } + try { + $JSonContent = Get-Content($JSonFile) -Raw | ConvertFrom-Json | Select-Object -expand "sdk" -ErrorAction SilentlyContinue + } + catch { + Say-Error "Json file unreadable: '$JSonFile'" + throw + } + if ($JSonContent) { + try { + $JSonContent.PSObject.Properties | ForEach-Object { + $PropertyName = $_.Name + if ($PropertyName -eq "version") { + $Version = $_.Value + Say-Verbose "Version = $Version" + } + } + } + catch { + Say-Error "Unable to parse the SDK node in '$JSonFile'" + throw + } + } + else { + throw "Unable to find the SDK node in '$JSonFile'" + } + If ($Version -eq $null) { + throw "Unable to find the SDK:version node in '$JSonFile'" + } + return $Version +} + +function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel, [string]$Version, [string]$JSonFile) { + Say-Invocation $MyInvocation + + if (-not $JSonFile) { + if ($Version.ToLowerInvariant() -eq "latest") { + $LatestVersionInfo = Get-Version-From-LatestVersion-File -AzureFeed $AzureFeed -Channel $Channel + return $LatestVersionInfo.Version + } + else { + return $Version + } + } + else { + return Parse-Jsonfile-For-Version $JSonFile + } +} + +function Get-Download-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) { + Say-Invocation $MyInvocation + + # If anything fails in this lookup it will default to $SpecificVersion + $SpecificProductVersion = Get-Product-Version -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion + + if ($Runtime -eq "dotnet") { + $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" + } + elseif ($Runtime -eq "aspnetcore") { + $PayloadURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/aspnetcore-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" + } + elseif ($Runtime -eq "windowsdesktop") { + # The windows desktop runtime is part of the core runtime layout prior to 5.0 + $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/windowsdesktop-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" + if ($SpecificVersion -match '^(\d+)\.(.*)$') + { + $majorVersion = [int]$Matches[1] + if ($majorVersion -ge 5) + { + $PayloadURL = "$AzureFeed/WindowsDesktop/$SpecificVersion/windowsdesktop-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" + } + } + } + elseif (-not $Runtime) { + $PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-sdk-$SpecificProductVersion-win-$CLIArchitecture.zip" + } + else { + throw "Invalid value for `$Runtime" + } + + Say-Verbose "Constructed primary named payload URL: $PayloadURL" + + return $PayloadURL, $SpecificProductVersion +} + +function Get-LegacyDownload-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) { + Say-Invocation $MyInvocation + + if (-not $Runtime) { + $PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-dev-win-$CLIArchitecture.$SpecificVersion.zip" + } + elseif ($Runtime -eq "dotnet") { + $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-win-$CLIArchitecture.$SpecificVersion.zip" + } + else { + return $null + } + + Say-Verbose "Constructed legacy named payload URL: $PayloadURL" + + return $PayloadURL +} + +function Get-Product-Version([string]$AzureFeed, [string]$SpecificVersion, [string]$PackageDownloadLink) { + Say-Invocation $MyInvocation + + # Try to get the version number, using the productVersion.txt file located next to the installer file. + $ProductVersionTxtURLs = (Get-Product-Version-Url $AzureFeed $SpecificVersion $PackageDownloadLink -Flattened $true), + (Get-Product-Version-Url $AzureFeed $SpecificVersion $PackageDownloadLink -Flattened $false) + + Foreach ($ProductVersionTxtURL in $ProductVersionTxtURLs) { + Say-Verbose "Checking for the existence of $ProductVersionTxtURL" + + try { + $productVersionResponse = GetHTTPResponse($productVersionTxtUrl) + + if ($productVersionResponse.StatusCode -eq 200) { + $productVersion = $productVersionResponse.Content.ReadAsStringAsync().Result.Trim() + if ($productVersion -ne $SpecificVersion) + { + Say "Using alternate version $productVersion found in $ProductVersionTxtURL" + } + return $productVersion + } + else { + Say-Verbose "Got StatusCode $($productVersionResponse.StatusCode) when trying to get productVersion.txt at $productVersionTxtUrl." + } + } + catch { + Say-Verbose "Could not read productVersion.txt at $productVersionTxtUrl (Exception: '$($_.Exception.Message)'. )" + } + } + + # Getting the version number with productVersion.txt has failed. Try parsing the download link for a version number. + if ([string]::IsNullOrEmpty($PackageDownloadLink)) + { + Say-Verbose "Using the default value '$SpecificVersion' as the product version." + return $SpecificVersion + } + + $productVersion = Get-ProductVersionFromDownloadLink $PackageDownloadLink $SpecificVersion + return $productVersion +} + +function Get-Product-Version-Url([string]$AzureFeed, [string]$SpecificVersion, [string]$PackageDownloadLink, [bool]$Flattened) { + Say-Invocation $MyInvocation + + $majorVersion=$null + if ($SpecificVersion -match '^(\d+)\.(.*)') { + $majorVersion = $Matches[1] -as[int] + } + + $pvFileName='productVersion.txt' + if($Flattened) { + if(-not $Runtime) { + $pvFileName='sdk-productVersion.txt' + } + elseif($Runtime -eq "dotnet") { + $pvFileName='runtime-productVersion.txt' + } + else { + $pvFileName="$Runtime-productVersion.txt" + } + } + + if ([string]::IsNullOrEmpty($PackageDownloadLink)) { + if ($Runtime -eq "dotnet") { + $ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/$pvFileName" + } + elseif ($Runtime -eq "aspnetcore") { + $ProductVersionTxtURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/$pvFileName" + } + elseif ($Runtime -eq "windowsdesktop") { + # The windows desktop runtime is part of the core runtime layout prior to 5.0 + $ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/$pvFileName" + if ($majorVersion -ne $null -and $majorVersion -ge 5) { + $ProductVersionTxtURL = "$AzureFeed/WindowsDesktop/$SpecificVersion/$pvFileName" + } + } + elseif (-not $Runtime) { + $ProductVersionTxtURL = "$AzureFeed/Sdk/$SpecificVersion/$pvFileName" + } + else { + throw "Invalid value '$Runtime' specified for `$Runtime" + } + } + else { + $ProductVersionTxtURL = $PackageDownloadLink.Substring(0, $PackageDownloadLink.LastIndexOf("/")) + "/$pvFileName" + } + + Say-Verbose "Constructed productVersion link: $ProductVersionTxtURL" + + return $ProductVersionTxtURL +} + +function Get-ProductVersionFromDownloadLink([string]$PackageDownloadLink, [string]$SpecificVersion) +{ + Say-Invocation $MyInvocation + + #product specific version follows the product name + #for filename 'dotnet-sdk-3.1.404-win-x64.zip': the product version is 3.1.400 + $filename = $PackageDownloadLink.Substring($PackageDownloadLink.LastIndexOf("/") + 1) + $filenameParts = $filename.Split('-') + if ($filenameParts.Length -gt 2) + { + $productVersion = $filenameParts[2] + Say-Verbose "Extracted product version '$productVersion' from download link '$PackageDownloadLink'." + } + else { + Say-Verbose "Using the default value '$SpecificVersion' as the product version." + $productVersion = $SpecificVersion + } + return $productVersion +} + +function Get-User-Share-Path() { + Say-Invocation $MyInvocation + + $InstallRoot = $env:DOTNET_INSTALL_DIR + if (!$InstallRoot) { + $InstallRoot = "$env:LocalAppData\Microsoft\dotnet" + } + elseif ($InstallRoot -like "$env:ProgramFiles\dotnet\?*") { + Say-Warning "The install root specified by the environment variable DOTNET_INSTALL_DIR points to the sub folder of $env:ProgramFiles\dotnet which is the default dotnet install root using .NET SDK installer. It is better to keep aligned with .NET SDK installer." + } + return $InstallRoot +} + +function Resolve-Installation-Path([string]$InstallDir) { + Say-Invocation $MyInvocation + + if ($InstallDir -eq "") { + return Get-User-Share-Path + } + return $InstallDir +} + +function Test-User-Write-Access([string]$InstallDir) { + try { + $tempFileName=[guid]::NewGuid().ToString() + $tempFilePath=Join-Path -Path $InstallDir -ChildPath $tempFileName + New-Item -Path $tempFilePath -ItemType File -Force + Remove-Item $tempFilePath -Force + return $true + } catch { + return $false + } +} + +function Is-Dotnet-Package-Installed([string]$InstallRoot, [string]$RelativePathToPackage, [string]$SpecificVersion) { + Say-Invocation $MyInvocation + + $DotnetPackagePath = Join-Path -Path $InstallRoot -ChildPath $RelativePathToPackage | Join-Path -ChildPath $SpecificVersion + Say-Verbose "Is-Dotnet-Package-Installed: DotnetPackagePath=$DotnetPackagePath" + return Test-Path $DotnetPackagePath -PathType Container +} + +function Get-Absolute-Path([string]$RelativeOrAbsolutePath) { + # Too much spam + # Say-Invocation $MyInvocation + + return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($RelativeOrAbsolutePath) +} + +function Get-Path-Prefix-With-Version($path) { + # example path with regex: shared/1.0.0-beta-12345/somepath + $match = [regex]::match($path, "/\d+\.\d+[^/]+/") + if ($match.Success) { + return $entry.FullName.Substring(0, $match.Index + $match.Length) + } + + return $null +} + +function Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package([System.IO.Compression.ZipArchive]$Zip, [string]$OutPath) { + Say-Invocation $MyInvocation + + $ret = @() + foreach ($entry in $Zip.Entries) { + $dir = Get-Path-Prefix-With-Version $entry.FullName + if ($null -ne $dir) { + $path = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $dir) + if (-Not (Test-Path $path -PathType Container)) { + $ret += $dir + } + } + } + + $ret = $ret | Sort-Object | Get-Unique + + $values = ($ret | foreach { "$_" }) -join ";" + Say-Verbose "Directories to unpack: $values" + + return $ret +} + +# Example zip content and extraction algorithm: +# Rule: files if extracted are always being extracted to the same relative path locally +# .\ +# a.exe # file does not exist locally, extract +# b.dll # file exists locally, override only if $OverrideFiles set +# aaa\ # same rules as for files +# ... +# abc\1.0.0\ # directory contains version and exists locally +# ... # do not extract content under versioned part +# abc\asd\ # same rules as for files +# ... +# def\ghi\1.0.1\ # directory contains version and does not exist locally +# ... # extract content +function Extract-Dotnet-Package([string]$ZipPath, [string]$OutPath) { + Say-Invocation $MyInvocation + + Load-Assembly -Assembly System.IO.Compression.FileSystem + Set-Variable -Name Zip + try { + $Zip = [System.IO.Compression.ZipFile]::OpenRead($ZipPath) + + $DirectoriesToUnpack = Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package -Zip $Zip -OutPath $OutPath + + foreach ($entry in $Zip.Entries) { + $PathWithVersion = Get-Path-Prefix-With-Version $entry.FullName + if (($null -eq $PathWithVersion) -Or ($DirectoriesToUnpack -contains $PathWithVersion)) { + $DestinationPath = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $entry.FullName) + $DestinationDir = Split-Path -Parent $DestinationPath + $OverrideFiles=$OverrideNonVersionedFiles -Or (-Not (Test-Path $DestinationPath)) + if ((-Not $DestinationPath.EndsWith("\")) -And $OverrideFiles) { + New-Item -ItemType Directory -Force -Path $DestinationDir | Out-Null + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($entry, $DestinationPath, $OverrideNonVersionedFiles) + } + } + } + } + catch + { + Say-Error "Failed to extract package. Exception: $_" + throw; + } + finally { + if ($null -ne $Zip) { + $Zip.Dispose() + } + } +} + +function DownloadFile($Source, [string]$OutPath) { + if ($Source -notlike "http*") { + # Using System.IO.Path.GetFullPath to get the current directory + # does not work in this context - $pwd gives the current directory + if (![System.IO.Path]::IsPathRooted($Source)) { + $Source = $(Join-Path -Path $pwd -ChildPath $Source) + } + $Source = Get-Absolute-Path $Source + Say "Copying file from $Source to $OutPath" + Copy-Item $Source $OutPath + return + } + + $Stream = $null + + try { + $Response = GetHTTPResponse -Uri $Source + $Stream = $Response.Content.ReadAsStreamAsync().Result + $File = [System.IO.File]::Create($OutPath) + $Stream.CopyTo($File) + $File.Close() + + ValidateRemoteLocalFileSizes -LocalFileOutPath $OutPath -SourceUri $Source + } + finally { + if ($null -ne $Stream) { + $Stream.Dispose() + } + } +} + +function ValidateRemoteLocalFileSizes([string]$LocalFileOutPath, $SourceUri) { + try { + $remoteFileSize = Get-Remote-File-Size -zipUri $SourceUri + $fileSize = [long](Get-Item $LocalFileOutPath).Length + Say "Downloaded file $SourceUri size is $fileSize bytes." + + if ((![string]::IsNullOrEmpty($remoteFileSize)) -and !([string]::IsNullOrEmpty($fileSize)) ) { + if ($remoteFileSize -ne $fileSize) { + Say "The remote and local file sizes are not equal. Remote file size is $remoteFileSize bytes and local size is $fileSize bytes. The local package may be corrupted." + } + else { + Say "The remote and local file sizes are equal." + } + } + else { + Say "Either downloaded or local package size can not be measured. One of them may be corrupted." + } + } + catch { + Say "Either downloaded or local package size can not be measured. One of them may be corrupted." + } +} + +function SafeRemoveFile($Path) { + try { + if (Test-Path $Path) { + Remove-Item $Path + Say-Verbose "The temporary file `"$Path`" was removed." + } + else { + Say-Verbose "The temporary file `"$Path`" does not exist, therefore is not removed." + } + } + catch { + Say-Warning "Failed to remove the temporary file: `"$Path`", remove it manually." + } +} + +function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot) { + $BinPath = Get-Absolute-Path $(Join-Path -Path $InstallRoot -ChildPath "") + if (-Not $NoPath) { + $SuffixedBinPath = "$BinPath;" + if (-Not $env:path.Contains($SuffixedBinPath)) { + Say "Adding to current process PATH: `"$BinPath`". Note: This change will not be visible if PowerShell was run as a child process." + $env:path = $SuffixedBinPath + $env:path + } else { + Say-Verbose "Current process PATH already contains `"$BinPath`"" + } + } + else { + Say "Binaries of dotnet can be found in $BinPath" + } +} + +function PrintDryRunOutput($Invocation, $DownloadLinks) +{ + Say "Payload URLs:" + + for ($linkIndex=0; $linkIndex -lt $DownloadLinks.count; $linkIndex++) { + Say "URL #$linkIndex - $($DownloadLinks[$linkIndex].type): $($DownloadLinks[$linkIndex].downloadLink)" + } + $RepeatableCommand = ".\$ScriptName -Version `"$SpecificVersion`" -InstallDir `"$InstallRoot`" -Architecture `"$CLIArchitecture`"" + if ($Runtime -eq "dotnet") { + $RepeatableCommand+=" -Runtime `"dotnet`"" + } + elseif ($Runtime -eq "aspnetcore") { + $RepeatableCommand+=" -Runtime `"aspnetcore`"" + } + + foreach ($key in $Invocation.BoundParameters.Keys) { + if (-not (@("Architecture","Channel","DryRun","InstallDir","Runtime","SharedRuntime","Version","Quality","FeedCredential") -contains $key)) { + $RepeatableCommand+=" -$key `"$($Invocation.BoundParameters[$key])`"" + } + } + if ($Invocation.BoundParameters.Keys -contains "FeedCredential") { + $RepeatableCommand+=" -FeedCredential `"`"" + } + Say "Repeatable invocation: $RepeatableCommand" + if ($SpecificVersion -ne $EffectiveVersion) + { + Say "NOTE: Due to finding a version manifest with this runtime, it would actually install with version '$EffectiveVersion'" + } +} + +function Get-AkaMSDownloadLink([string]$Channel, [string]$Quality, [bool]$Internal, [string]$Product, [string]$Architecture) { + Say-Invocation $MyInvocation + + #quality is not supported for LTS or STS channel + if (![string]::IsNullOrEmpty($Quality) -and (@("LTS", "STS") -contains $Channel)) { + $Quality = "" + Say-Warning "Specifying quality for STS or LTS channel is not supported, the quality will be ignored." + } + Say-Verbose "Retrieving primary payload URL from aka.ms link for channel: '$Channel', quality: '$Quality' product: '$Product', os: 'win', architecture: '$Architecture'." + + #construct aka.ms link + $akaMsLink = "https://aka.ms/dotnet" + if ($Internal) { + $akaMsLink += "/internal" + } + $akaMsLink += "/$Channel" + if (-not [string]::IsNullOrEmpty($Quality)) { + $akaMsLink +="/$Quality" + } + $akaMsLink +="/$Product-win-$Architecture.zip" + Say-Verbose "Constructed aka.ms link: '$akaMsLink'." + $akaMsDownloadLink=$null + + for ($maxRedirections = 9; $maxRedirections -ge 0; $maxRedirections--) + { + #get HTTP response + #do not pass credentials as a part of the $akaMsLink and do not apply credentials in the GetHTTPResponse function + #otherwise the redirect link would have credentials as well + #it would result in applying credentials twice to the resulting link and thus breaking it, and in echoing credentials to the output as a part of redirect link + $Response= GetHTTPResponse -Uri $akaMsLink -HeaderOnly $true -DisableRedirect $true -DisableFeedCredential $true + Say-Verbose "Received response:`n$Response" + + if ([string]::IsNullOrEmpty($Response)) { + Say-Verbose "The link '$akaMsLink' is not valid: failed to get redirect location. The resource is not available." + return $null + } + + #if HTTP code is 301 (Moved Permanently), the redirect link exists + if ($Response.StatusCode -eq 301) + { + try { + $akaMsDownloadLink = $Response.Headers.GetValues("Location")[0] + + if ([string]::IsNullOrEmpty($akaMsDownloadLink)) { + Say-Verbose "The link '$akaMsLink' is not valid: server returned 301 (Moved Permanently), but the headers do not contain the redirect location." + return $null + } + + Say-Verbose "The redirect location retrieved: '$akaMsDownloadLink'." + # This may yet be a link to another redirection. Attempt to retrieve the page again. + $akaMsLink = $akaMsDownloadLink + continue + } + catch { + Say-Verbose "The link '$akaMsLink' is not valid: failed to get redirect location." + return $null + } + } + elseif ((($Response.StatusCode -lt 300) -or ($Response.StatusCode -ge 400)) -and (-not [string]::IsNullOrEmpty($akaMsDownloadLink))) + { + # Redirections have ended. + return $akaMsDownloadLink + } + + Say-Verbose "The link '$akaMsLink' is not valid: failed to retrieve the redirection location." + return $null + } + + Say-Verbose "Aka.ms links have redirected more than the maximum allowed redirections. This may be caused by a cyclic redirection of aka.ms links." + return $null + +} + +function Get-AkaMsLink-And-Version([string] $NormalizedChannel, [string] $NormalizedQuality, [bool] $Internal, [string] $ProductName, [string] $Architecture) { + $AkaMsDownloadLink = Get-AkaMSDownloadLink -Channel $NormalizedChannel -Quality $NormalizedQuality -Internal $Internal -Product $ProductName -Architecture $Architecture + + if ([string]::IsNullOrEmpty($AkaMsDownloadLink)){ + if (-not [string]::IsNullOrEmpty($NormalizedQuality)) { + # if quality is specified - exit with error - there is no fallback approach + Say-Error "Failed to locate the latest version in the channel '$NormalizedChannel' with '$NormalizedQuality' quality for '$ProductName', os: 'win', architecture: '$Architecture'." + Say-Error "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support." + throw "aka.ms link resolution failure" + } + Say-Verbose "Falling back to latest.version file approach." + return ($null, $null, $null) + } + else { + Say-Verbose "Retrieved primary named payload URL from aka.ms link: '$AkaMsDownloadLink'." + Say-Verbose "Downloading using legacy url will not be attempted." + + #get version from the path + $pathParts = $AkaMsDownloadLink.Split('/') + if ($pathParts.Length -ge 2) { + $SpecificVersion = $pathParts[$pathParts.Length - 2] + Say-Verbose "Version: '$SpecificVersion'." + } + else { + Say-Error "Failed to extract the version from download link '$AkaMsDownloadLink'." + return ($null, $null, $null) + } + + #retrieve effective (product) version + $EffectiveVersion = Get-Product-Version -SpecificVersion $SpecificVersion -PackageDownloadLink $AkaMsDownloadLink + Say-Verbose "Product version: '$EffectiveVersion'." + + return ($AkaMsDownloadLink, $SpecificVersion, $EffectiveVersion); + } +} + +function Get-Feeds-To-Use() +{ + $feeds = @( + "https://dotnetcli.azureedge.net/dotnet", + "https://dotnetbuilds.azureedge.net/public" + ) + + if (-not [string]::IsNullOrEmpty($AzureFeed)) { + $feeds = @($AzureFeed) + } + + if ($NoCdn) { + $feeds = @( + "https://dotnetcli.blob.core.windows.net/dotnet", + "https://dotnetbuilds.blob.core.windows.net/public" + ) + + if (-not [string]::IsNullOrEmpty($UncachedFeed)) { + $feeds = @($UncachedFeed) + } + } + + return $feeds +} + +function Resolve-AssetName-And-RelativePath([string] $Runtime) { + + if ($Runtime -eq "dotnet") { + $assetName = ".NET Core Runtime" + $dotnetPackageRelativePath = "shared\Microsoft.NETCore.App" + } + elseif ($Runtime -eq "aspnetcore") { + $assetName = "ASP.NET Core Runtime" + $dotnetPackageRelativePath = "shared\Microsoft.AspNetCore.App" + } + elseif ($Runtime -eq "windowsdesktop") { + $assetName = ".NET Core Windows Desktop Runtime" + $dotnetPackageRelativePath = "shared\Microsoft.WindowsDesktop.App" + } + elseif (-not $Runtime) { + $assetName = ".NET Core SDK" + $dotnetPackageRelativePath = "sdk" + } + else { + throw "Invalid value for `$Runtime" + } + + return ($assetName, $dotnetPackageRelativePath) +} + +function Prepare-Install-Directory { + $diskSpaceWarning = "Failed to check the disk space. Installation will continue, but it may fail if you do not have enough disk space."; + + if ($PSVersionTable.PSVersion.Major -lt 7) { + Say-Verbose $diskSpaceWarning + return + } + + New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null + + $installDrive = $((Get-Item $InstallRoot -Force).PSDrive.Name); + $diskInfo = $null + try { + $diskInfo = Get-PSDrive -Name $installDrive + } + catch { + Say-Warning $diskSpaceWarning + } + + # The check is relevant for PS version >= 7, the result can be irrelevant for older versions. See https://github.com/PowerShell/PowerShell/issues/12442. + if ( ($null -ne $diskInfo) -and ($diskInfo.Free / 1MB -le 100)) { + throw "There is not enough disk space on drive ${installDrive}:" + } +} + +if ($Help) +{ + Get-Help $PSCommandPath -Examples + exit +} + +Say-Verbose "Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:" +Say-Verbose "- The SDK needs to be installed without user interaction and without admin rights." +Say-Verbose "- The SDK installation doesn't need to persist across multiple CI runs." +Say-Verbose "To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer.`r`n" + +if ($SharedRuntime -and (-not $Runtime)) { + $Runtime = "dotnet" +} + +$OverrideNonVersionedFiles = !$SkipNonVersionedFiles + +Measure-Action "Product discovery" { + $script:CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture + $script:NormalizedQuality = Get-NormalizedQuality $Quality + Say-Verbose "Normalized quality: '$NormalizedQuality'" + $script:NormalizedChannel = Get-NormalizedChannel $Channel + Say-Verbose "Normalized channel: '$NormalizedChannel'" + $script:NormalizedProduct = Get-NormalizedProduct $Runtime + Say-Verbose "Normalized product: '$NormalizedProduct'" + $script:FeedCredential = ValidateFeedCredential $FeedCredential +} + +$InstallRoot = Resolve-Installation-Path $InstallDir +if (-not (Test-User-Write-Access $InstallRoot)) { + Say-Error "The current user doesn't have write access to the installation root '$InstallRoot' to install .NET. Please try specifying a different installation directory using the -InstallDir parameter, or ensure the selected directory has the appropriate permissions." + throw +} +Say-Verbose "InstallRoot: $InstallRoot" +$ScriptName = $MyInvocation.MyCommand.Name +($assetName, $dotnetPackageRelativePath) = Resolve-AssetName-And-RelativePath -Runtime $Runtime + +$feeds = Get-Feeds-To-Use +$DownloadLinks = @() + +if ($Version.ToLowerInvariant() -ne "latest" -and -not [string]::IsNullOrEmpty($Quality)) { + throw "Quality and Version options are not allowed to be specified simultaneously. See https:// learn.microsoft.com/dotnet/core/tools/dotnet-install-script#options for details." +} + +# aka.ms links can only be used if the user did not request a specific version via the command line or a global.json file. +if ([string]::IsNullOrEmpty($JSonFile) -and ($Version -eq "latest")) { + ($DownloadLink, $SpecificVersion, $EffectiveVersion) = Get-AkaMsLink-And-Version $NormalizedChannel $NormalizedQuality $Internal $NormalizedProduct $CLIArchitecture + + if ($null -ne $DownloadLink) { + $DownloadLinks += New-Object PSObject -Property @{downloadLink="$DownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='aka.ms'} + Say-Verbose "Generated aka.ms link $DownloadLink with version $EffectiveVersion" + + if (-Not $DryRun) { + Say-Verbose "Checking if the version $EffectiveVersion is already installed" + if (Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $EffectiveVersion) + { + Say "$assetName with version '$EffectiveVersion' is already installed." + Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot + return + } + } + } +} + +# Primary and legacy links cannot be used if a quality was specified. +# If we already have an aka.ms link, no need to search the blob feeds. +if ([string]::IsNullOrEmpty($NormalizedQuality) -and 0 -eq $DownloadLinks.count) +{ + foreach ($feed in $feeds) { + try { + $SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $feed -Channel $Channel -Version $Version -JSonFile $JSonFile + $DownloadLink, $EffectiveVersion = Get-Download-Link -AzureFeed $feed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture + $LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $feed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture + + $DownloadLinks += New-Object PSObject -Property @{downloadLink="$DownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='primary'} + Say-Verbose "Generated primary link $DownloadLink with version $EffectiveVersion" + + if (-not [string]::IsNullOrEmpty($LegacyDownloadLink)) { + $DownloadLinks += New-Object PSObject -Property @{downloadLink="$LegacyDownloadLink";specificVersion="$SpecificVersion";effectiveVersion="$EffectiveVersion";type='legacy'} + Say-Verbose "Generated legacy link $LegacyDownloadLink with version $EffectiveVersion" + } + + if (-Not $DryRun) { + Say-Verbose "Checking if the version $EffectiveVersion is already installed" + if (Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $EffectiveVersion) + { + Say "$assetName with version '$EffectiveVersion' is already installed." + Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot + return + } + } + } + catch + { + Say-Verbose "Failed to acquire download links from feed $feed. Exception: $_" + } + } +} + +if ($DownloadLinks.count -eq 0) { + throw "Failed to resolve the exact version number." +} + +if ($DryRun) { + PrintDryRunOutput $MyInvocation $DownloadLinks + return +} + +Measure-Action "Installation directory preparation" { Prepare-Install-Directory } + +Say-Verbose "Zip path: $ZipPath" + +$DownloadSucceeded = $false +$DownloadedLink = $null +$ErrorMessages = @() + +foreach ($link in $DownloadLinks) +{ + Say-Verbose "Downloading `"$($link.type)`" link $($link.downloadLink)" + + try { + Measure-Action "Package download" { DownloadFile -Source $link.downloadLink -OutPath $ZipPath } + Say-Verbose "Download succeeded." + $DownloadSucceeded = $true + $DownloadedLink = $link + break + } + catch { + $StatusCode = $null + $ErrorMessage = $null + + if ($PSItem.Exception.Data.Contains("StatusCode")) { + $StatusCode = $PSItem.Exception.Data["StatusCode"] + } + + if ($PSItem.Exception.Data.Contains("ErrorMessage")) { + $ErrorMessage = $PSItem.Exception.Data["ErrorMessage"] + } else { + $ErrorMessage = $PSItem.Exception.Message + } + + Say-Verbose "Download failed with status code $StatusCode. Error message: $ErrorMessage" + $ErrorMessages += "Downloading from `"$($link.type)`" link has failed with error:`nUri: $($link.downloadLink)`nStatusCode: $StatusCode`nError: $ErrorMessage" + } + + # This link failed. Clean up before trying the next one. + SafeRemoveFile -Path $ZipPath +} + +if (-not $DownloadSucceeded) { + foreach ($ErrorMessage in $ErrorMessages) { + Say-Error $ErrorMessages + } + + throw "Could not find `"$assetName`" with version = $($DownloadLinks[0].effectiveVersion)`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET support" +} + +Say "Extracting the archive." +Measure-Action "Package extraction" { Extract-Dotnet-Package -ZipPath $ZipPath -OutPath $InstallRoot } + +# Check if the SDK version is installed; if not, fail the installation. +$isAssetInstalled = $false + +# if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. +if ($DownloadedLink.effectiveVersion -Match "rtm" -or $DownloadedLink.effectiveVersion -Match "servicing") { + $ReleaseVersion = $DownloadedLink.effectiveVersion.Split("-")[0] + Say-Verbose "Checking installation: version = $ReleaseVersion" + $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $ReleaseVersion +} + +# Check if the SDK version is installed. +if (!$isAssetInstalled) { + Say-Verbose "Checking installation: version = $($DownloadedLink.effectiveVersion)" + $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $DownloadedLink.effectiveVersion +} + +# Version verification failed. More likely something is wrong either with the downloaded content or with the verification algorithm. +if (!$isAssetInstalled) { + Say-Error "Failed to verify the version of installed `"$assetName`".`nInstallation source: $($DownloadedLink.downloadLink).`nInstallation location: $InstallRoot.`nReport the bug at https://github.com/dotnet/install-scripts/issues." + throw "`"$assetName`" with version = $($DownloadedLink.effectiveVersion) failed to install with an unknown error." +} + +if (-not $KeepZip) { + SafeRemoveFile -Path $ZipPath +} + +Measure-Action "Setting up shell environment" { Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot } + +Say "Note that the script does not ensure your Windows version is supported during the installation." +Say "To check the list of supported versions, go to https://learn.microsoft.com/dotnet/core/install/windows#supported-versions" +Say "Installed version is $($DownloadedLink.effectiveVersion)" +Say "Installation finished" +# SIG # Begin signature block +# MIIoOAYJKoZIhvcNAQcCoIIoKTCCKCUCAQExDzANBglghkgBZQMEAgEFADB5Bgor +# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG +# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCDYphBPv3rmt1ZA +# JA1hMjk83/zFfoKJ/Mw+tp739UQWRKCCDYUwggYDMIID66ADAgECAhMzAAAEA73V +# lV0POxitAAAAAAQDMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD +# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy +# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p +# bmcgUENBIDIwMTEwHhcNMjQwOTEyMjAxMTEzWhcNMjUwOTExMjAxMTEzWjB0MQsw +# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u +# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy +# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +# AQCfdGddwIOnbRYUyg03O3iz19XXZPmuhEmW/5uyEN+8mgxl+HJGeLGBR8YButGV +# LVK38RxcVcPYyFGQXcKcxgih4w4y4zJi3GvawLYHlsNExQwz+v0jgY/aejBS2EJY +# oUhLVE+UzRihV8ooxoftsmKLb2xb7BoFS6UAo3Zz4afnOdqI7FGoi7g4vx/0MIdi +# kwTn5N56TdIv3mwfkZCFmrsKpN0zR8HD8WYsvH3xKkG7u/xdqmhPPqMmnI2jOFw/ +# /n2aL8W7i1Pasja8PnRXH/QaVH0M1nanL+LI9TsMb/enWfXOW65Gne5cqMN9Uofv +# ENtdwwEmJ3bZrcI9u4LZAkujAgMBAAGjggGCMIIBfjAfBgNVHSUEGDAWBgorBgEE +# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQU6m4qAkpz4641iK2irF8eWsSBcBkw +# VAYDVR0RBE0wS6RJMEcxLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh +# dGlvbnMgTGltaXRlZDEWMBQGA1UEBRMNMjMwMDEyKzUwMjkyNjAfBgNVHSMEGDAW +# gBRIbmTlUAXTgqoXNzcitW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8v +# d3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIw +# MTEtMDctMDguY3JsMGEGCCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDov +# L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDEx +# XzIwMTEtMDctMDguY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIB +# AFFo/6E4LX51IqFuoKvUsi80QytGI5ASQ9zsPpBa0z78hutiJd6w154JkcIx/f7r +# EBK4NhD4DIFNfRiVdI7EacEs7OAS6QHF7Nt+eFRNOTtgHb9PExRy4EI/jnMwzQJV +# NokTxu2WgHr/fBsWs6G9AcIgvHjWNN3qRSrhsgEdqHc0bRDUf8UILAdEZOMBvKLC +# rmf+kJPEvPldgK7hFO/L9kmcVe67BnKejDKO73Sa56AJOhM7CkeATrJFxO9GLXos +# oKvrwBvynxAg18W+pagTAkJefzneuWSmniTurPCUE2JnvW7DalvONDOtG01sIVAB +# +ahO2wcUPa2Zm9AiDVBWTMz9XUoKMcvngi2oqbsDLhbK+pYrRUgRpNt0y1sxZsXO +# raGRF8lM2cWvtEkV5UL+TQM1ppv5unDHkW8JS+QnfPbB8dZVRyRmMQ4aY/tx5x5+ +# sX6semJ//FbiclSMxSI+zINu1jYerdUwuCi+P6p7SmQmClhDM+6Q+btE2FtpsU0W +# +r6RdYFf/P+nK6j2otl9Nvr3tWLu+WXmz8MGM+18ynJ+lYbSmFWcAj7SYziAfT0s +# IwlQRFkyC71tsIZUhBHtxPliGUu362lIO0Lpe0DOrg8lspnEWOkHnCT5JEnWCbzu +# iVt8RX1IV07uIveNZuOBWLVCzWJjEGa+HhaEtavjy6i7MIIHejCCBWKgAwIBAgIK +# YQ6Q0gAAAAAAAzANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNV +# BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv +# c29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlm +# aWNhdGUgQXV0aG9yaXR5IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEw +# OTA5WjB+MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE +# BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYD +# VQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG +# 9w0BAQEFAAOCAg8AMIICCgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+la +# UKq4BjgaBEm6f8MMHt03a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc +# 6Whe0t+bU7IKLMOv2akrrnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4D +# dato88tt8zpcoRb0RrrgOGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+ +# lD3v++MrWhAfTVYoonpy4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nk +# kDstrjNYxbc+/jLTswM9sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6 +# A4aN91/w0FK/jJSHvMAhdCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmd +# X4jiJV3TIUs+UsS1Vz8kA/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL +# 5zmhD+kjSbwYuER8ReTBw3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zd +# sGbiwZeBe+3W7UvnSSmnEyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3 +# T8HhhUSJxAlMxdSlQy90lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS +# 4NaIjAsCAwEAAaOCAe0wggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRI +# bmTlUAXTgqoXNzcitW2oynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTAL +# BgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBD +# uRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jv +# c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf +# MDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3 +# dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf +# MDNfMjIuY3J0MIGfBgNVHSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEF +# BQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1h +# cnljcHMuaHRtMEAGCCsGAQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkA +# YwB5AF8AcwB0AGEAdABlAG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn +# 8oalmOBUeRou09h0ZyKbC5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7 +# v0epo/Np22O/IjWll11lhJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0b +# pdS1HXeUOeLpZMlEPXh6I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/ +# KmtYSWMfCWluWpiW5IP0wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvy +# CInWH8MyGOLwxS3OW560STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBp +# mLJZiWhub6e3dMNABQamASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJi +# hsMdYzaXht/a8/jyFqGaJ+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYb +# BL7fQccOKO7eZS/sl/ahXJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbS +# oqKfenoi+kiVH6v7RyOA9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sL +# gOppO6/8MO0ETI7f33VtY5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtX +# cVZOSEXAQsmbdlsKgEhr/Xmfwb1tbWrJUnMTDXpQzTGCGgkwghoFAgEBMIGVMH4x +# CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt +# b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01p +# Y3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTECEzMAAAQDvdWVXQ87GK0AAAAA +# BAMwDQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw +# HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIHDK +# fLl/hsfJ6CkgWVWjFR9NwXaKlucW5lfHmeLoy/TDMEIGCisGAQQBgjcCAQwxNDAy +# oBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5j +# b20wDQYJKoZIhvcNAQEBBQAEggEADW/XX0wys3VOeAJI08P0ak/niXL6mF0GKQwU +# vRg8CcSZppZav25fFzmT9aoKh27W2peO7JaAcL5vnUJ2rxF6zsP8FcKZggiTPRu3 +# 47+oidy80efzw8qe11tWOpQh4+eivCRfmPuIpDwXS5GvT58Bs2YkVX1D1YHohP8l +# PySRlOsbL0KJeJphjB0ovSGK8rCheQAR3vrDIXss9pARiC/aJenXth2fgdytDw1R +# OcpvG745BsJYSL5R1+bwUBQp4zEG10SAsb7asYP1J79bHgLuD4HRZuoD9Ds6d/WC +# CDgB4L1qhiFhot1HInI2V4ObsM/Ux2yuT6Wn6Kqqt6HBq2dHfKGCF5MwghePBgor +# BgEEAYI3AwMBMYIXfzCCF3sGCSqGSIb3DQEHAqCCF2wwghdoAgEDMQ8wDQYJYIZI +# AWUDBAIBBQAwggFSBgsqhkiG9w0BCRABBKCCAUEEggE9MIIBOQIBAQYKKwYBBAGE +# WQoDATAxMA0GCWCGSAFlAwQCAQUABCAVGL7p5kcAxTtMsksJdHdDlUTMizJlfC9Q +# 8d5i6XvZ1AIGZzXkkqjKGBMyMDI0MTExNDE1NTkxNS41NDJaMASAAgH0oIHRpIHO +# MIHLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH +# UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQL +# ExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxk +# IFRTUyBFU046REMwMC0wNUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1l +# LVN0YW1wIFNlcnZpY2WgghHpMIIHIDCCBQigAwIBAgITMwAAAehQsIDPK3KZTQAB +# AAAB6DANBgkqhkiG9w0BAQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2Fz +# aGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENv +# cnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAx +# MDAeFw0yMzEyMDYxODQ1MjJaFw0yNTAzMDUxODQ1MjJaMIHLMQswCQYDVQQGEwJV +# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE +# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1l +# cmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046REMwMC0w +# NUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2Uw +# ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDhQXdE0WzXG7wzeC9SGdH6 +# eVwdGlF6YgpU7weOFBkpW9yuEmJSDE1ADBx/0DTuRBaplSD8CR1QqyQmxRDD/Cdv +# DyeZFAcZ6l2+nlMssmZyC8TPt1GTWAUt3GXUU6g0F0tIrFNLgofCjOvm3G0j482V +# utKS4wZT6bNVnBVsChr2AjmVbGDN/6Qs/EqakL5cwpGel1te7UO13dUwaPjOy0Wi +# 1qYNmR8i7T1luj2JdFdfZhMPyqyq/NDnZuONSbj8FM5xKBoar12ragC8/1CXaL1O +# MXBwGaRoJTYtksi9njuq4wDkcAwitCZ5BtQ2NqPZ0lLiQB7O10Bm9zpHWn9x1/Hm +# dAn4koMWKUDwH5sd/zDu4vi887FWxm54kkWNvk8FeQ7ZZ0Q5gqGKW4g6revV2IdA +# xBobWdorqwvzqL70WdsgDU/P5c0L8vYIskUJZedCGHM2hHIsNRyw9EFoSolDM+yC +# edkz69787s8nIp55icLfDoKw5hak5G6MWF6d71tcNzV9+v9RQKMa6Uwfyquredd5 +# sqXWCXv++hek4A15WybIc6ufT0ilazKYZvDvoaswgjP0SeLW7mvmcw0FELzF1/uW +# aXElLHOXIlieKF2i/YzQ6U50K9dbhnMaDcJSsG0hXLRTy/LQbsOD0hw7FuK0nmzo +# tSx/5fo9g7fCzoFjk3tDEwIDAQABo4IBSTCCAUUwHQYDVR0OBBYEFPo5W8o980kM +# fRVQba6T34HwelLaMB8GA1UdIwQYMBaAFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMF8G +# A1UdHwRYMFYwVKBSoFCGTmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMv +# Y3JsL01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNybDBs +# BggrBgEFBQcBAQRgMF4wXAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0 +# LmNvbS9wa2lvcHMvY2VydHMvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUy +# MDIwMTAoMSkuY3J0MAwGA1UdEwEB/wQCMAAwFgYDVR0lAQH/BAwwCgYIKwYBBQUH +# AwgwDgYDVR0PAQH/BAQDAgeAMA0GCSqGSIb3DQEBCwUAA4ICAQCWfcJm2rwXtPi7 +# 4km6PKAkni9+BWotq+QtDGgeT5F3ro7PsIUNKRkUytuGqI8thL3Jcrb03x6DOppY +# JEA+pb6o2qPjFddO1TLqvSXrYm+OgCLL+7+3FmRmfkRu8rHvprab0O19wDbukgO8 +# I5Oi1RegMJl8t5k/UtE0Wb3zAlOHnCjLGSzP/Do3ptwhXokk02IvD7SZEBbPboGb +# tw4LCHsT2pFakpGOBh+ISUMXBf835CuVNfddwxmyGvNSzyEyEk5h1Vh7tpwP7z7r +# J+HsiP4sdqBjj6Avopuf4rxUAfrEbV6aj8twFs7WVHNiIgrHNna/55kyrAG9Yt19 +# CPvkUwxYK0uZvPl2WC39nfc0jOTjivC7s/IUozE4tfy3JNkyQ1cNtvZftiX3j5Dt +# +eLOeuGDjvhJvYMIEkpkV68XLNH7+ZBfYa+PmfRYaoFFHCJKEoRSZ3PbDJPBiEhZ +# 9yuxMddoMMQ19Tkyftot6Ez0XhSmwjYBq39DvBFWhlyDGBhrU3GteDWiVd9YGSB2 +# WnxuFMy5fbAK6o8PWz8QRMiptXHK3HDBr2wWWEcrrgcTuHZIJTqepNoYlx9VRFvj +# /vCXaAFcmkW1nk7VE+owaXr5RJjryDq9ubkyDq1mdrF/geaRALXcNZbfNXIkhXzX +# A6a8CiamcQW/DgmLJpiVQNriZYCHIDCCB3EwggVZoAMCAQICEzMAAAAVxedrngKb +# SZkAAAAAABUwDQYJKoZIhvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +# EwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3Nv +# ZnQgQ29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmlj +# YXRlIEF1dGhvcml0eSAyMDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIy +# NVowfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcT +# B1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UE +# AxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEB +# AQUAA4ICDwAwggIKAoICAQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXI +# yjVX9gF/bErg4r25PhdgM/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjo +# YH1qUoNEt6aORmsHFPPFdvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1y +# aa8dq6z2Nr41JmTamDu6GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v +# 3byNpOORj7I5LFGc6XBpDco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pG +# ve2krnopN6zL64NF50ZuyjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viS +# kR4dPf0gz3N9QZpGdc3EXzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYr +# bqgSUei/BQOj0XOmTTd0lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlM +# jgK8QmguEOqEUUbi0b1qGFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSL +# W6CmgyFdXzB0kZSU2LlQ+QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AF +# emzFER1y7435UsSFF5PAPBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIu +# rQIDAQABo4IB3TCCAdkwEgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIE +# FgQUKqdS/mTEmr6CkTxGNSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWn +# G1M1GelyMFwGA1UdIARVMFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEW +# M2h0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5 +# Lmh0bTATBgNVHSUEDDAKBggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBi +# AEMAQTALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV +# 9lbLj+iiXGJo0T2UkFvXzpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3Js +# Lm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAx +# MC0wNi0yMy5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8v +# d3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2 +# LTIzLmNydDANBgkqhkiG9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv +# 6lwUtj5OR2R4sQaTlz0xM7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZn +# OlNN3Zi6th542DYunKmCVgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1 +# bSNU5HhTdSRXud2f8449xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4 +# rPf5KYnDvBewVIVCs/wMnosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU +# 6ZGyqVvfSaN0DLzskYDSPeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDF +# NLB62FD+CljdQDzHVG2dY3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/ +# HltEAY5aGZFrDZ+kKNxnGSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdU +# CbFpAUR+fKFhbHP+CrvsQWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKi +# excdFYmNcP7ntdAoGokLjzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTm +# dHRbatGePu1+oDEzfbzL6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZq +# ELQdVTNYs6FwZvKhggNMMIICNAIBATCB+aGB0aSBzjCByzELMAkGA1UEBhMCVVMx +# EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT +# FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJp +# Y2EgT3BlcmF0aW9uczEnMCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOkRDMDAtMDVF +# MC1EOTQ3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMK +# AQEwBwYFKw4DAhoDFQCMJG4vg0juMOVn2BuKACUvP80FuqCBgzCBgKR+MHwxCzAJ +# BgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25k +# MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jv +# c29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMA0GCSqGSIb3DQEBCwUAAgUA6uBjDTAi +# GA8yMDI0MTExNDExNTI0NVoYDzIwMjQxMTE1MTE1MjQ1WjBzMDkGCisGAQQBhFkK +# BAExKzApMAoCBQDq4GMNAgEAMAYCAQACASswBwIBAAICEtwwCgIFAOrhtI0CAQAw +# NgYKKwYBBAGEWQoEAjEoMCYwDAYKKwYBBAGEWQoDAqAKMAgCAQACAwehIKEKMAgC +# AQACAwGGoDANBgkqhkiG9w0BAQsFAAOCAQEAO9dAHJydwpQmKP7RB58A6uytAueA +# pOj5XLZAqbJ+hMZaZE44u9Wyxa1HZarThqCCoaa0LcMriFy/FyQjAihnao/isbQT +# VeqCr4r4a4C2qRXIU5McachjMYuXhmkUJzgxRHnDekKT7X5ONkM3dd03SyO3gIdf +# r7Mx1AcLvMAjzX3Cy6R70FYvPVHuUFwa80pIiovp80keB79qdg4kLc8w5JSIxpZ7 +# jnHwrHhoHlrJTqeTmNmlNL785N/R219KrFSBT+uufpMveyuX7HjCl13jUP2WygeC +# GacWvUnTdKkuGGkW2hFVqIctbM3VW6mA7TnLnud+3D29vYssZsBESMNNAjGCBA0w +# ggQJAgEBMIGTMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw +# DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x +# JjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB6FCw +# gM8rcplNAAEAAAHoMA0GCWCGSAFlAwQCAQUAoIIBSjAaBgkqhkiG9w0BCQMxDQYL +# KoZIhvcNAQkQAQQwLwYJKoZIhvcNAQkEMSIEICbqWDk+cKHhkAUoFYumceeBi9ES +# prHHdXqaqtRXWdzQMIH6BgsqhkiG9w0BCRACLzGB6jCB5zCB5DCBvQQgKtLaxNUC +# hCCCQdHn2k2qKB7TF8lPYndTxbVJzwf46x0wgZgwgYCkfjB8MQswCQYDVQQGEwJV +# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE +# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGlt +# ZS1TdGFtcCBQQ0EgMjAxMAITMwAAAehQsIDPK3KZTQABAAAB6DAiBCD1VQZ0eCNS +# U4KPXZ2JQcqYMihqMI5Eayx2mSwN1h+g2TANBgkqhkiG9w0BAQsFAASCAgBqjNdR +# RWtrY193UweKiix14+DiDTXShxiesVyuXIA1l9V4L5YyKarJUE2jFYYDz4ehi66X +# dmcVUXyUsXadrCg4GbG4xtkgIc5f0usfa0pr4j5t8jdMZw02GjGd4ypfRNiGFolr +# 4BgsAkBTm7jDj49OCmI/LJTLkI94i83gsph9XfkQRSARcPIzkrEJZcJvtw1qEaqx +# QREf6uYUa7DgMncNeXkk+DtNBM0fKQmBcwxta1/xzCrQFJTvacUPo0rhkJnTUwdz +# 6rZowgh2fnMuWZxB9PXegQiY+NmXDniiNRa/n7pLJUBy6+M0FS10QRu2JA80M6/c +# IPlpXugV5IxjMo6vJoc0GuPXRIXe41m/8pofDuPHOtTFpKLQ5TJ/y2YPCxR3mXnz +# iUs1/pmqA1QWKJRG7b6drw1+lpLOzKyPIFA8xxurofMikP8vAgqaQV1GfYZeaHxG +# A6cxiZyG73XrqezOx7sNspUaj5CLE0krZYVj6vmWul0BDIZDY20ZXy2c2lmqjuU+ +# nhgc6kDz7g5DwkRJwxFDPS7XRVlA9WybKecpKQ5ftnk2UPFTPfFn8O2kyj12apbA +# CvNdDlLNMPmeNKGFaUA0Mov4JlPThpWAu/A8EZQywHlfW32RGQBHj39BpEl8K5pE +# ZGQYPnWZbUqFBrODhY7PnRKAfZ+HDZF2AugcNQ== +# SIG # End signature block