- SatcoDX (.sdx): fixed handling of format version 105, which contain…

…s trailing data after the last channel

- SatcoDX: changing the character set in the menu now instantly corrects channel names with non-ASCII characters

source/ChanSort.Api/Utils/DvbStringDecoder.cs

@@ -156,12 +156,12 @@ private void GetChannelNamesCore(byte[] name, int off, int len, out string longN
         }
         if (ch == '\0')
         {
-          // read as many bytes as necessary to get a character
+          // read as many bytes as necessary to get a character. Note that the decoder keeps internal state of all previously unprocessed bytes
           char[] charArray = new char[5];
-          int byteCnt;
-          for (byteCnt = 1; decoder.GetChars(name, i, byteCnt, charArray, 0) == 0; byteCnt++)
-          {
-          }
+          while (c < len && decoder.GetChars(name, i++, 1, charArray, 0) == 0)
+            c++;
+          if (c >= len)
+            break;
           ch = charArray[0];
         }
         if (ch == '\0')

source/ChanSort.Loader.SatcoDX/Channel.cs

@@ -1,19 +1,22 @@
 using System;
 using System.Globalization;
 using System.IO;
+using System.Text;
 using ChanSort.Api;
 
 namespace ChanSort.Loader.SatcoDX
 {
   internal class Channel : ChannelInfo
   {
+    private byte[] data;
     public int FileOffset { get; }
     public int Length { get; }
 
     #region ctor()
 
     internal Channel(int pos, string line, byte[] data, int start, int length, DvbStringDecoder decoder)
     {
+      this.data = data;
       this.FileOffset = start;
       this.Length = length;
       this.RecordIndex = pos;
@@ -41,14 +44,8 @@ internal Channel(int pos, string line, byte[] data, int start, int length, DvbSt
       // 42: polarity
       this.Polarity = line[42] == '1' ? 'H' : 'V';
 
-      // 43-50 + (115-126 or 115-131): channel name
-      byte[] nameBytes = new byte[8+17];
-      var nameLen2 = Math.Min(length - 115, 17); // version 103 has 12 extra bytes for channel name, version 105 has 17
-      Array.Copy(data, start + 43, nameBytes, 0, 8);
-      Array.Copy(data, start + 115, nameBytes, 8, nameLen2);
-      decoder.GetChannelNames(nameBytes,0, nameBytes.Length, out var longName, out var shortName);
-      this.Name = longName.TrimEnd();
-      this.ShortName = shortName.TrimEnd();
+      // 43-50 + 115-126 in version 103 or 115-131 in version 105: channel name
+      this.ParseName(decoder);
 
       // 51-54: sat position
       var spos = line.Substring(51, 4).TrimStart('0');
@@ -83,5 +80,33 @@ internal Channel(int pos, string line, byte[] data, int start, int length, DvbSt
     }
 
     #endregion
+
+    #region ParseName()
+
+    /// <summary>
+    /// SATCODX103 files can contain channel names with unspecified implicit encoding, so we support reparsing based on a user selected default code page
+    /// </summary>
+    /// <param name="decoder"></param>
+    public void ParseName(DvbStringDecoder decoder)
+    {
+      var length = this.Length;
+      var start = this.FileOffset;
+
+      // 43-50 + 115-126 in version 103 or 115-131 in version 105: channel name
+      byte[] nameBytes = new byte[8 + 17];
+      var nameLen2 = Math.Min(length - 115, 17); // version 103 has 12 extra bytes for channel name, version 105 has 17
+      Array.Copy(data, start + 43, nameBytes, 0, 8);
+      Array.Copy(data, start + 115, nameBytes, 8, nameLen2);
+
+      // I have seen format 103 files using only implicit CP1252 encoding for Umlauts, as well as format 105 with implicit UTF-8/explicit DVB-encoding
+      var oldDefaultEncoding = decoder.DefaultEncoding;
+      if (nameLen2 > 12)
+        decoder.DefaultEncoding = Encoding.UTF8;
+      decoder.GetChannelNames(nameBytes, 0, nameBytes.Length, out var longName, out var shortName);
+      decoder.DefaultEncoding = oldDefaultEncoding;
+      this.Name = longName.TrimEnd();
+      this.ShortName = shortName.TrimEnd();
+    }
+    #endregion
   }
 }

source/ChanSort.Loader.SatcoDX/Serializer.cs

@@ -10,6 +10,7 @@ internal class Serializer : SerializerBase
     private readonly ChannelList allChannels = new ChannelList(0, "All");
 
     private byte[] content;
+    private int trailingDataPos;
 
     #region ctor()
 
@@ -51,7 +52,7 @@ public override void Load()
       var pos = 0;
       content = File.ReadAllBytes(this.FileName);
       int prevPos = 0, nextPos;
-      while (prevPos < content.Length && (nextPos = Array.FindIndex(content, prevPos, ch => ch == (byte)'\n')) >= 0)
+      while (prevPos < content.Length && content[prevPos] != 0 && (nextPos = Array.FindIndex(content, prevPos, ch => ch == (byte)'\n')) >= 0)
       {
         if (nextPos - prevPos == 0)
           continue;
@@ -61,6 +62,9 @@ public override void Load()
         pos++;
         prevPos = nextPos + 1;
       }
+
+      // SATCODX105 files contain a \0 character to mark the end, followed by an arbitrary number or spaces (or whatever data). We'll preserve it as-is.
+      this.trailingDataPos = prevPos;
     }
 
     #endregion
@@ -85,11 +89,37 @@ public override void Save(string tvOutputFile)
           if (channel is Channel realChannel)
             file.Write(this.content, realChannel.FileOffset, realChannel.Length + 1);
         }
+
+        file.Write(this.content, this.trailingDataPos, this.content.Length - this.trailingDataPos);
       }
     }
 
     #endregion
 
+
+    #region DefaultEncoding
+
+    /// SATCODX103 files can contain channel names with unspecified implicit encoding, so we support reparsing based on a user selected default code page
+
+    public override Encoding DefaultEncoding
+    {
+      get => base.DefaultEncoding;
+      set
+      {
+        if (value == this.DefaultEncoding)
+          return;
+        base.DefaultEncoding = value;
+
+        var decoder = new DvbStringDecoder(value);
+        foreach (var chan in this.allChannels.Channels)
+        {
+          if (chan is Channel ch)
+            ch.ParseName(decoder);
+        }
+      }
+    }
+    #endregion
+
     #region GetFileInformation()
 
     public override string GetFileInformation()

source/Test.Loader.SatcoDX/SdxTest.cs

@@ -12,13 +12,27 @@ public class SdxTest
     [TestMethod]
     public void TestSatChannelsAddedToCorrectLists()
     {
-      this.TestChannelsAddedToCorrectLists("silva_schneider.sdx", SignalSource.DvbS, 1108, 948, 160);
+      var list = this.TestChannelsAddedToCorrectLists("silva_schneider.sdx", SignalSource.DvbS, 1108, 948, 160);
+
+      // Test encoding as CP1252
+      Assert.AreEqual("SAT.1 Gold Österreic", list.Channels[9].Name);
+    }
+    #endregion
+
+    #region TestFormatVersion105UtfEncoding
+    [TestMethod]
+    public void TestFormatVersion105UtfEncoding()
+    {
+      var list = this.TestChannelsAddedToCorrectLists("telefunken105.sdx", SignalSource.DvbS, 737, 650, 87);
+
+      // Test encoding as UTF-8
+      Assert.AreEqual("WDR HD Köln", list.Channels[5].Name);
     }
     #endregion
 
 
     #region TestChannelsAddedToCorrectList
-    private void TestChannelsAddedToCorrectLists(string fileName, SignalSource signalSource, int expectedTotal, int expectedTv, int expectedRadio, int dataProgramSid = 0, string dataProgramName = null)
+    private ChannelList TestChannelsAddedToCorrectLists(string fileName, SignalSource signalSource, int expectedTotal, int expectedTv, int expectedRadio, int dataProgramSid = 0, string dataProgramName = null)
     {
       var tempFile = TestUtils.DeploymentItem("Test.Loader.SatcoDX\\TestFiles\\" + fileName);
       var plugin = new SatcoDxPlugin();
@@ -40,6 +54,8 @@ private void TestChannelsAddedToCorrectLists(string fileName, SignalSource signa
         Assert.IsNotNull(chan);
         Assert.AreEqual(dataProgramName, chan.Name);
       }
+
+      return list;
     }
     #endregion
 

source/Test.Loader.SatcoDX/Test.Loader.SatcoDX.csproj

@@ -223,6 +223,7 @@
     <None Include="app.config" />
     <None Include="packages.config" />
     <None Include="TestFiles\silva_schneider.sdx" />
+    <None Include="TestFiles\telefunken105.sdx" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\ChanSort.Api\ChanSort.Api.csproj">