- added unit test project for m3u lists

- fixed handling of reference lists (matching sat/cable/antenna, digital/analog, tv/radio/data criteria), especially for m3u lists which don't contain ONID-TSID-SID to identify channels

source/ChanSort.Api/Controller/Editor.cs

@@ -233,6 +233,8 @@ public void ApplyReferenceList(DataRoot refDataRoot, ChannelList refList, Channe
       foreach (var channel in tvList.Channels)
       {
         var key = DvbKey(channel.OriginalNetworkId, channel.TransportStreamId, channel.ServiceId);
+        if (key == 0)
+          continue;
         var list = onidTsidSid.TryGet(key);
         if (list == null)
         {
@@ -248,14 +250,13 @@ public void ApplyReferenceList(DataRoot refDataRoot, ChannelList refList, Channe
         if (!(chanFilter?.Invoke(refChannel, true) ?? true))
           continue;
 
-        var tvChannels = tvList.GetChannelByUid(refChannel.Uid);
+        var tvChannels = refChannel.Uid == "0-0-0" ? null : tvList.GetChannelByUid(refChannel.Uid);
 
         // try to find matching channels based on ONID+TSID+SID
-        if (tvChannels.Count == 0)
+        if ((tvChannels?.Count ?? 0) == 0)
         {
           var key = DvbKey(refChannel.OriginalNetworkId, refChannel.TransportStreamId, refChannel.ServiceId);
-          List<ChannelInfo> candidates;
-          if (key != 0 && onidTsidSid.TryGetValue(key, out candidates))
+          if (key != 0 && onidTsidSid.TryGetValue(key, out var candidates))
           {
             tvChannels = candidates;
 
@@ -270,7 +271,7 @@ public void ApplyReferenceList(DataRoot refDataRoot, ChannelList refList, Channe
         }
 
         // try to find matching channels by name
-        if (tvChannels.Count == 0 && !string.IsNullOrWhiteSpace(refChannel.Name))
+        if ((tvChannels?.Count ?? 0) == 0 && !string.IsNullOrWhiteSpace(refChannel.Name))
           tvChannels = tvList.GetChannelByName(refChannel.Name).ToList();
 
         // get the first unassigned channel from the candidates (e.g. when matching by non-unique names), or fall back to the first matching channel (e.g. by unique ID)

source/ChanSort.Loader.M3u/Serializer.cs

@@ -9,8 +9,8 @@
 namespace ChanSort.Loader.M3u
 {
   /*
-   * This serializer reads .m3u files that are used for SAT>IP lists. Some hardware SAT>IP servers use this format as well as VNC.
-   * There is no official standard for .m3u and files may have a UTF-8 BOM or not, may be encoded in UTF-8 or a locale specific encoding and my have different new-line sequences.
+   * This serializer reads .m3u files that are used for SAT>IP lists. Some hardware SAT>IP servers use this format as well as VLC.
+   * There is no official standard for .m3u and files may have a UTF-8 BOM or not, may be encoded in UTF-8 or a locale specific and my have different new-line sequences.
    * This loader attempts to maintain the original file as much as possible, including comment lines that are not directly understood by ChanSort.
    */
   class Serializer : SerializerBase
@@ -39,7 +39,7 @@ public Serializer(string inputFile) : base(inputFile)
       base.DefaultEncoding = new UTF8Encoding(false);
       this.allChannels.VisibleColumnFieldNames = new List<string>()
       {
-        "OldPosition", "Position", "Name", "FreqInMhz", "Polarity", "SymbolRate", "VideoPid", "AudioPid", "Satellite", "Provider"
+        "+OldPosition", "+Position", "+Name", "+FreqInMhz", "+Polarity", "+SymbolRate", "+VideoPid", "+AudioPid", "+Satellite", "+Provider"
       };
     }
     #endregion
@@ -56,7 +56,7 @@ public override void Load()
 
       // detect line separator
       int idx = Array.IndexOf(content, '\n');
-      this.newLine = idx >= 1 && content[idx] - 1 == '\r' ? "\r\n" : "\n";
+      this.newLine = idx >= 1 && content[idx-1] == '\r' ? "\r\n" : "\n";
 
       var rdr = new StreamReader(new MemoryStream(content), overrideEncoding ?? this.DefaultEncoding);
       string line = rdr.ReadLine()?.TrimEnd();
@@ -143,6 +143,7 @@ private void ReadChannel(int uriLineNr, string uriLine, string extInfLine, strin
         progNr = this.allChannels.Count + 1;
 
       var chan = new Channel(uriLineNr, progNr, name, allLines);
+      chan.Uid = uriLine;
       chan.ExtInfTrackNameIndex = extInfTrackNameIndex;
       chan.Provider = group;
 

source/ChanSort.sln

@@ -67,6 +67,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spike.LgWebOs5", "Spike.LgW
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ChanSort.Loader.Grundig", "ChanSort.Loader.Grundig\ChanSort.Loader.Grundig.csproj", "{4D5AF0A3-1B96-42C8-910D-0C4852EA22F4}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Test.Loader.M3u", "Test.Loader.M3u\Test.Loader.M3u.csproj", "{052692BF-D782-4888-B34D-89D6B1379340}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -365,6 +367,18 @@ Global
 		{4D5AF0A3-1B96-42C8-910D-0C4852EA22F4}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{4D5AF0A3-1B96-42C8-910D-0C4852EA22F4}.Release|x86.ActiveCfg = Release|Any CPU
 		{4D5AF0A3-1B96-42C8-910D-0C4852EA22F4}.Release|x86.Build.0 = Release|Any CPU
+		{052692BF-D782-4888-B34D-89D6B1379340}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{052692BF-D782-4888-B34D-89D6B1379340}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{052692BF-D782-4888-B34D-89D6B1379340}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{052692BF-D782-4888-B34D-89D6B1379340}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{052692BF-D782-4888-B34D-89D6B1379340}.Debug|x86.ActiveCfg = Debug|x86
+		{052692BF-D782-4888-B34D-89D6B1379340}.Debug|x86.Build.0 = Debug|x86
+		{052692BF-D782-4888-B34D-89D6B1379340}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{052692BF-D782-4888-B34D-89D6B1379340}.Release|Any CPU.Build.0 = Release|Any CPU
+		{052692BF-D782-4888-B34D-89D6B1379340}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{052692BF-D782-4888-B34D-89D6B1379340}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{052692BF-D782-4888-B34D-89D6B1379340}.Release|x86.ActiveCfg = Release|Any CPU
+		{052692BF-D782-4888-B34D-89D6B1379340}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

source/ChanSort/ReferenceListForm.cs

@@ -207,14 +207,18 @@ private bool FilterChannel(ChannelInfo ch, bool source)
       var ss = ch.SignalSource;
       if (source)
       {
-        if (
-          !(this.cbAntenna.Checked && (ss & SignalSource.Antenna) != 0 || this.cbCable.Checked && (ss & SignalSource.Cable) != 0 || this.cbSat.Checked && (ss & SignalSource.Sat) != 0 ||
+        if ((ss & SignalSource.MaskAntennaCableSat) != 0 && 
+            !(this.cbAntenna.Checked && (ss & SignalSource.Antenna) != 0 || this.cbCable.Checked && (ss & SignalSource.Cable) != 0 || this.cbSat.Checked && (ss & SignalSource.Sat) != 0 ||
             this.cbIp.Checked && (ss & SignalSource.IP) != 0))
           return false;
       }
-      if (!(this.cbAnalog.Checked && (ss & SignalSource.Analog) != 0 || this.cbDigital.Checked && (ss & SignalSource.Digital) != 0))
+
+      if ((ss & SignalSource.MaskAnalogDigital) != 0 &&
+        !(this.cbAnalog.Checked && (ss & SignalSource.Analog) != 0 || this.cbDigital.Checked && (ss & SignalSource.Digital) != 0))
         return false;
-      if (!(this.cbTv.Checked && (ss & SignalSource.Tv) != 0 || this.cbRadio.Checked && (ss & SignalSource.Radio) != 0 || this.cbData.Checked && (ss & SignalSource.Data) != 0))
+
+      if ((ss & SignalSource.MaskTvRadioData) != 0 &&
+        !(this.cbTv.Checked && (ss & SignalSource.Tv) != 0 || this.cbRadio.Checked && (ss & SignalSource.Radio) != 0 || this.cbData.Checked && (ss & SignalSource.Data) != 0))
         return false;
       return true;
     }

source/Test.Loader.M3u/M3uTest.cs

@@ -0,0 +1,48 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Linq;
+using ChanSort.Api;
+
+namespace Test.Loader.M3u
+{
+  [TestClass]
+  public class M3uTest
+  {
+    [TestMethod]
+    public void TestMethod1()
+    {
+      var m3uFile = TestUtils.DeploymentItem("Test.Loader.M3u\\TestFiles\\example.m3u");
+      var refFile = TestUtils.DeploymentItem("Test.Loader.M3u\\TestFiles\\example-ref.txt");
+
+      var loader = new ChanSort.Loader.M3u.M3uPlugin();
+      var ser = loader.CreateSerializer(m3uFile);
+      ser.Load();
+      Assert.IsNotNull(ser);
+      var root = ser.DataRoot;
+      Assert.IsNotNull(root);
+
+      root.ApplyCurrentProgramNumbers();
+      var lists = root.ChannelLists.ToList();
+      Assert.AreEqual(1, lists.Count);
+      var chans = lists[0].Channels;
+      Assert.AreEqual(6, chans.Count);
+      Assert.AreEqual("Russia Today", chans[0].Name);
+      Assert.AreEqual(1, chans[0].NewProgramNr);
+      Assert.AreEqual("MP4", chans[5].Name);
+      Assert.AreEqual(6, chans[5].NewProgramNr);
+
+
+      var refLoader = new RefSerializerPlugin();
+      var refSer = refLoader.CreateSerializer(refFile);
+      refSer.Load();
+      var ed = new Editor();
+      ed.DataRoot = ser.DataRoot;
+      ed.ChannelList = lists[0];
+      //ed.ApplyReferenceList(refSer.DataRoot);
+      ed.ApplyReferenceList(refSer.DataRoot, refSer.DataRoot.ChannelLists.First(), lists[0], false, 0, null, true, false);
+
+      Assert.AreEqual(1, chans[5].NewProgramNr);
+      Assert.AreEqual(2, chans[4].NewProgramNr);
+    }
+  }
+}

source/Test.Loader.M3u/Properties/AssemblyInfo.cs

@@ -0,0 +1,20 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+[assembly: AssemblyTitle("Test.Loader.M3u")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("Test.Loader.M3u")]
+[assembly: AssemblyCopyright("Copyright ©  2021")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+[assembly: ComVisible(false)]
+
+[assembly: Guid("052692bf-d782-4888-b34d-89d6b1379340")]
+
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]

source/Test.Loader.M3u/Test.Loader.M3u.csproj

@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\packages\MSTest.TestAdapter.2.1.1\build\net45\MSTest.TestAdapter.props" Condition="Exists('..\packages\MSTest.TestAdapter.2.1.1\build\net45\MSTest.TestAdapter.props')" />
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{052692BF-D782-4888-B34D-89D6B1379340}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>Test.Loader.M3u</RootNamespace>
+    <AssemblyName>Test.Loader.M3u</AssemblyName>
+    <TargetFrameworkVersion>v4.8</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <ProjectTypeGuids>{3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">15.0</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+    <ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages</ReferencePath>
+    <IsCodedUITest>False</IsCodedUITest>
+    <TestProjectType>UnitTest</TestProjectType>
+    <NuGetPackageImportStamp>
+    </NuGetPackageImportStamp>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
+    <DebugSymbols>true</DebugSymbols>
+    <OutputPath>bin\x86\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <DebugType>full</DebugType>
+    <PlatformTarget>x86</PlatformTarget>
+    <LangVersion>7.3</LangVersion>
+    <ErrorReport>prompt</ErrorReport>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
+    <OutputPath>bin\x86\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <Optimize>true</Optimize>
+    <DebugType>pdbonly</DebugType>
+    <PlatformTarget>x86</PlatformTarget>
+    <LangVersion>7.3</LangVersion>
+    <ErrorReport>prompt</ErrorReport>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="Microsoft.VisualStudio.TestPlatform.TestFramework, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\MSTest.TestFramework.2.1.1\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\MSTest.TestFramework.2.1.1\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="M3uTest.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="packages.config" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\ChanSort.Api\ChanSort.Api.csproj">
+      <Project>{dccffa08-472b-4d17-bb90-8f513fc01392}</Project>
+      <Name>ChanSort.Api</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\ChanSort.Loader.M3u\ChanSort.Loader.M3u.csproj">
+      <Project>{484028b6-3aae-4f7e-a88a-76beeb70203b}</Project>
+      <Name>ChanSort.Loader.M3u</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\Test.Loader\Test.Loader.csproj">
+      <Project>{68cfcb2f-b52a-43a1-aa5c-5d64a1d655d2}</Project>
+      <Name>Test.Loader</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup />
+  <ItemGroup>
+    <None Include="TestFiles\example-ref.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Include="TestFiles\example.m3u">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+  <Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" />
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\packages\MSTest.TestAdapter.2.1.1\build\net45\MSTest.TestAdapter.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\MSTest.TestAdapter.2.1.1\build\net45\MSTest.TestAdapter.props'))" />
+    <Error Condition="!Exists('..\packages\MSTest.TestAdapter.2.1.1\build\net45\MSTest.TestAdapter.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\MSTest.TestAdapter.2.1.1\build\net45\MSTest.TestAdapter.targets'))" />
+  </Target>
+  <Import Project="..\packages\MSTest.TestAdapter.2.1.1\build\net45\MSTest.TestAdapter.targets" Condition="Exists('..\packages\MSTest.TestAdapter.2.1.1\build\net45\MSTest.TestAdapter.targets')" />
+</Project>

source/Test.Loader.M3u/TestFiles/example-ref.txt

@@ -0,0 +1,6 @@
+1;MP4;0-0-0
+2;MKV;0-0-0
+3;Russia Today;0-0-0
+4;RBK;0-0-0
+5;Brodilo TV;0-0-0
+6;Youtube Live;0-0-0

source/Test.Loader.M3u/TestFiles/example.m3u

@@ -0,0 +1,14 @@
+#EXTM3U
+#EXTINF:0 group-title="Streams",Russia Today
+https://rt-news-gd.secure2.footprint.net/1103.m3u8
+#EXTINF:0,RBK
+http://e3.online.video.rbc.ru/online2/rbctv_576p/index.m3u8
+#EXTINF:0 tvg-id="brodilo",Brodilo TV
+http://brodilo.tv/channel.php
+#EXTINF:0,Youtube Live
+https://youtu.be/9Auq9mYxFEE,live
+
+#EXTINF:0,MKV
+http://jell.yfish.us/media/jellyfish-3-mbps-hd-h264.mkv
+#EXTINF:0,MP4
+http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4

source/Test.Loader.M3u/packages.config

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="MSTest.TestAdapter" version="2.1.1" targetFramework="net48" />
+  <package id="MSTest.TestFramework" version="2.1.1" targetFramework="net48" />
+</packages>

source/Test.Loader.Samsung/Zip/SamsungZipTest.cs

@@ -36,7 +36,7 @@ public void TestAntennaChannelsAddedToCorrectLists()
     #region TestChannelsAddedToCorrectList
     private void TestChannelsAddedToCorrectLists(string fileName, SignalSource signalSource, int expectedTotal, int expectedTv, int expectedRadio, int dataProgramSid = 0, string dataProgramName = null)
     {
-      var tempFile = TestUtils.DeploymentItem("Test.Loader.SamsungJ\\TestFiles\\" + fileName);
+      var tempFile = TestUtils.DeploymentItem("Test.Loader.Samsung\\Zip\\TestFiles\\" + fileName);
       var plugin = new SamsungPlugin();
       var ser = plugin.CreateSerializer(tempFile);
       ser.Load();
@@ -64,7 +64,7 @@ private void TestChannelsAddedToCorrectLists(string fileName, SignalSource signa
     [TestMethod]
     public void TestDeletingChannel()
     {
-      var tempFile = TestUtils.DeploymentItem("Test.Loader.SamsungJ\\TestFiles\\Channel_list_T-KTSUDEUC-1007.3.zip");
+      var tempFile = TestUtils.DeploymentItem("Test.Loader.Samsung\\Zip\\TestFiles\\Channel_list_T-KTSUDEUC-1007.3.zip");
       var plugin = new SamsungPlugin();
       var ser = plugin.CreateSerializer(tempFile);
       ser.Load();

source/changelog.md

@@ -1,6 +1,9 @@
 ChanSort Change Log 
 ===================
 
+2021-01-24
+- fixed issues with applying reference lists (especially .m3u)
+
 2021-01-23
 - Toshiba: added support for settingsDB.db lists
 - SatcoDX (*.sdx format used by ITT, Telefunken, Silva-Schneider, ...): minor bug fixes