Complete XMLTV-revamp to avoid memory leak on DS

Several fixes

js/tvstreamrecord.basic.js

@@ -830,11 +830,13 @@ $(function() {
                 $(this).css("margin-top", cnt==0?10:(80+(cnt-1)*100) + 'px');
                 $(this).css("margin-left", x+'%');
                 $(this).css("width", w+'%');
+//                if (cnt==0) $(this).css("position", 'fixed');
             } else {
                 $(this).css("left", cnt==0?0:((cnt-1) * 250 + 60) + 'px');
                 $(this).css("top", x+'%');
                 $(this).css("width", cnt==0?'50px':'240px');
                 $(this).css("height", w + "%");
+//                if (cnt==0) $(this).css("position", 'fixed');
             }
         });
 

sql.py

@@ -23,7 +23,7 @@
 def sqlRun(sql, t=-1, many=0):
     fa = []
     try:
-        conn = sqlite3.connect('settings.db')
+        conn = sqlite3.connect('settings.db', timeout=20)
         c = conn.cursor()
         c.execute('PRAGMA journal_mode = OFF;')
         conn.text_factory = sqlite3.OptimizedUnicode
@@ -42,8 +42,7 @@ def sqlRun(sql, t=-1, many=0):
         conn.commit()
         conn.close()
     except Exception as ex:
-        print ("SQL Exception: %s" % sql)
-#        print (ex)
+        print ("SQL Exception '%s' with '%s'" % (ex,sql))
         pass
     return fa
 

synology/INFO

@@ -1,5 +1,5 @@
 package="tvstreamrecord"
-version="0.6.4a"
+version="0.6.4b"
 maintainer="Pavion"
 description="Program to record TV streams. Python 2.x required. Server runs at port 8030 (changeable)"
 reloadui="yes"

tvstreamrecord.py

@@ -40,6 +40,7 @@
 import sys
 from mylogging import logInit, logRenew, logStop
 import hashlib
+import codecs
 
 def total(timedelta):
     try: 
@@ -52,7 +53,7 @@ def total(timedelta):
 localtime = "%H:%M"
 localdate = "%d.%m.%Y"
 dayshown = datetime.combine(date.today(), time.min)
-version = '0.6.4a'
+version = '0.6.4b'
 
 @route('/live/<filename>')
 def server_static9(filename):
@@ -64,13 +65,12 @@ def server_static9(filename):
     return static_file("/live.m3u", root='', mimetype='video')
 
 def write_m3u(name, path):
-    f = open("live.m3u", "w")
+    f = codecs.open("live.m3u", "w", "utf-8")
     f.write("#EXTM3U\n")
     f.write("#EXTINF:0,"+name+"\n")
     f.write(path+"\n")
     f.close()
 
-
 @route('/channels.m3u')
 def server_static8():
     return static_file("/channels.m3u", root='')
@@ -276,7 +276,7 @@ def list_p():
 def clgen_p():
     rows = sqlRun("select cid, cname, cpath from channels where cenabled=1 ORDER BY cid")
     if rows:
-        f = open("channels.m3u", "w")
+        f = codecs.open("channels.m3u", "w", "utf-8")
         f.write("#EXTM3U\n")
         for row in rows:
             f.write("#EXTINF:0,"+row[1]+"\n")
@@ -512,8 +512,8 @@ def doGrab(self, override=False):
     def grabXML(self):
         try:
             xmltv.getProgList(version)
-        except:
-            print ("XMLTV import could not be completed, please try again later (%s)" % sys.exc_info()[0])
+        except Exception as ex:
+            print ("XMLTV import could not be completed, please try again later (%s)" % ex)
         self.epggrabberstate[0] += 1
 
     def grabStream(self):
@@ -836,7 +836,7 @@ def doRecord(self):
         if streamtype in fftypes:
             delta = total(self.bis - datetime.now())
             deltasec = '%d' % delta
-            attr = [config.cfg_ffmpeg_path,"-i", self.url, '-y', '-loglevel', 'error', '-t', deltasec] + ffargs + [fn]
+            attr = [config.cfg_ffmpeg_path,"-i", self.url, '-y', '-loglevel', 'fatal', '-t', deltasec] + ffargs + [fn]
             print ("FFMPEG (%s) record '%s' called with:" % (streamtype, self.name))
             print (attr)
             try:

xmltv.py

@@ -19,7 +19,6 @@
 from __future__ import unicode_literals
 
 
-import xml.etree.ElementTree as et
 from datetime import datetime, timedelta
 try:
     import httplib
@@ -28,95 +27,130 @@
     import http.client as httplib
     import urllib.request as urllib32
 import zlib
-from sql import sqlRun
-import config
-from sys import version_info
+try:
+    from sql import sqlRun
+    import config
+except:
+    def sqlRun(v1="", v2="", v3=""):
+        return None
+
+def getAttr(stri, attr):
+    ret = ""
+    p1 = stri.find(attr) + len(attr) + 2
+    p2 = stri.find(b'"', p1)
+    if p1 >= len(attr) + 2 and p2>p1:
+        ret = stri[p1:p2]
+    return ret
 
-version = ''
+def getFirst(stri, attr):
+    ret = ""
+    p1 = stri.find('<'+attr) 
+    if p1 != -1:
+        p1 = p1 + len(attr) + 1
+        p2 = stri.find('>', p1) + 1 
+        p3 = stri.find('</'+attr, p2) 
+        ret = stri[p2:p3].strip()
+    return ret
+
+def getList(stri, attr):
+#    treelist = list()
+    p1 = stri.find('<'+attr) 
+    while p1!=-1:
+        p1 = p1 + len(attr) + 1 
+        p2 = stri.find('>', p1) + 1     
+        p3 = stri.find('</'+attr, p2) 
+        att = stri[p1:p2-1].strip()
+        txt = stri[p2:p3].strip()
+        yield((att, txt))
+        p1 = stri.find('<'+attr, p3)   
 
 def getProgList(ver=''):
-    global version
-    version = ver    
-    print ("tvstreamrecord v.%s / XMLTV import started" % version)
-    stri = getFile(config.cfg_xmltvinitpath, 1)
-    channellist = []
-    if stri:    
-        tree = et.fromstring(stri)
-        type = tree.attrib.get("generator-info-name")
-        for dict_el in tree.findall('channel'):
-            g_id = dict_el.attrib.get("id")
-            name = dict_el.find('display-name').text
-            if type == "nonametv":
-                url = dict_el.find('base-url').text
-            elif type[:4] == "TVxb":
-                pass
-            else: 
-                print ("Unknown XMLTV generator '%s', please contact me if it fails" % type)
-                url = dict_el.find('base-url').text
-
-            rows=sqlRun("SELECT cname from channels WHERE cname = '%s' and cenabled=1 GROUP BY cname" % name)
-            if rows:
-                timerows=sqlRun("SELECT g_lasttime FROM guide_chan WHERE g_id='%s'" % (g_id))
-                dtmax = datetime.now()
-                if type[:4] == "TVxb":   # same file
-                    channellist.append(g_id)
-                else:               # separate files
-                    lastdate = datetime.now()-timedelta(days=30)
-                    if timerows:
-                        lastdate = datetime.strptime(timerows[0][0], "%Y-%m-%d %H:%M:%S")
-                    dtmax = datetime.min
-                    for tim in dict_el.findall("datafor"):
-                        dttext = tim.text
-                        dtepg  = datetime.strptime(dttext, "%Y-%m-%d")
-                        dt = datetime.strptime(tim.attrib.get("lastmodified")[0:14],"%Y%m%d%H%M%S")
-                        if dt>lastdate and dtepg>=datetime.now()-timedelta(days=1):
-                            source = url+g_id+"_"+dttext+".xml.gz"
-                            stri = getFile(source)
-                            getProg(stri)    
-                        if dt>dtmax:
-                            dtmax = dt
+    print ("tvstreamrecord v.%s / XMLTV import started" % ver)
+    stri = getFile(config.cfg_xmltvinitpath, 1, ver)
+
+    channellist = []        
+    typ = getAttr(stri[:200], "generator-info-name")
+
+    for attr,innertxt in getList(stri, "channel"):
+        g_id = getAttr(attr, "id")
+        name = getFirst(innertxt, 'display-name')
+        if typ == "nonametv":
+            url = getFirst(innertxt, 'base-url')
+        elif typ[:4] == "TVxb":
+            pass
+        else: 
+            print ("Unknown XMLTV generator '%s', please contact me if it fails" % typ)
+            url = getFirst(innertxt, 'base-url')
+
+        rows=sqlRun("SELECT cname from channels WHERE cname = '%s' and cenabled=1 GROUP BY cname" % name)
+        if rows:
+            timerows=sqlRun("SELECT g_lasttime FROM guide_chan WHERE g_id='%s'" % (g_id))
+            dtmax = datetime.now()
+            if typ[:4] == "TVxb":   # same file
+                channellist.append(g_id)
+            else:               # separate files
+                lastdate = datetime.now()-timedelta(days=30)
+                if timerows:
+                    lastdate = datetime.strptime(timerows[0][0], "%Y-%m-%d %H:%M:%S")
+                dtmax = datetime.min
+
+                for t_attr, dttext in getList(innertxt, "datafor"):
+                    dtepg  = datetime.strptime(dttext, "%Y-%m-%d")
+                    dt = datetime.strptime(getAttr(t_attr, "lastmodified")[0:14],"%Y%m%d%H%M%S")
+                    if dt>lastdate and dtepg>=datetime.now()-timedelta(days=1):
+                        source = url+g_id+"_"+dttext+".xml.gz"
+                        getProg(getFile(source,0,ver))
+                    if dt>dtmax:
+                        dtmax = dt
+            if not timerows:
+                sqlRun("INSERT OR IGNORE INTO guide_chan VALUES (?, ?, ?)", (g_id, name, datetime.strftime(dtmax, "%Y-%m-%d %H:%M:%S") ))
+            else:
+                sqlRun("UPDATE guide_chan SET g_lasttime=? WHERE g_id=?", (datetime.strftime(dtmax, "%Y-%m-%d %H:%M:%S"), g_id))                       
 
-                if not timerows:
-                    sqlRun("INSERT OR IGNORE INTO guide_chan VALUES (?, ?, ?)", (g_id, name, datetime.strftime(dtmax, "%Y-%m-%d %H:%M:%S") ))
-                else:
-                    sqlRun("UPDATE guide_chan SET g_lasttime=? WHERE g_id=?", (datetime.strftime(dtmax, "%Y-%m-%d %H:%M:%S"), g_id))                       
-        if type[:4] == "TVxb" and len(channellist)>0:   # same file
-            getProg(stri, channellist)        
+    if typ[:4] == "TVxb" and len(channellist)>0:   # same file
+        getProg(stri, channellist)
+
+    del (stri)        
     print ("XMLTV import completed")
-
-def getProg(stri, channellist=[]):    
+    return
+
+
+def getProg(strp, channellist=[]):    
     sqllist = []
-    #f = open("test.xml", "r")
-    #stri = f.read()
-    if stri: 
-        tree = et.fromstring(stri)
-        for dict_el in tree.findall('programme'):
-            dt1 = datetime.strptime(dict_el.attrib.get("start")[0:14],"%Y%m%d%H%M%S")        
-            dt2 = datetime.strptime(dict_el.attrib.get("stop")[0:14],"%Y%m%d%H%M%S")        
-            p_id = dict_el.attrib.get("channel")
-            if len(channellist)==0 or p_id in channellist:
-                title = ""
-                desc = ""
-                if dict_el.find('title') is not None:
-                    title = dict_el.find('title').text
-                if dict_el.find('sub-title') is not None:
-                    sub_title = dict_el.find('sub-title').text
-                    if not "http://" in sub_title: # fix for corrupted XML data
-                        if title != "": title = title + " - "
-                        title = title + sub_title
-#                if dict_el.find('episode-num[@system="onscreen"]') is not None:
-#                    desc = dict_el.find('episode-num[@system="onscreen"]').text + ". "
-                if dict_el.find('episode-num') is not None:
-                    for en in dict_el:
-                        if en.attrib.get('system') == 'onscreen':
-                            desc = en.text + ". "
-                            break
-                if dict_el.find('desc') is not None:
-                    desc = desc + dict_el.find('desc').text
-                sqllist.append([p_id, title, datetime.strftime(dt1, "%Y-%m-%d %H:%M:%S"), datetime.strftime(dt2, "%Y-%m-%d %H:%M:%S"), desc])
-        sqlRun("INSERT OR IGNORE INTO guide VALUES (?, ?, ?, ?, ?)", sqllist, 1)
+
+    for attr,innertxt in getList(strp, 'programme'):    
+        dt1 = datetime.strptime(getAttr(attr, "start")[0:14],"%Y%m%d%H%M%S")        
+        dt2 = datetime.strptime(getAttr(attr, "stop")[0:14],"%Y%m%d%H%M%S")        
+        p_id = getAttr(attr, "channel")
+        if len(channellist)==0 or p_id in channellist:
+            desc = ""
+            title = getFirst(innertxt, 'title')
+            sub_title = getFirst(innertxt, 'sub-title')
+            if not "http://" in sub_title: # fix for corrupted XML data
+                if title != "": title = title + " - "
+                title = title + sub_title
+            eplist = getFirst(innertxt, 'episode-num')
+            for epatt, epin in getList(eplist, 'system'):
+                if getAttr(epatt, 'system') == 'onscreen':
+                    desc = epin + ". "
+                    break
+            tmpdesc = getFirst(innertxt, 'desc')
+            desc = desc + tmpdesc 
+            sqllist.append([p_id, title, datetime.strftime(dt1, "%Y-%m-%d %H:%M:%S"), datetime.strftime(dt2, "%Y-%m-%d %H:%M:%S"), desc])
+    sqlRun("INSERT OR IGNORE INTO guide VALUES (?, ?, ?, ?, ?)", sqllist, 1)
+
+    return    
 
-def getFile(file_in, override=0):
+def getTestFile():
+    with open('e.xml', 'r') as content_file:
+        stri = content_file.read()
+    try:
+        stri = stri.decode("UTF-8")
+    except:
+        pass
+    return stri
+
+def getFile(file_in, override=0, ver=""):
     rows=sqlRun("SELECT * FROM caching WHERE url='%s'" % file_in)    
     lastmod = ""
     etag = "" 
@@ -127,25 +161,23 @@ def getFile(file_in, override=0):
     try:
         httplib.HTTPConnection.debuglevel = 0                            
         request = urllib32.Request(file_in)
-        request.add_header('User-Agent', 'tvstreamrecord/' + version)
+        request.add_header('User-Agent', 'tvstreamrecord/' + ver)
         if override==0:
             request.add_header('If-Modified-Since', lastmod)
             request.add_header('If-None-Match', etag)                
         opener = urllib32.build_opener()
-        response = opener.open(request)
-        feeddata = response.read()
-        if version_info[0] < 3:
-            response = response.info()
-        lastmod = response.getheader('Last-Modified')
-        etag = response.getheader('ETag')
-        if rows:
+        hresponse = opener.open(request)
+        feeddata = hresponse.read()
+        hr = hresponse.info()        
+        lastmod = hr.get('Last-Modified')
+        etag = hr.get('ETag')
+        if rows and lastmod and etag:
             sqlRun("UPDATE caching SET crTime=datetime('now', 'localtime'), Last_Modified=?, ETag=? WHERE url='%s'" % file_in, (lastmod, etag))
-        else:
+        elif lastmod and etag:
             sqlRun("INSERT INTO caching VALUES (datetime('now', 'localtime'), ?, ?, ?)", (file_in, lastmod, etag))        
         d = zlib.decompressobj(16+zlib.MAX_WBITS)
         out = d.decompress(feeddata)
-        print ("XMLTV: reading URL %s" % file_in)
-
+        print ("XMLTV: reading URL %s with %s bytes" % (file_in, len(out)))
         if not b"</tv>" in out[-1000:]:
             print ("Possibly corrupted XML file, attempting to repair...")
             pos = out.rfind(b"</programme>") 
@@ -156,12 +188,15 @@ def getFile(file_in, override=0):
                 if pos != -1:
                     out = out[:pos+10]  + b"</tv>" 
     except Exception as ex:
-        print ("XMLTV: no new data, try again later")
+        #print (ex)
+        print ("XMLTV: no new data, try again later (%s)" % file_in)
         pass
 
-    if type(out) is bytes and not type(out) is str:
+    try:
         out = out.decode("UTF-8")
-
+    except:
+        pass
+
     return out
 
 def main(argv=None):