From 06bd8673b8b1750605db4be9280f01392f1cef35 Mon Sep 17 00:00:00 2001 From: Pavion Date: Thu, 21 Aug 2014 18:19:05 +0200 Subject: [PATCH] Complete XMLTV-revamp to avoid memory leak on DS Several fixes --- js/tvstreamrecord.basic.js | 2 + sql.py | 5 +- synology/INFO | 2 +- tvstreamrecord.py | 14 +-- xmltv.py | 229 +++++++++++++++++++++---------------- 5 files changed, 144 insertions(+), 108 deletions(-) diff --git a/js/tvstreamrecord.basic.js b/js/tvstreamrecord.basic.js index 6f52a57..0ca9e1c 100644 --- a/js/tvstreamrecord.basic.js +++ b/js/tvstreamrecord.basic.js @@ -830,11 +830,13 @@ $(function() { $(this).css("margin-top", cnt==0?10:(80+(cnt-1)*100) + 'px'); $(this).css("margin-left", x+'%'); $(this).css("width", w+'%'); +// if (cnt==0) $(this).css("position", 'fixed'); } else { $(this).css("left", cnt==0?0:((cnt-1) * 250 + 60) + 'px'); $(this).css("top", x+'%'); $(this).css("width", cnt==0?'50px':'240px'); $(this).css("height", w + "%"); +// if (cnt==0) $(this).css("position", 'fixed'); } }); diff --git a/sql.py b/sql.py index 9964c67..44662c2 100644 --- a/sql.py +++ b/sql.py @@ -23,7 +23,7 @@ def sqlRun(sql, t=-1, many=0): fa = [] try: - conn = sqlite3.connect('settings.db') + conn = sqlite3.connect('settings.db', timeout=20) c = conn.cursor() c.execute('PRAGMA journal_mode = OFF;') conn.text_factory = sqlite3.OptimizedUnicode @@ -42,8 +42,7 @@ def sqlRun(sql, t=-1, many=0): conn.commit() conn.close() except Exception as ex: - print ("SQL Exception: %s" % sql) -# print (ex) + print ("SQL Exception '%s' with '%s'" % (ex,sql)) pass return fa diff --git a/synology/INFO b/synology/INFO index 1b4a710..f020168 100644 --- a/synology/INFO +++ b/synology/INFO @@ -1,5 +1,5 @@ package="tvstreamrecord" -version="0.6.4a" +version="0.6.4b" maintainer="Pavion" description="Program to record TV streams. Python 2.x required. Server runs at port 8030 (changeable)" reloadui="yes" diff --git a/tvstreamrecord.py b/tvstreamrecord.py index 71e851e..ee61bd3 100644 --- a/tvstreamrecord.py +++ b/tvstreamrecord.py @@ -40,6 +40,7 @@ import sys from mylogging import logInit, logRenew, logStop import hashlib +import codecs def total(timedelta): try: @@ -52,7 +53,7 @@ def total(timedelta): localtime = "%H:%M" localdate = "%d.%m.%Y" dayshown = datetime.combine(date.today(), time.min) -version = '0.6.4a' +version = '0.6.4b' @route('/live/') def server_static9(filename): @@ -64,13 +65,12 @@ def server_static9(filename): return static_file("/live.m3u", root='', mimetype='video') def write_m3u(name, path): - f = open("live.m3u", "w") + f = codecs.open("live.m3u", "w", "utf-8") f.write("#EXTM3U\n") f.write("#EXTINF:0,"+name+"\n") f.write(path+"\n") f.close() - @route('/channels.m3u') def server_static8(): return static_file("/channels.m3u", root='') @@ -276,7 +276,7 @@ def list_p(): def clgen_p(): rows = sqlRun("select cid, cname, cpath from channels where cenabled=1 ORDER BY cid") if rows: - f = open("channels.m3u", "w") + f = codecs.open("channels.m3u", "w", "utf-8") f.write("#EXTM3U\n") for row in rows: f.write("#EXTINF:0,"+row[1]+"\n") @@ -512,8 +512,8 @@ def doGrab(self, override=False): def grabXML(self): try: xmltv.getProgList(version) - except: - print ("XMLTV import could not be completed, please try again later (%s)" % sys.exc_info()[0]) + except Exception as ex: + print ("XMLTV import could not be completed, please try again later (%s)" % ex) self.epggrabberstate[0] += 1 def grabStream(self): @@ -836,7 +836,7 @@ def doRecord(self): if streamtype in fftypes: delta = total(self.bis - datetime.now()) deltasec = '%d' % delta - attr = [config.cfg_ffmpeg_path,"-i", self.url, '-y', '-loglevel', 'error', '-t', deltasec] + ffargs + [fn] + attr = [config.cfg_ffmpeg_path,"-i", self.url, '-y', '-loglevel', 'fatal', '-t', deltasec] + ffargs + [fn] print ("FFMPEG (%s) record '%s' called with:" % (streamtype, self.name)) print (attr) try: diff --git a/xmltv.py b/xmltv.py index f790117..49a02d3 100644 --- a/xmltv.py +++ b/xmltv.py @@ -19,7 +19,6 @@ from __future__ import unicode_literals -import xml.etree.ElementTree as et from datetime import datetime, timedelta try: import httplib @@ -28,95 +27,130 @@ import http.client as httplib import urllib.request as urllib32 import zlib -from sql import sqlRun -import config -from sys import version_info +try: + from sql import sqlRun + import config +except: + def sqlRun(v1="", v2="", v3=""): + return None + +def getAttr(stri, attr): + ret = "" + p1 = stri.find(attr) + len(attr) + 2 + p2 = stri.find(b'"', p1) + if p1 >= len(attr) + 2 and p2>p1: + ret = stri[p1:p2] + return ret -version = '' +def getFirst(stri, attr): + ret = "" + p1 = stri.find('<'+attr) + if p1 != -1: + p1 = p1 + len(attr) + 1 + p2 = stri.find('>', p1) + 1 + p3 = stri.find('', p1) + 1 + p3 = stri.find('lastdate and dtepg>=datetime.now()-timedelta(days=1): - source = url+g_id+"_"+dttext+".xml.gz" - stri = getFile(source) - getProg(stri) - if dt>dtmax: - dtmax = dt + print ("tvstreamrecord v.%s / XMLTV import started" % ver) + stri = getFile(config.cfg_xmltvinitpath, 1, ver) + + channellist = [] + typ = getAttr(stri[:200], "generator-info-name") + + for attr,innertxt in getList(stri, "channel"): + g_id = getAttr(attr, "id") + name = getFirst(innertxt, 'display-name') + if typ == "nonametv": + url = getFirst(innertxt, 'base-url') + elif typ[:4] == "TVxb": + pass + else: + print ("Unknown XMLTV generator '%s', please contact me if it fails" % typ) + url = getFirst(innertxt, 'base-url') + + rows=sqlRun("SELECT cname from channels WHERE cname = '%s' and cenabled=1 GROUP BY cname" % name) + if rows: + timerows=sqlRun("SELECT g_lasttime FROM guide_chan WHERE g_id='%s'" % (g_id)) + dtmax = datetime.now() + if typ[:4] == "TVxb": # same file + channellist.append(g_id) + else: # separate files + lastdate = datetime.now()-timedelta(days=30) + if timerows: + lastdate = datetime.strptime(timerows[0][0], "%Y-%m-%d %H:%M:%S") + dtmax = datetime.min + + for t_attr, dttext in getList(innertxt, "datafor"): + dtepg = datetime.strptime(dttext, "%Y-%m-%d") + dt = datetime.strptime(getAttr(t_attr, "lastmodified")[0:14],"%Y%m%d%H%M%S") + if dt>lastdate and dtepg>=datetime.now()-timedelta(days=1): + source = url+g_id+"_"+dttext+".xml.gz" + getProg(getFile(source,0,ver)) + if dt>dtmax: + dtmax = dt + if not timerows: + sqlRun("INSERT OR IGNORE INTO guide_chan VALUES (?, ?, ?)", (g_id, name, datetime.strftime(dtmax, "%Y-%m-%d %H:%M:%S") )) + else: + sqlRun("UPDATE guide_chan SET g_lasttime=? WHERE g_id=?", (datetime.strftime(dtmax, "%Y-%m-%d %H:%M:%S"), g_id)) - if not timerows: - sqlRun("INSERT OR IGNORE INTO guide_chan VALUES (?, ?, ?)", (g_id, name, datetime.strftime(dtmax, "%Y-%m-%d %H:%M:%S") )) - else: - sqlRun("UPDATE guide_chan SET g_lasttime=? WHERE g_id=?", (datetime.strftime(dtmax, "%Y-%m-%d %H:%M:%S"), g_id)) - if type[:4] == "TVxb" and len(channellist)>0: # same file - getProg(stri, channellist) + if typ[:4] == "TVxb" and len(channellist)>0: # same file + getProg(stri, channellist) + + del (stri) print ("XMLTV import completed") - -def getProg(stri, channellist=[]): + return + + +def getProg(strp, channellist=[]): sqllist = [] - #f = open("test.xml", "r") - #stri = f.read() - if stri: - tree = et.fromstring(stri) - for dict_el in tree.findall('programme'): - dt1 = datetime.strptime(dict_el.attrib.get("start")[0:14],"%Y%m%d%H%M%S") - dt2 = datetime.strptime(dict_el.attrib.get("stop")[0:14],"%Y%m%d%H%M%S") - p_id = dict_el.attrib.get("channel") - if len(channellist)==0 or p_id in channellist: - title = "" - desc = "" - if dict_el.find('title') is not None: - title = dict_el.find('title').text - if dict_el.find('sub-title') is not None: - sub_title = dict_el.find('sub-title').text - if not "http://" in sub_title: # fix for corrupted XML data - if title != "": title = title + " - " - title = title + sub_title -# if dict_el.find('episode-num[@system="onscreen"]') is not None: -# desc = dict_el.find('episode-num[@system="onscreen"]').text + ". " - if dict_el.find('episode-num') is not None: - for en in dict_el: - if en.attrib.get('system') == 'onscreen': - desc = en.text + ". " - break - if dict_el.find('desc') is not None: - desc = desc + dict_el.find('desc').text - sqllist.append([p_id, title, datetime.strftime(dt1, "%Y-%m-%d %H:%M:%S"), datetime.strftime(dt2, "%Y-%m-%d %H:%M:%S"), desc]) - sqlRun("INSERT OR IGNORE INTO guide VALUES (?, ?, ?, ?, ?)", sqllist, 1) + + for attr,innertxt in getList(strp, 'programme'): + dt1 = datetime.strptime(getAttr(attr, "start")[0:14],"%Y%m%d%H%M%S") + dt2 = datetime.strptime(getAttr(attr, "stop")[0:14],"%Y%m%d%H%M%S") + p_id = getAttr(attr, "channel") + if len(channellist)==0 or p_id in channellist: + desc = "" + title = getFirst(innertxt, 'title') + sub_title = getFirst(innertxt, 'sub-title') + if not "http://" in sub_title: # fix for corrupted XML data + if title != "": title = title + " - " + title = title + sub_title + eplist = getFirst(innertxt, 'episode-num') + for epatt, epin in getList(eplist, 'system'): + if getAttr(epatt, 'system') == 'onscreen': + desc = epin + ". " + break + tmpdesc = getFirst(innertxt, 'desc') + desc = desc + tmpdesc + sqllist.append([p_id, title, datetime.strftime(dt1, "%Y-%m-%d %H:%M:%S"), datetime.strftime(dt2, "%Y-%m-%d %H:%M:%S"), desc]) + sqlRun("INSERT OR IGNORE INTO guide VALUES (?, ?, ?, ?, ?)", sqllist, 1) + + return -def getFile(file_in, override=0): +def getTestFile(): + with open('e.xml', 'r') as content_file: + stri = content_file.read() + try: + stri = stri.decode("UTF-8") + except: + pass + return stri + +def getFile(file_in, override=0, ver=""): rows=sqlRun("SELECT * FROM caching WHERE url='%s'" % file_in) lastmod = "" etag = "" @@ -127,25 +161,23 @@ def getFile(file_in, override=0): try: httplib.HTTPConnection.debuglevel = 0 request = urllib32.Request(file_in) - request.add_header('User-Agent', 'tvstreamrecord/' + version) + request.add_header('User-Agent', 'tvstreamrecord/' + ver) if override==0: request.add_header('If-Modified-Since', lastmod) request.add_header('If-None-Match', etag) opener = urllib32.build_opener() - response = opener.open(request) - feeddata = response.read() - if version_info[0] < 3: - response = response.info() - lastmod = response.getheader('Last-Modified') - etag = response.getheader('ETag') - if rows: + hresponse = opener.open(request) + feeddata = hresponse.read() + hr = hresponse.info() + lastmod = hr.get('Last-Modified') + etag = hr.get('ETag') + if rows and lastmod and etag: sqlRun("UPDATE caching SET crTime=datetime('now', 'localtime'), Last_Modified=?, ETag=? WHERE url='%s'" % file_in, (lastmod, etag)) - else: + elif lastmod and etag: sqlRun("INSERT INTO caching VALUES (datetime('now', 'localtime'), ?, ?, ?)", (file_in, lastmod, etag)) d = zlib.decompressobj(16+zlib.MAX_WBITS) out = d.decompress(feeddata) - print ("XMLTV: reading URL %s" % file_in) - + print ("XMLTV: reading URL %s with %s bytes" % (file_in, len(out))) if not b"" in out[-1000:]: print ("Possibly corrupted XML file, attempting to repair...") pos = out.rfind(b"") @@ -156,12 +188,15 @@ def getFile(file_in, override=0): if pos != -1: out = out[:pos+10] + b"" except Exception as ex: - print ("XMLTV: no new data, try again later") + #print (ex) + print ("XMLTV: no new data, try again later (%s)" % file_in) pass - if type(out) is bytes and not type(out) is str: + try: out = out.decode("UTF-8") - + except: + pass + return out def main(argv=None):