From 12ad16f29218ecad59ba0544c2f094733c7c8bf7 Mon Sep 17 00:00:00 2001 From: firm1 Date: Tue, 12 Aug 2014 00:45:11 +0200 Subject: [PATCH 1/3] correction des bugs de download pendant la validation --- fixtures/noir_black.png | Bin 0 -> 350 bytes zds/tutorial/factories.py | 30 +++++++++++++++-------- zds/tutorial/views.py | 49 ++++++++++++++++++++++---------------- 3 files changed, 49 insertions(+), 30 deletions(-) create mode 100644 fixtures/noir_black.png diff --git a/fixtures/noir_black.png b/fixtures/noir_black.png new file mode 100644 index 0000000000000000000000000000000000000000..118bf04693c693d4f8e731777685e997d20f64e1 GIT binary patch literal 350 zcmeAS@N?(olHy`uVBq!ia0vp^(?D2&2}nNvZ1oLDaTa()76WMyFm^kcZ3hx8D{xE) z(qPpLKfk4(26AS4x;TbZ+(O#u=NMg|6!xWol_gnp|vd$@?2>`?uI>-P3 literal 0 HcmV?d00001 diff --git a/zds/tutorial/factories.py b/zds/tutorial/factories.py index 5bbf344fa4..1ec34ca23d 100644 --- a/zds/tutorial/factories.py +++ b/zds/tutorial/factories.py @@ -20,7 +20,17 @@ from zds.utils.models import SubCategory from zds.utils.tutorials import export_tutorial - +contenu = ( +u'Ceci est un contenu de tutoriel utile et à tester un peu partout' +u'Ce contenu ira aussi bien dans les introductions, que dans les conclusions et les extraits ' +u'le gros intéret étant qu\'il renferme des images pour tester l\'execution coté pandoc ' +u'Exemple d\'image ![Ma pepite souris](http://blog.science-infuse.fr/public/souris.jpg)' +u'\nExemple d\'image ![Image inexistante](http://blog.science-infuse.fr/public/inv_souris.jpg)' +u'\nExemple de gif ![](http://corigif.free.fr/oiseau/img/oiseau_004.gif)' +u'\nExemple de gif inexistant ![](http://corigif.free.fr/oiseau/img/ironman.gif)' +u'\n Attention les tests ne doivent pas crasher ' +u'qu\'un sujet abandonné !') + class BigTutorialFactory(factory.DjangoModelFactory): FACTORY_FOR = Tutorial @@ -47,10 +57,10 @@ def _prepare(cls, create, **kwargs): f.write(json_writer.dumps(man, indent=4, ensure_ascii=False).encode('utf-8')) f.close() f = open(os.path.join(path, tuto.introduction), "w") - f.write(u'Test') + f.write(contenu.encode('utf-8')) f.close() f = open(os.path.join(path, tuto.conclusion), "w") - f.write(u'Test') + f.write(contenu.encode('utf-8')) f.close() repo.index.add(['manifest.json', tuto.introduction, tuto.conclusion]) cm = repo.index.commit("Init Tuto") @@ -90,10 +100,10 @@ def _prepare(cls, create, **kwargs): ensure_ascii=False).encode('utf-8')) file.close() file = open(os.path.join(path, tuto.introduction), "w") - file.write(u'Test') + file.write(contenu.encode('utf-8')) file.close() file = open(os.path.join(path, tuto.conclusion), "w") - file.write(u'Test') + file.write(contenu.encode('utf-8')) file.close() repo.index.add(['manifest.json', tuto.introduction, tuto.conclusion]) @@ -124,11 +134,11 @@ def _prepare(cls, create, **kwargs): part.save() f = open(os.path.join(tutorial.get_path(), part.introduction), "w") - f.write(u'Test') + f.write(contenu.encode('utf-8')) f.close() repo.index.add([part.introduction]) f = open(os.path.join(tutorial.get_path(), part.conclusion), "w") - f.write(u'Test') + f.write(contenu.encode('utf-8')) f.close() repo.index.add([part.conclusion]) @@ -202,14 +212,14 @@ def _prepare(cls, create, **kwargs): part.tutorial.get_path(), chapter.introduction), "w") - f.write(u'Test') + f.write(contenu.encode('utf-8')) f.close() f = open( os.path.join( part.tutorial.get_path(), chapter.conclusion), "w") - f.write(u'Test') + f.write(contenu.encode('utf-8')) f.close() part.tutorial.save() repo = Repo(part.tutorial.get_path()) @@ -289,4 +299,4 @@ class SubCategoryFactory(factory.DjangoModelFactory): class VaidationFactory(factory.DjangoModelFactory): - FACTORY_FOR = Validation + FACTORY_FOR = Validation \ No newline at end of file diff --git a/zds/tutorial/views.py b/zds/tutorial/views.py index db783eb451..e513ab9456 100644 --- a/zds/tutorial/views.py +++ b/zds/tutorial/views.py @@ -2774,6 +2774,7 @@ def get_url_images(md_text, pt): """find images urls in markdown text and download this.""" regex = ur"(!\[.*?\]\()(.+?)(\))" + unknow_path = os.path.join(settings.SITE_ROOT, "fixtures", "noir_black.png") # if text is empty don't download @@ -2786,8 +2787,7 @@ def get_url_images(md_text, pt): parse_object = urlparse(img[1]) # if link is http type - - if parse_object.scheme in ("http", "https", "ftp") or \ + if parse_object.scheme in ["http", "https", "ftp"] or \ parse_object.netloc[:3]=="www" or \ parse_object.path[:3]=="www": (filepath, filename) = os.path.split(parse_object.path) @@ -2795,20 +2795,23 @@ def get_url_images(md_text, pt): os.makedirs(os.path.join(pt, "images")) # download image - - urlretrieve(img[1], os.path.abspath(os.path.join(pt, "images", - filename))) - ext = filename.split(".")[-1] - - # if image is gif, convert to png - - if ext == "gif": - im = ImagePIL.open(os.path.join(pt, img[1])) - im.save(os.path.join(pt, filename.split(".")[0] + ".png")) + down_path=os.path.abspath(os.path.join(pt, "images", filename)) + urlretrieve(img[1], down_path) + try: + ext = filename.split(".")[-1] + im = ImagePIL.open(down_path) + # if image is gif, convert to png + if ext == "gif": + im.save(os.path.join(pt, "images", filename.split(".")[0] + ".png")) + except IOError: + ext = filename.split(".")[-1] + im = ImagePIL.open(unknow_path) + if ext == "gif": + im.save(os.path.join(pt, "images", filename.split(".")[0] + ".png")) + else: + im.save(os.path.join(pt, "images", filename)) else: - # relative link - srcfile = settings.SITE_ROOT + img[1] if os.path.isfile(srcfile): dstroot = pt + img[1] @@ -2816,13 +2819,19 @@ def get_url_images(md_text, pt): if not os.path.exists(dstdir): os.makedirs(dstdir) shutil.copy(srcfile, dstroot) - ext = dstroot.split(".")[-1] - - # if image is gif, convert to png - - if ext == "gif": + try: + ext = dstroot.split(".")[-1] im = ImagePIL.open(dstroot) - im.save(os.path.join(dstroot.split(".")[0] + ".png")) + # if image is gif, convert to png + if ext == "gif": + im.save(os.path.join(dstroot.split(".")[0] + ".png")) + except IOError: + ext = dstroot.split(".")[-1] + im = ImagePIL.open(unknow_path) + if ext == "gif": + im.save(os.path.join(dstroot.split(".")[0] + ".png")) + else: + im.save(os.path.join(dstroot)) def sub_urlimg(g): From 46cebf1c729ca0929700b6703b703940119455ae Mon Sep 17 00:00:00 2001 From: firm1 Date: Sun, 17 Aug 2014 20:24:21 +0200 Subject: [PATCH 2/3] correction des images demandant une requete --- zds/tutorial/factories.py | 1 + zds/tutorial/views.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/zds/tutorial/factories.py b/zds/tutorial/factories.py index 1ec34ca23d..c066921153 100644 --- a/zds/tutorial/factories.py +++ b/zds/tutorial/factories.py @@ -28,6 +28,7 @@ u'\nExemple d\'image ![Image inexistante](http://blog.science-infuse.fr/public/inv_souris.jpg)' u'\nExemple de gif ![](http://corigif.free.fr/oiseau/img/oiseau_004.gif)' u'\nExemple de gif inexistant ![](http://corigif.free.fr/oiseau/img/ironman.gif)' +u'Une image de type wikipedia qui fait tomber des tests ![](https://s.qwant.com/thumbr/?u=http%3A%2F%2Fwww.blogoergosum.com%2Fwp-content%2Fuploads%2F2010%2F02%2Fwikipedia-logo.jpg&h=338&w=600)' u'\n Attention les tests ne doivent pas crasher ' u'qu\'un sujet abandonné !') diff --git a/zds/tutorial/views.py b/zds/tutorial/views.py index e513ab9456..e6f11c6602 100644 --- a/zds/tutorial/views.py +++ b/zds/tutorial/views.py @@ -4,7 +4,7 @@ from datetime import datetime from operator import attrgetter from urllib import urlretrieve -from urlparse import urlparse +from urlparse import urlparse, parse_qs try: import ujson as json_reader except: @@ -2782,9 +2782,11 @@ def get_url_images(md_text, pt): imgs = re.findall(regex, md_text) for img in imgs: - # decompose images - + # decompose images parse_object = urlparse(img[1]) + if parse_object.query!='': + resp = parse_qs(urlparse(img[1]).query, keep_blank_values=True) + parse_object = urlparse(resp["u"][0]) # if link is http type if parse_object.scheme in ["http", "https", "ftp"] or \ @@ -2838,6 +2840,9 @@ def sub_urlimg(g): start = g.group("start") url = g.group("url") parse_object = urlparse(url) + if parse_object.query!='': + resp = parse_qs(urlparse(url).query, keep_blank_values=True) + parse_object = urlparse(resp["u"][0]) (filepath, filename) = os.path.split(parse_object.path) ext = filename.split(".")[-1] if ext == "gif": From a3d3020462dc66a5967e8812b7988004245d185e Mon Sep 17 00:00:00 2001 From: firm1 Date: Mon, 18 Aug 2014 00:52:41 +0200 Subject: [PATCH 3/3] corrige les cas d'image avec dns introuvables --- zds/tutorial/factories.py | 1 + zds/tutorial/views.py | 76 +++++++++++++++++++++------------------ 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/zds/tutorial/factories.py b/zds/tutorial/factories.py index c066921153..08897513b7 100644 --- a/zds/tutorial/factories.py +++ b/zds/tutorial/factories.py @@ -29,6 +29,7 @@ u'\nExemple de gif ![](http://corigif.free.fr/oiseau/img/oiseau_004.gif)' u'\nExemple de gif inexistant ![](http://corigif.free.fr/oiseau/img/ironman.gif)' u'Une image de type wikipedia qui fait tomber des tests ![](https://s.qwant.com/thumbr/?u=http%3A%2F%2Fwww.blogoergosum.com%2Fwp-content%2Fuploads%2F2010%2F02%2Fwikipedia-logo.jpg&h=338&w=600)' +u'Image dont le serveur n\'existe pas ![](http://unknown.image.zds)' u'\n Attention les tests ne doivent pas crasher ' u'qu\'un sujet abandonné !') diff --git a/zds/tutorial/views.py b/zds/tutorial/views.py index e6f11c6602..a4897d89e6 100644 --- a/zds/tutorial/views.py +++ b/zds/tutorial/views.py @@ -2781,12 +2781,13 @@ def get_url_images(md_text, pt): if md_text is not None: imgs = re.findall(regex, md_text) for img in imgs: - + real_url=img[1] # decompose images - parse_object = urlparse(img[1]) + parse_object = urlparse(real_url) if parse_object.query!='': resp = parse_qs(urlparse(img[1]).query, keep_blank_values=True) - parse_object = urlparse(resp["u"][0]) + real_url = resp["u"][0] + parse_object = urlparse(real_url) # if link is http type if parse_object.scheme in ["http", "https", "ftp"] or \ @@ -2798,25 +2799,28 @@ def get_url_images(md_text, pt): # download image down_path=os.path.abspath(os.path.join(pt, "images", filename)) - urlretrieve(img[1], down_path) try: - ext = filename.split(".")[-1] - im = ImagePIL.open(down_path) - # if image is gif, convert to png - if ext == "gif": - im.save(os.path.join(pt, "images", filename.split(".")[0] + ".png")) + urlretrieve(real_url, down_path) + try: + ext = filename.split(".")[-1] + im = ImagePIL.open(down_path) + # if image is gif, convert to png + if ext == "gif": + im.save(os.path.join(pt, "images", filename.split(".")[0] + ".png")) + except IOError: + ext = filename.split(".")[-1] + im = ImagePIL.open(unknow_path) + if ext == "gif": + im.save(os.path.join(pt, "images", filename.split(".")[0] + ".png")) + else: + im.save(os.path.join(pt, "images", filename)) except IOError: - ext = filename.split(".")[-1] - im = ImagePIL.open(unknow_path) - if ext == "gif": - im.save(os.path.join(pt, "images", filename.split(".")[0] + ".png")) - else: - im.save(os.path.join(pt, "images", filename)) + pass else: # relative link - srcfile = settings.SITE_ROOT + img[1] + srcfile = settings.SITE_ROOT + real_url if os.path.isfile(srcfile): - dstroot = pt + img[1] + dstroot = pt + real_url dstdir = os.path.dirname(dstroot) if not os.path.exists(dstdir): os.makedirs(dstdir) @@ -2844,27 +2848,31 @@ def sub_urlimg(g): resp = parse_qs(urlparse(url).query, keep_blank_values=True) parse_object = urlparse(resp["u"][0]) (filepath, filename) = os.path.split(parse_object.path) - ext = filename.split(".")[-1] - if ext == "gif": - if parse_object.scheme in ("http", "https") or \ - parse_object.netloc[:3]=="www" or \ - parse_object.path[:3]=="www": - url = os.path.join("images", filename.split(".")[0] + ".png") + if filename!='': + mark= g.group("mark") + ext = filename.split(".")[-1] + if ext == "gif": + if parse_object.scheme in ("http", "https") or \ + parse_object.netloc[:3]=="www" or \ + parse_object.path[:3]=="www": + url = os.path.join("images", filename.split(".")[0] + ".png") + else: + url = (url.split(".")[0])[1:] + ".png" else: - url = (url.split(".")[0])[1:] + ".png" + if parse_object.scheme in ("http", "https") or \ + parse_object.netloc[:3]=="www" or \ + parse_object.path[:3]=="www": + url = os.path.join("images", filename) + else: + url = url[1:] + end = g.group("end") + return start + mark+ url + end else: - if parse_object.scheme in ("http", "https") or \ - parse_object.netloc[:3]=="www" or \ - parse_object.path[:3]=="www": - url = os.path.join("images", filename) - else: - url = url[1:] - end = g.group("end") - return start + url + end - + return start + def markdown_to_out(md_text): - return re.sub(ur"(?P!\[.*?\]\()(?P.+?)(?P\))", sub_urlimg, + return re.sub(ur"(?P)(?P!\[.*?\]\()(?P.+?)(?P\))", sub_urlimg, md_text)