From f821954413447a450a318d601a1a3735a3697d94 Mon Sep 17 00:00:00 2001 From: davean Date: Tue, 26 Nov 2013 22:45:50 -0500 Subject: [PATCH 1/5] Check that an image is not too long. --- danceparty/main.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/danceparty/main.py b/danceparty/main.py index 56728b7..9ed0f9d 100644 --- a/danceparty/main.py +++ b/danceparty/main.py @@ -12,12 +12,27 @@ def check_gif(data): img_stream = cStringIO.StringIO(data) + # Confirm it is a GIF first. try: img = Image.open(img_stream) - return img.format =='GIF' + if img.format !='GIF': + return False except IOError: return False - + #Loop through its frames adding up delays untill we run out of frames or exceed 1 second. + #Note that PIL uses seek and tell for frames. I have no idea why. + #It would be nice to iterate over them, but it seems there is just an exception at the end. + duration = img.info['duration'] + try: + #Go through frames summing the durations untill we run out of ms in the second limit. (or, an extra frame) + while duration <= 1050: + img.seek(img.tell()+1) + duration += img.info['duration'] + #If we leave the while loop without an error, we exceeded the time bound. + return False + #We reached the last frame without exceeding the while loops ms time bound. + except EOFError: + return True @app.route('/') def dances_plz(): From 02c36244fb083bf72d8d024f715501e4937e27c4 Mon Sep 17 00:00:00 2001 From: davean Date: Wed, 4 Dec 2013 01:33:29 -0500 Subject: [PATCH 2/5] Improve handling the database instantiation and initialization. --- danceparty/main.py | 70 ++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/danceparty/main.py b/danceparty/main.py index 5840a0e..f09fbe8 100644 --- a/danceparty/main.py +++ b/danceparty/main.py @@ -27,12 +27,44 @@ from danceparty import app +@app.before_first_request +def setup_app(): + if not app.debug: + file_handler = logging.FileHandler(app.config['LOG_FILE']) + file_handler.setLevel(logging.WARNING) + app.logger.addHandler(file_handler) + create_db() + +def create_db(): + couch = couchdb.client.Server() + db_name = app.config['DB_NAME'] + if not db_name in couch: + couch.create(db_name) + db = couch[db_name] -if not app.debug: - file_handler = logging.FileHandler(app.config['LOG_FILE']) - file_handler.setLevel(logging.WARNING) - app.logger.addHandler(file_handler) + views = { + '_id': '_design/' + db_name, + 'language': 'javascript', + 'views': { + 'approved': { + 'map': "function(doc) { if (doc.status == 'approved') { emit(doc.ts, doc) } }" + }, + 'review-queue': { + 'map': "function(doc) { if (doc.status == 'new') { emit(doc.ts, doc) } }" + }, + 'all': { + 'map': "function(doc) { emit(doc.ts, doc) }" + }, + }, + } + doc = db.get(views['_id'], {}) + if doc.get('views') != views['views']: + doc.update(views) + db.save(doc) +def connect_db(): + couch = couchdb.client.Server() + g.db = couch[app.config['DB_NAME']] def check_gif(data): img_stream = cStringIO.StringIO(data) @@ -58,34 +90,6 @@ def check_gif(data): except EOFError: return True -def connect_db(): - g.couch = couchdb.client.Server() - db_name = app.config['DB_NAME'] - if not db_name in g.couch: - g.couch.create(db_name) - g.db = g.couch[db_name] - - views = { - '_id': '_design/' + db_name, - 'language': 'javascript', - 'views': { - 'approved': { - 'map': "function(doc) { if (doc.status == 'approved') { emit(doc.ts, doc) } }" - }, - 'review-queue': { - 'map': "function(doc) { if (doc.status == 'new') { emit(doc.ts, doc) } }" - }, - 'all': { - 'map': "function(doc) { emit(doc.ts, doc) }" - }, - }, - } - doc = g.db.get(views['_id'], {}) - if doc.get('views') != views['views']: - doc.update(views) - g.db.save(doc) - - def dance_json(dance): data = {} data['id'] = dance['_id'] @@ -132,7 +136,7 @@ def csrf_token(salt=None): @app.before_request def before_request(): - g.couch = connect_db() + connect_db() if request.method not in ['GET', 'HEAD', 'OPTIONS']: if (not request.headers.get('X-CSRFT') or From c09fada20a08df0bd32c0f7d25bbf725b38656ef Mon Sep 17 00:00:00 2001 From: davean Date: Wed, 4 Dec 2013 02:37:26 -0500 Subject: [PATCH 3/5] Use a polling-based cache for the main page's json. --- danceparty/default_settings.py | 1 + danceparty/main.py | 27 +++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/danceparty/default_settings.py b/danceparty/default_settings.py index 2c33cf0..ec8f3c7 100644 --- a/danceparty/default_settings.py +++ b/danceparty/default_settings.py @@ -1,3 +1,4 @@ DEBUG = False MAX_CONTENT_LENGTH = 2 * 1024 * 1024 # MB DB_NAME = "danceparty" +POLL_INTERVAL = 30 diff --git a/danceparty/main.py b/danceparty/main.py index f09fbe8..bbb2046 100644 --- a/danceparty/main.py +++ b/danceparty/main.py @@ -6,6 +6,7 @@ import random import time import uuid +from threading import Thread from functools import wraps import bcrypt @@ -18,7 +19,7 @@ redirect, request, Response, - render_template, + render_template, render_template_string, session, send_from_directory, url_for, @@ -27,6 +28,17 @@ from danceparty import app + +def poll_cache(): + while True: + time.sleep(app.config['POLL_INTERVAL']) + update_dances_cache() + + +dances_cache = None +poller = Thread(target=poll_cache) + + @app.before_first_request def setup_app(): if not app.debug: @@ -34,6 +46,15 @@ def setup_app(): file_handler.setLevel(logging.WARNING) app.logger.addHandler(file_handler) create_db() + update_dances_cache() #Make sure it is defined before any requests occur. + poller.daemon=True + poller.start() + +def update_dances_cache(): + global dances_cache + with app.app_context(): + connect_db() + dances_cache = dances_json('danceparty/approved') def create_db(): couch = couchdb.client.Server() @@ -62,10 +83,12 @@ def create_db(): doc.update(views) db.save(doc) + def connect_db(): couch = couchdb.client.Server() g.db = couch[app.config['DB_NAME']] + def check_gif(data): img_stream = cStringIO.StringIO(data) # Confirm it is a GIF first. @@ -156,7 +179,7 @@ def before_request(): @app.route('/') def dances_plz(): return render_template('dance.html', - dances_json=dances_json('danceparty/approved'), + dances_json=dances_cache, config={'mode': 'party', 'csrft': csrf_token()}, ) From cad97ea04fe9c4a5f95738255f6f77b283f5cc2d Mon Sep 17 00:00:00 2001 From: davean Date: Wed, 4 Dec 2013 17:17:37 -0500 Subject: [PATCH 4/5] Basic rate limit based on a simple leaking bucket. --- danceparty/default_settings.py | 2 ++ danceparty/main.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/danceparty/default_settings.py b/danceparty/default_settings.py index ba96739..d3e9627 100644 --- a/danceparty/default_settings.py +++ b/danceparty/default_settings.py @@ -6,3 +6,5 @@ CDN_HTTPS_HOST = None CACHE_POLL_INTERVAL = 30 GA_ID = None +UPLOAD_RATE_PERIOD = 90 +UPLOAD_RATE_COUNT = 3 diff --git a/danceparty/main.py b/danceparty/main.py index 5ab6ab7..f9c8464 100644 --- a/danceparty/main.py +++ b/danceparty/main.py @@ -79,7 +79,11 @@ def create_db(): 'all': { 'map': "function(doc) { emit(doc.ts, doc) }" }, - }, + 'upload_rate': { + 'map': "function(doc) { emit(doc.ip, [doc.ts]) }", + 'reduce': "function (key, values, rereduce) { return [].concat.apply([], values).sort().reverse().slice(0,%d); }"%app.config['UPLOAD_RATE_COUNT'] + }, + } } doc = db.get(views['_id'], {}) if doc.get('views') != views['views']: @@ -250,6 +254,12 @@ def remove_dance(dance_id): @app.route('/dance', methods=['POST']) def upload_dance(): + recent = g.db.view('danceparty/upload_rate',group=True, group_level=1, stale='update_after', key=request.remote_addr) + if recent: + now = time.time() + if app.config['UPLOAD_RATE_COUNT'] <= \ + len(filter(lambda t: t>(now - app.config['UPLOAD_RATE_PERIOD']), recent.rows[0].value)): + abort(403) gif = request.files['moves'] gif_data = gif.read() if gif and check_gif(gif_data): From db4237389c2b06121b41db6376da1ad8ad5373a5 Mon Sep 17 00:00:00 2001 From: davean Date: Wed, 4 Dec 2013 21:47:10 -0500 Subject: [PATCH 5/5] Don't count approved uploads against one's quota. Also, due to an implimentation detail, don't prevent uploads from people who the majority of recent uploads have been approved. --- danceparty/default_settings.py | 4 ++-- danceparty/main.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/danceparty/default_settings.py b/danceparty/default_settings.py index d3e9627..aa39dcf 100644 --- a/danceparty/default_settings.py +++ b/danceparty/default_settings.py @@ -6,5 +6,5 @@ CDN_HTTPS_HOST = None CACHE_POLL_INTERVAL = 30 GA_ID = None -UPLOAD_RATE_PERIOD = 90 -UPLOAD_RATE_COUNT = 3 +UPLOAD_RATE_PERIOD = 86400 # 1 day +UPLOAD_RATE_COUNT = 2 diff --git a/danceparty/main.py b/danceparty/main.py index f9c8464..3aa8d74 100644 --- a/danceparty/main.py +++ b/danceparty/main.py @@ -80,8 +80,8 @@ def create_db(): 'map': "function(doc) { emit(doc.ts, doc) }" }, 'upload_rate': { - 'map': "function(doc) { emit(doc.ip, [doc.ts]) }", - 'reduce': "function (key, values, rereduce) { return [].concat.apply([], values).sort().reverse().slice(0,%d); }"%app.config['UPLOAD_RATE_COUNT'] + 'map': "function(doc) { if(doc.status != 'approved') { emit(doc.ip, [doc.ts]) } }", + 'reduce': "function (key, values, rereduce) { return [].concat.apply([], values).sort().reverse().slice(0,%d); }"%(app.config['UPLOAD_RATE_COUNT']*2) #it is assumed if the majority of the uploads in the bucket are approved that they aren't spammers. }, } }