Merge pull request #310 from MerginMaps/gevent_timeout

Introduce gevent request timeout

.prod.env

@@ -177,3 +177,5 @@ GLOBAL_STORAGE=10737418240
 
 # Gunicorn server socket
 PORT=5000
+
+GEVENT_WORKER=True

server/application.py

@@ -9,14 +9,17 @@
 # Modules that had direct imports (NOT patched): ['urllib3.util, 'urllib3.util.ssl']"
 # which comes from using requests (its deps) lib in webhooks
 
-import os
-from random import randint
+from mergin.config import Configuration as MainConfig
 
-if not os.getenv("NO_MONKEY_PATCH", False):
+if MainConfig.GEVENT_WORKER:
     import gevent.monkey
+    import psycogreen.gevent
+
+    gevent.monkey.patch_all()
+    psycogreen.gevent.patch_psycopg()
 
-    gevent.monkey.patch_all(subprocess=True)
 
+from random import randint
 from celery.schedules import crontab
 from mergin.app import create_app
 from mergin.auth.tasks import anonymize_removed_users

server/config.py

@@ -25,18 +25,6 @@
 """
 import logging
 
-try:
-    from psycogreen.gevent import patch_psycopg
-except ImportError:
-    import sys
-    import traceback
-
-    exception_info = traceback.format_exc()
-    sys.stderr.write(
-        f"Failed to load required functions from the psycogreen library: { exception_info }\n"
-    )
-    sys.exit(1)
-
 worker_class = "gevent"
 
 workers = 2
@@ -59,12 +47,7 @@
 
 max_requests_jitter = 5000
 
-
-def do_post_fork(server, worker):
-    patch_psycopg()
-
-
-post_fork = do_post_fork
+timeout = 30
 
 
 """

server/mergin/__init__.py

@@ -1,5 +1,3 @@
 # Copyright (C) Lutra Consulting Limited
 #
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
-
-from .app import db, mail, ma, create_app

server/mergin/app.py

@@ -7,11 +7,12 @@
 import os
 import connexion
 import wtforms_json
+import gevent
 from marshmallow import fields
 from sqlalchemy.schema import MetaData
 from flask_sqlalchemy import SQLAlchemy
 from flask_marshmallow import Marshmallow
-from flask import json, jsonify, request, abort, current_app, Flask
+from flask import json, jsonify, request, abort, current_app, Flask, Request, Response
 from flask_login import current_user, LoginManager
 from flask_wtf.csrf import generate_csrf, CSRFProtect
 from flask_migrate import Migrate
@@ -27,6 +28,7 @@
 from typing import List, Dict, Optional
 
 from .sync.utils import get_blacklisted_dirs, get_blacklisted_files
+from .config import Configuration
 
 convention = {
     "ix": "ix_%(column_0_label)s",
@@ -105,9 +107,24 @@ def update_obj(self, obj):
                 field.populate_obj(obj, name)
 
 
-def create_simple_app() -> Flask:
-    from .config import Configuration
+class GeventTimeoutMiddleware:
+    """Middleware to implement gevent.Timeout() for all requests"""
+
+    def __init__(self, app):
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        request = Request(environ)
+        try:
+            with gevent.Timeout(Configuration.GEVENT_REQUEST_TIMEOUT):
+                return self.app(environ, start_response)
+        except gevent.Timeout:
+            logging.error(f"Gevent worker: Request {request.path} timed out")
+            resp = Response("Gateway Timeout", mimetype="text/plain", status=504)
+            return resp(environ, start_response)
+
 
+def create_simple_app() -> Flask:
     app = connexion.FlaskApp(__name__, specification_dir=os.path.join(this_dir))
     flask_app = app.app
 
@@ -117,6 +134,9 @@ def create_simple_app() -> Flask:
     ma.init_app(flask_app)
     Migrate(flask_app, db)
     flask_app.connexion_app = app
+    # in case of gevent worker type use middleware to implement custom request timeout
+    if Configuration.GEVENT_WORKER:
+        flask_app.wsgi_app = GeventTimeoutMiddleware(flask_app.wsgi_app)
 
     @flask_app.cli.command()
     def init_db():
@@ -133,7 +153,6 @@ def init_db():
 def create_app(public_keys: List[str] = None) -> Flask:
     """Factory function to create Flask app instance"""
     from itsdangerous import BadTimeSignature, BadSignature
-    from .config import Configuration
     from .auth import auth_required, decode_token
     from .auth.models import User
 

server/mergin/auth/app.py

@@ -12,7 +12,7 @@
 from .commands import add_commands
 from .config import Configuration
 from .models import User, UserProfile
-from .. import db
+from ..app import db
 
 # signal for other versions to listen to
 user_account_closed = signal("user_account_closed")

server/mergin/auth/commands.py

@@ -6,7 +6,7 @@
 from flask import Flask
 from sqlalchemy import or_, func
 
-from .. import db
+from ..app import db
 from .models import User, UserProfile
 
 

server/mergin/auth/controller.py

@@ -35,8 +35,7 @@
     UserChangePasswordForm,
     ApiLoginForm,
 )
-from .. import db
-from ..app import DEPRECATION_API_MSG
+from ..app import DEPRECATION_API_MSG, db
 from ..utils import format_time_delta
 
 

server/mergin/auth/models.py

@@ -9,7 +9,7 @@
 from flask import current_app, request
 from sqlalchemy import or_, func
 
-from .. import db
+from ..app import db
 from ..sync.utils import get_user_agent, get_ip, get_device_id
 
 

server/mergin/auth/schemas.py

@@ -5,9 +5,8 @@
 from flask import current_app
 from marshmallow import fields
 
-from .. import ma
 from .models import User, UserProfile
-from ..app import DateTimeWithZ
+from ..app import DateTimeWithZ, ma
 
 
 class UserProfileSchema(ma.SQLAlchemyAutoSchema):

server/mergin/auth/tasks.py

@@ -6,7 +6,7 @@
 from sqlalchemy.sql.operators import isnot
 
 from ..celery import celery
-from .. import db
+from .app import db
 from .models import User
 from .config import Configuration
 

server/mergin/celery.py

@@ -9,7 +9,7 @@
 from smtplib import SMTPException, SMTPServerDisconnected
 
 from .config import Configuration
-from . import mail
+from .app import mail
 
 
 # create on flask app independent object

server/mergin/config.py

@@ -3,10 +3,9 @@
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
 
 import os
-from .version import get_version
 from decouple import config, Csv
 
-config_dir = os.path.abspath(os.path.dirname(__file__))
+from .version import get_version
 
 
 class Configuration(object):
@@ -95,10 +94,16 @@ class Configuration(object):
 
     # build hash number
     BUILD_HASH = config("BUILD_HASH", default="")
+
     # Allow changing access to admin panel
     ENABLE_SUPERADMIN_ASSIGNMENT = config(
         "ENABLE_SUPERADMIN_ASSIGNMENT", default=True, cast=bool
     )
     # backend version
     VERSION = config("VERSION", default=get_version())
     SERVER_TYPE = config("SERVER_TYPE", default="ce")
+
+    # whether to run flask app with gevent worker type in gunicorn
+    # using gevent type of worker impose some requirements on code, e.g. to be greenlet safe, custom timeouts
+    GEVENT_WORKER = config("GEVENT_WORKER", default=False, cast=bool)
+    GEVENT_REQUEST_TIMEOUT = config("GEVENT_REQUEST_TIMEOUT", default=30, cast=int)

server/mergin/stats/models.py

@@ -5,7 +5,7 @@
 import uuid
 from sqlalchemy.dialects.postgresql import UUID
 
-from .. import db
+from ..app import db
 
 
 class MerginInfo(db.Model):

server/mergin/sync/commands.py

@@ -10,7 +10,7 @@
 from flask import Flask, current_app
 
 from .files import UploadChanges
-from .. import db
+from ..app import db
 from .models import Project, ProjectAccess, ProjectVersion
 from .utils import split_project_path
 

server/mergin/sync/db_events.py

@@ -6,7 +6,7 @@
 from flask import current_app, abort
 from sqlalchemy import event
 
-from .. import db
+from ..app import db
 
 
 def check(session):

server/mergin/sync/files.py

@@ -8,8 +8,7 @@
 from marshmallow import fields, EXCLUDE, pre_load, post_load, post_dump
 from pathvalidate import sanitize_filename
 
-from .. import ma
-from ..app import DateTimeWithZ
+from ..app import DateTimeWithZ, ma
 
 
 def mergin_secure_filename(filename: str) -> str:

server/mergin/sync/models.py

@@ -29,7 +29,7 @@
     ProjectFile,
 )
 from .storages.disk import move_to_tmp
-from .. import db
+from ..app import db
 from .storages import DiskStorage
 from .utils import is_versioned_file, is_qgis
 

server/mergin/sync/private_api_controller.py

@@ -8,7 +8,7 @@
 from sqlalchemy.orm import defer
 from sqlalchemy import text, and_, desc, asc
 
-from .. import db
+from ..app import db
 from ..auth import auth_required
 from ..auth.models import User, UserProfile
 from .forms import AccessPermissionForm

server/mergin/sync/public_api_controller.py

@@ -32,7 +32,7 @@
 from gevent import sleep
 import base64
 from werkzeug.exceptions import HTTPException
-from .. import db
+from ..app import db
 from ..auth import auth_required
 from ..auth.models import User
 from .models import (

server/mergin/sync/public_api_v2_controller.py

@@ -8,7 +8,7 @@
 from flask import abort, jsonify
 from flask_login import current_user
 
-from mergin import db
+from mergin.app import db
 from mergin.auth import auth_required
 from mergin.sync.models import Project
 from mergin.sync.permissions import ProjectPermissions, require_project_by_uuid

server/mergin/sync/schemas.py

@@ -7,11 +7,10 @@
 from flask_login import current_user
 from flask import current_app
 
-from .. import ma
 from .files import ProjectFileSchema, FileSchema
 from .permissions import ProjectPermissions
 from .models import Project, ProjectVersion, AccessRequest, FileHistory, PushChangeType
-from ..app import DateTimeWithZ
+from ..app import DateTimeWithZ, ma
 from ..auth.models import User
 
 

server/mergin/sync/storages/disk.py

@@ -16,7 +16,7 @@
 from result import Err, Ok, Result
 
 from .storage import ProjectStorage, FileNotFound, InitializationError
-from ... import db
+from ...app import db
 from ..utils import (
     generate_checksum,
     is_versioned_file,

server/mergin/sync/tasks.py

@@ -13,7 +13,7 @@
 from .storages.disk import move_to_tmp
 from .config import Configuration
 from ..celery import celery
-from .. import db
+from ..app import db
 
 
 @celery.task

server/mergin/sync/workspace.py

@@ -18,7 +18,7 @@
 )
 from .permissions import projects_query, ProjectPermissions
 from .public_api_controller import parse_project_access_update_request
-from .. import db
+from ..app import db
 from ..auth.models import User
 from ..config import Configuration
 from .interfaces import AbstractWorkspace, WorkspaceHandler

server/mergin/tests/fixtures.py

@@ -12,7 +12,7 @@
 from pygeodiff import GeoDiff
 import pytest
 
-from .. import db, create_app
+from ..app import db, create_app
 from ..sync.models import Project, ProjectVersion
 from ..stats.app import register
 from ..stats.models import MerginInfo

server/mergin/tests/test_auth.py

@@ -12,7 +12,7 @@
 
 from ..auth.models import User, UserProfile, LoginHistory
 from ..auth.tasks import anonymize_removed_users
-from .. import db
+from ..app import db
 from ..sync.models import Project
 from . import (
     test_workspace_id,

server/mergin/tests/test_celery.py

@@ -8,7 +8,7 @@
 from flask_mail import Mail
 from unittest.mock import patch
 
-from .. import db
+from ..app import db
 from ..config import Configuration
 from ..sync.models import Project, AccessRequest, ProjectVersion
 from ..celery import send_email_async

server/mergin/tests/test_db_hooks.py

@@ -20,7 +20,7 @@
 )
 from ..sync.files import UploadChanges
 from ..auth.models import User
-from .. import db
+from ..app import db
 from . import DEFAULT_USER
 from .utils import add_user, create_project, create_workspace, cleanup
 

server/mergin/tests/test_file_restore.py

@@ -6,7 +6,7 @@
 import shutil
 from sqlalchemy.orm.attributes import flag_modified
 
-from .. import db
+from ..app import db
 from ..auth.models import User
 from ..sync.models import ProjectVersion, Project, GeodiffActionHistory
 from . import test_project_dir, TMP_DIR