From ae457be43dce4fc8738f6d0e4490c75762ff5929 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Mon, 25 Dec 2023 16:18:30 +0000 Subject: [PATCH 01/20] wip Fix #1276 use python 3.12 using rye & update all packages --- .python-version | 1 + pyproject.toml | 61 ++++++++++++++++++++++++++++++++++++ requirements.lock | 79 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 .python-version create mode 100644 pyproject.toml create mode 100644 requirements.lock diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..57ef42fd --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +cpython-x86_64-linux@3.12.0 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..71f62400 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,61 @@ +[project] +name = "subscribie" +version = "v0.1.181" +description = "Collect recurring payments online - subscription payments collection automation" +authors = [ + { name = "chrisjsimpson", email = "chris.j.simpson@live.co.uk" } +] +dependencies = [ + "graphviz", + "graphlib", + "wheel", + "Flask", + "flask_cors", + "Flask-Reuploaded", + "Flask-WTF", + "email-validator", + "Flask-Mail", + "requests", + "blinker", + "stripe", + "GitPython", + "pathlib", + "pytest", + "Flask-SQLAlchemy", + "Flask-Migrate", + "Flask-Babel", + "python-dotenv", + "pyjwt[crypto]", + "py-auth-header-parser", + "pydantic", + "backoff", + "coloredlogs", + "env-validate", + "python-dateutil", + "currency-symbols", + "pycryptodome", + "pycountry", + "SQLAlchemy", + "scipy", + "pandas", + "scikit-learn", + "black>=23.12.1", +] +readme = "README.md" +requires-python = ">= 3.8" + +[project.scripts] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.rye] +managed = true +dev-dependencies = [] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["./"] diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 00000000..6c54a8d1 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,79 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false + +-e file:. +alembic==1.13.1 +annotated-types==0.6.0 +babel==2.14.0 +backoff==2.2.1 +black==23.12.1 +blinker==1.7.0 +certifi==2023.11.17 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +coloredlogs==15.0.1 +cryptography==41.0.7 +currency-symbols==2.0.3 +dnspython==2.4.2 +email-validator==2.1.0.post1 +env-validate==0.5 +flask==3.0.0 +flask-babel==4.0.0 +flask-cors==4.0.0 +flask-mail==0.9.1 +flask-migrate==4.0.5 +flask-reuploaded==1.4.0 +flask-sqlalchemy==3.1.1 +flask-wtf==1.2.1 +gitdb==4.0.11 +gitpython==3.1.40 +graphlib==0.9.5 +graphviz==0.20.1 +greenlet==3.0.3 +humanfriendly==10.0 +idna==3.6 +iniconfig==2.0.0 +itsdangerous==2.1.2 +jinja2==3.1.2 +joblib==1.3.2 +mako==1.3.0 +markupsafe==2.1.3 +mypy-extensions==1.0.0 +numpy==1.26.2 +packaging==23.2 +pandas==2.1.4 +pathlib==1.0.1 +pathspec==0.12.1 +platformdirs==4.1.0 +pluggy==1.3.0 +py-auth-header-parser==1.0.2 +pycountry==23.12.11 +pycparser==2.21 +pycryptodome==3.19.0 +pydantic==2.5.3 +pydantic-core==2.14.6 +pyjwt==2.8.0 +pytest==7.4.3 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +pytz==2023.3.post1 +requests==2.31.0 +scikit-learn==1.3.2 +scipy==1.11.4 +six==1.16.0 +smmap==5.0.1 +sqlalchemy==2.0.23 +stripe==7.10.0 +threadpoolctl==3.2.0 +typing-extensions==4.9.0 +tzdata==2023.3 +urllib3==2.1.0 +werkzeug==3.0.1 +wheel==0.42.0 +wtforms==3.1.1 From 7f6822509a726bcfc50728470cc0653f5dc02278 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Mon, 25 Dec 2023 16:26:03 +0000 Subject: [PATCH 02/20] wip #1276 upgrade markupsafe, remove before_app_first_request usage, MAX_CONTEXT_LENGTH update --- subscribie/__init__.py | 73 +++++++++++--------- subscribie/auth.py | 2 +- subscribie/blueprints/admin/__init__.py | 3 +- subscribie/blueprints/pages/__init__.py | 3 +- subscribie/blueprints/subscriber/__init__.py | 2 +- subscribie/views.py | 12 +--- 6 files changed, 44 insertions(+), 51 deletions(-) diff --git a/subscribie/__init__.py b/subscribie/__init__.py index e52d239b..6049cfa3 100644 --- a/subscribie/__init__.py +++ b/subscribie/__init__.py @@ -34,13 +34,12 @@ configure_uploads, UploadSet, IMAGES, - patch_request_class, ) # noqa: E501 import importlib import urllib from pathlib import Path import sqlalchemy -from flask_migrate import Migrate +from flask_migrate import Migrate, upgrade import click from jinja2 import Template from .models import PaymentProvider, Person, Company, Module, Plan, PriceList @@ -56,17 +55,16 @@ def seed_db(): def create_app(test_config=None): app = Flask(__name__, instance_relative_config=True) - babel = Babel(app) LANGUAGES = ["en", "de", "es", "fr", "hr"] load_dotenv(verbose=True) PERMANENT_SESSION_LIFETIME = int(os.environ.pop("PERMANENT_SESSION_LIFETIME", 1800)) app.config.update(os.environ) app.config["PERMANENT_SESSION_LIFETIME"] = PERMANENT_SESSION_LIFETIME + app.config["MAX_CONTENT_LENGTH"] = int(os.getenv("MAX_CONTENT_LENGTH", 52428800)) if test_config is not None: app.config.update(test_config) - @babel.localeselector def get_locale(): language_code = session.get("language_code", None) if language_code is not None: @@ -79,6 +77,8 @@ def get_locale(): log.info(f"language_code best match set to: {language_code}") return request.accept_languages.best_match(LANGUAGES) + Babel(app, locale_selector=get_locale) + @app.before_request def start_session(): session.permanent = True @@ -88,38 +88,8 @@ def start_session(): session["sid"] = urllib.parse.quote_plus(b64encode(os.urandom(10))) log.info(f"Starting with sid {session['sid']}") - @app.before_first_request - def register_modules(): - """Import any custom modules""" - # Set custom modules path - sys.path.append(app.config["MODULES_PATH"]) - modules = Module.query.all() - log.info(f"sys.path contains: {sys.path}") - for module in modules: - # Assume standard python module - try: - log.info(f"Attempting to importing module: {module.name}") - importlib.import_module(module.name) - except ModuleNotFoundError: - log.debug(f"Error: Could not import module: {module.name}") - # Register modules as blueprint (if it is one) - try: - importedModule = importlib.import_module(module.name) - if isinstance(getattr(importedModule, module.name), Blueprint): - # Load any config the Blueprint declares - blueprint = getattr(importedModule, module.name) - blueprintConfig = "".join([blueprint.root_path, "/", "config.py"]) - app.config.from_pyfile(blueprintConfig, silent=True) - # Register the Blueprint - app.register_blueprint(getattr(importedModule, module.name)) - log.info(f"Imported {module.name} as flask Blueprint") - - except (ModuleNotFoundError, AttributeError): - log.error(f"Error: Could not import module as blueprint: {module.name}") - CORS(app) images = UploadSet("images", IMAGES) - patch_request_class(app, int(app.config.get("MAX_CONTENT_LENGTH", 2 * 1024 * 1024))) configure_uploads(app, images) from . import auth @@ -155,6 +125,41 @@ def register_modules(): database.init_app(app) Migrate(app, database) + """Migrate database when app first boots""" + log.info("Migrating database") + upgrade( + directory=Path( + current_app.config["SUBSCRIBIE_REPO_DIRECTORY"] + "/migrations" + ) + ) + + """Import any custom modules""" + # Set custom modules path + sys.path.append(app.config["MODULES_PATH"]) + modules = Module.query.all() + log.info(f"sys.path contains: {sys.path}") + for module in modules: + # Assume standard python module + try: + log.info(f"Attempting to importing module: {module.name}") + importlib.import_module(module.name) + except ModuleNotFoundError: + log.debug(f"Error: Could not import module: {module.name}") + # Register modules as blueprint (if it is one) + try: + importedModule = importlib.import_module(module.name) + if isinstance(getattr(importedModule, module.name), Blueprint): + # Load any config the Blueprint declares + blueprint = getattr(importedModule, module.name) + blueprintConfig = "".join([blueprint.root_path, "/", "config.py"]) + app.config.from_pyfile(blueprintConfig, silent=True) + # Register the Blueprint + app.register_blueprint(getattr(importedModule, module.name)) + log.info(f"Imported {module.name} as flask Blueprint") + + except (ModuleNotFoundError, AttributeError): + log.error(f"Error: Could not import module as blueprint: {module.name}") + try: payment_provider = PaymentProvider.query.first() if payment_provider is None: diff --git a/subscribie/auth.py b/subscribie/auth.py index f4d2989f..b6f0adf6 100644 --- a/subscribie/auth.py +++ b/subscribie/auth.py @@ -12,8 +12,8 @@ url_for, current_app, render_template_string, - Markup, ) +from markupsafe import Markup from subscribie.email import EmailMessageQueue from subscribie.utils import get_stripe_secret_key, get_stripe_connect_account from base64 import urlsafe_b64encode diff --git a/subscribie/blueprints/admin/__init__.py b/subscribie/blueprints/admin/__init__.py index 0126be6f..428d4873 100644 --- a/subscribie/blueprints/admin/__init__.py +++ b/subscribie/blueprints/admin/__init__.py @@ -15,9 +15,8 @@ request, session, Response, - Markup, - escape, ) +from markupsafe import Markup, escape import jinja2 import requests from subscribie.utils import ( diff --git a/subscribie/blueprints/pages/__init__.py b/subscribie/blueprints/pages/__init__.py index e1701f72..59ae94f3 100644 --- a/subscribie/blueprints/pages/__init__.py +++ b/subscribie/blueprints/pages/__init__.py @@ -7,9 +7,8 @@ current_app, url_for, flash, - Markup, - escape, ) +from markupsafe import Markup, escape from subscribie.auth import login_required from subscribie.models import database, Page from pathlib import Path diff --git a/subscribie/blueprints/subscriber/__init__.py b/subscribie/blueprints/subscriber/__init__.py index 12f433ca..2512faf9 100644 --- a/subscribie/blueprints/subscriber/__init__.py +++ b/subscribie/blueprints/subscriber/__init__.py @@ -14,8 +14,8 @@ g, current_app, request, - Markup, ) +from markupsafe import Markup from subscribie.forms import ( PasswordLoginForm, SubscriberForgotPasswordForm, diff --git a/subscribie/views.py b/subscribie/views.py index d56c1c5e..db2a4412 100644 --- a/subscribie/views.py +++ b/subscribie/views.py @@ -14,10 +14,9 @@ current_app, g, send_from_directory, - Markup, ) +from markupsafe import Markup from .models import Company, Plan, Integration, Page, Category, Setting, PaymentProvider -from flask_migrate import upgrade from subscribie.blueprints.style import inject_custom_style from subscribie.database import database from subscribie.signals import register_signal_handlers @@ -45,15 +44,6 @@ register_signal_handlers() -@bp.before_app_first_request -def migrate_database(): - """Migrate database when app first boots""" - log.info("Migrating database") - upgrade( - directory=Path(current_app.config["SUBSCRIBIE_REPO_DIRECTORY"] + "/migrations") - ) - - @bp.before_app_request def on_each_request(): # Detect country code if present in the request from proxy From 7a6298e99a655ee59567c4b26f83e31e19587a75 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Mon, 25 Dec 2023 16:18:30 +0000 Subject: [PATCH 03/20] wip Fix #1276 use python 3.12 using rye & update all packages --- .python-version | 1 + pyproject.toml | 61 ++++++++++++++++++++++++++++++++++++ requirements.lock | 79 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 .python-version create mode 100644 pyproject.toml create mode 100644 requirements.lock diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..57ef42fd --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +cpython-x86_64-linux@3.12.0 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..71f62400 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,61 @@ +[project] +name = "subscribie" +version = "v0.1.181" +description = "Collect recurring payments online - subscription payments collection automation" +authors = [ + { name = "chrisjsimpson", email = "chris.j.simpson@live.co.uk" } +] +dependencies = [ + "graphviz", + "graphlib", + "wheel", + "Flask", + "flask_cors", + "Flask-Reuploaded", + "Flask-WTF", + "email-validator", + "Flask-Mail", + "requests", + "blinker", + "stripe", + "GitPython", + "pathlib", + "pytest", + "Flask-SQLAlchemy", + "Flask-Migrate", + "Flask-Babel", + "python-dotenv", + "pyjwt[crypto]", + "py-auth-header-parser", + "pydantic", + "backoff", + "coloredlogs", + "env-validate", + "python-dateutil", + "currency-symbols", + "pycryptodome", + "pycountry", + "SQLAlchemy", + "scipy", + "pandas", + "scikit-learn", + "black>=23.12.1", +] +readme = "README.md" +requires-python = ">= 3.8" + +[project.scripts] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.rye] +managed = true +dev-dependencies = [] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["./"] diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 00000000..6c54a8d1 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,79 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false + +-e file:. +alembic==1.13.1 +annotated-types==0.6.0 +babel==2.14.0 +backoff==2.2.1 +black==23.12.1 +blinker==1.7.0 +certifi==2023.11.17 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +coloredlogs==15.0.1 +cryptography==41.0.7 +currency-symbols==2.0.3 +dnspython==2.4.2 +email-validator==2.1.0.post1 +env-validate==0.5 +flask==3.0.0 +flask-babel==4.0.0 +flask-cors==4.0.0 +flask-mail==0.9.1 +flask-migrate==4.0.5 +flask-reuploaded==1.4.0 +flask-sqlalchemy==3.1.1 +flask-wtf==1.2.1 +gitdb==4.0.11 +gitpython==3.1.40 +graphlib==0.9.5 +graphviz==0.20.1 +greenlet==3.0.3 +humanfriendly==10.0 +idna==3.6 +iniconfig==2.0.0 +itsdangerous==2.1.2 +jinja2==3.1.2 +joblib==1.3.2 +mako==1.3.0 +markupsafe==2.1.3 +mypy-extensions==1.0.0 +numpy==1.26.2 +packaging==23.2 +pandas==2.1.4 +pathlib==1.0.1 +pathspec==0.12.1 +platformdirs==4.1.0 +pluggy==1.3.0 +py-auth-header-parser==1.0.2 +pycountry==23.12.11 +pycparser==2.21 +pycryptodome==3.19.0 +pydantic==2.5.3 +pydantic-core==2.14.6 +pyjwt==2.8.0 +pytest==7.4.3 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +pytz==2023.3.post1 +requests==2.31.0 +scikit-learn==1.3.2 +scipy==1.11.4 +six==1.16.0 +smmap==5.0.1 +sqlalchemy==2.0.23 +stripe==7.10.0 +threadpoolctl==3.2.0 +typing-extensions==4.9.0 +tzdata==2023.3 +urllib3==2.1.0 +werkzeug==3.0.1 +wheel==0.42.0 +wtforms==3.1.1 From 801a427259e283eb74e4a9fa5db4e0b123a767a6 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Mon, 25 Dec 2023 16:26:03 +0000 Subject: [PATCH 04/20] wip #1276 upgrade markupsafe, remove before_app_first_request usage, MAX_CONTEXT_LENGTH update --- subscribie/__init__.py | 73 +++++++++++--------- subscribie/auth.py | 2 +- subscribie/blueprints/admin/__init__.py | 3 +- subscribie/blueprints/pages/__init__.py | 3 +- subscribie/blueprints/subscriber/__init__.py | 2 +- subscribie/views.py | 12 +--- 6 files changed, 44 insertions(+), 51 deletions(-) diff --git a/subscribie/__init__.py b/subscribie/__init__.py index e52d239b..6049cfa3 100644 --- a/subscribie/__init__.py +++ b/subscribie/__init__.py @@ -34,13 +34,12 @@ configure_uploads, UploadSet, IMAGES, - patch_request_class, ) # noqa: E501 import importlib import urllib from pathlib import Path import sqlalchemy -from flask_migrate import Migrate +from flask_migrate import Migrate, upgrade import click from jinja2 import Template from .models import PaymentProvider, Person, Company, Module, Plan, PriceList @@ -56,17 +55,16 @@ def seed_db(): def create_app(test_config=None): app = Flask(__name__, instance_relative_config=True) - babel = Babel(app) LANGUAGES = ["en", "de", "es", "fr", "hr"] load_dotenv(verbose=True) PERMANENT_SESSION_LIFETIME = int(os.environ.pop("PERMANENT_SESSION_LIFETIME", 1800)) app.config.update(os.environ) app.config["PERMANENT_SESSION_LIFETIME"] = PERMANENT_SESSION_LIFETIME + app.config["MAX_CONTENT_LENGTH"] = int(os.getenv("MAX_CONTENT_LENGTH", 52428800)) if test_config is not None: app.config.update(test_config) - @babel.localeselector def get_locale(): language_code = session.get("language_code", None) if language_code is not None: @@ -79,6 +77,8 @@ def get_locale(): log.info(f"language_code best match set to: {language_code}") return request.accept_languages.best_match(LANGUAGES) + Babel(app, locale_selector=get_locale) + @app.before_request def start_session(): session.permanent = True @@ -88,38 +88,8 @@ def start_session(): session["sid"] = urllib.parse.quote_plus(b64encode(os.urandom(10))) log.info(f"Starting with sid {session['sid']}") - @app.before_first_request - def register_modules(): - """Import any custom modules""" - # Set custom modules path - sys.path.append(app.config["MODULES_PATH"]) - modules = Module.query.all() - log.info(f"sys.path contains: {sys.path}") - for module in modules: - # Assume standard python module - try: - log.info(f"Attempting to importing module: {module.name}") - importlib.import_module(module.name) - except ModuleNotFoundError: - log.debug(f"Error: Could not import module: {module.name}") - # Register modules as blueprint (if it is one) - try: - importedModule = importlib.import_module(module.name) - if isinstance(getattr(importedModule, module.name), Blueprint): - # Load any config the Blueprint declares - blueprint = getattr(importedModule, module.name) - blueprintConfig = "".join([blueprint.root_path, "/", "config.py"]) - app.config.from_pyfile(blueprintConfig, silent=True) - # Register the Blueprint - app.register_blueprint(getattr(importedModule, module.name)) - log.info(f"Imported {module.name} as flask Blueprint") - - except (ModuleNotFoundError, AttributeError): - log.error(f"Error: Could not import module as blueprint: {module.name}") - CORS(app) images = UploadSet("images", IMAGES) - patch_request_class(app, int(app.config.get("MAX_CONTENT_LENGTH", 2 * 1024 * 1024))) configure_uploads(app, images) from . import auth @@ -155,6 +125,41 @@ def register_modules(): database.init_app(app) Migrate(app, database) + """Migrate database when app first boots""" + log.info("Migrating database") + upgrade( + directory=Path( + current_app.config["SUBSCRIBIE_REPO_DIRECTORY"] + "/migrations" + ) + ) + + """Import any custom modules""" + # Set custom modules path + sys.path.append(app.config["MODULES_PATH"]) + modules = Module.query.all() + log.info(f"sys.path contains: {sys.path}") + for module in modules: + # Assume standard python module + try: + log.info(f"Attempting to importing module: {module.name}") + importlib.import_module(module.name) + except ModuleNotFoundError: + log.debug(f"Error: Could not import module: {module.name}") + # Register modules as blueprint (if it is one) + try: + importedModule = importlib.import_module(module.name) + if isinstance(getattr(importedModule, module.name), Blueprint): + # Load any config the Blueprint declares + blueprint = getattr(importedModule, module.name) + blueprintConfig = "".join([blueprint.root_path, "/", "config.py"]) + app.config.from_pyfile(blueprintConfig, silent=True) + # Register the Blueprint + app.register_blueprint(getattr(importedModule, module.name)) + log.info(f"Imported {module.name} as flask Blueprint") + + except (ModuleNotFoundError, AttributeError): + log.error(f"Error: Could not import module as blueprint: {module.name}") + try: payment_provider = PaymentProvider.query.first() if payment_provider is None: diff --git a/subscribie/auth.py b/subscribie/auth.py index f4d2989f..b6f0adf6 100644 --- a/subscribie/auth.py +++ b/subscribie/auth.py @@ -12,8 +12,8 @@ url_for, current_app, render_template_string, - Markup, ) +from markupsafe import Markup from subscribie.email import EmailMessageQueue from subscribie.utils import get_stripe_secret_key, get_stripe_connect_account from base64 import urlsafe_b64encode diff --git a/subscribie/blueprints/admin/__init__.py b/subscribie/blueprints/admin/__init__.py index 0126be6f..428d4873 100644 --- a/subscribie/blueprints/admin/__init__.py +++ b/subscribie/blueprints/admin/__init__.py @@ -15,9 +15,8 @@ request, session, Response, - Markup, - escape, ) +from markupsafe import Markup, escape import jinja2 import requests from subscribie.utils import ( diff --git a/subscribie/blueprints/pages/__init__.py b/subscribie/blueprints/pages/__init__.py index e1701f72..59ae94f3 100644 --- a/subscribie/blueprints/pages/__init__.py +++ b/subscribie/blueprints/pages/__init__.py @@ -7,9 +7,8 @@ current_app, url_for, flash, - Markup, - escape, ) +from markupsafe import Markup, escape from subscribie.auth import login_required from subscribie.models import database, Page from pathlib import Path diff --git a/subscribie/blueprints/subscriber/__init__.py b/subscribie/blueprints/subscriber/__init__.py index 12f433ca..2512faf9 100644 --- a/subscribie/blueprints/subscriber/__init__.py +++ b/subscribie/blueprints/subscriber/__init__.py @@ -14,8 +14,8 @@ g, current_app, request, - Markup, ) +from markupsafe import Markup from subscribie.forms import ( PasswordLoginForm, SubscriberForgotPasswordForm, diff --git a/subscribie/views.py b/subscribie/views.py index d56c1c5e..db2a4412 100644 --- a/subscribie/views.py +++ b/subscribie/views.py @@ -14,10 +14,9 @@ current_app, g, send_from_directory, - Markup, ) +from markupsafe import Markup from .models import Company, Plan, Integration, Page, Category, Setting, PaymentProvider -from flask_migrate import upgrade from subscribie.blueprints.style import inject_custom_style from subscribie.database import database from subscribie.signals import register_signal_handlers @@ -45,15 +44,6 @@ register_signal_handlers() -@bp.before_app_first_request -def migrate_database(): - """Migrate database when app first boots""" - log.info("Migrating database") - upgrade( - directory=Path(current_app.config["SUBSCRIBIE_REPO_DIRECTORY"] + "/migrations") - ) - - @bp.before_app_request def on_each_request(): # Detect country code if present in the request from proxy From 40f5c2dd253899c6e969dd1d630317a8ad78580b Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Mon, 1 Jan 2024 17:08:09 +0000 Subject: [PATCH 05/20] #1276 set vassal virtualenv venv path to .venv (rye default) --- vassals-inject-config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vassals-inject-config.ini b/vassals-inject-config.ini index 1af37cf7..e6b6c769 100644 --- a/vassals-inject-config.ini +++ b/vassals-inject-config.ini @@ -27,7 +27,7 @@ cron2 = minute=-1440 curl -L %(vassal_name)\/admin\/refresh-invoices # %d absolute path of the directory containing the configuration file chdir = %(vassal_dir) -virtualenv = /path-to-shared-subscribie-python/venv/ +virtualenv = /path-to-shared-subscribie-python/.venv/ wsgi-file = /path-to/subscribie.wsgi From a2ae12f93b872e45036a24ffa68f60e8366e1a85 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Mon, 1 Jan 2024 17:08:09 +0000 Subject: [PATCH 06/20] #1276 set vassal virtualenv venv path to .venv (rye default) --- vassals-inject-config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vassals-inject-config.ini b/vassals-inject-config.ini index 1af37cf7..e6b6c769 100644 --- a/vassals-inject-config.ini +++ b/vassals-inject-config.ini @@ -27,7 +27,7 @@ cron2 = minute=-1440 curl -L %(vassal_name)\/admin\/refresh-invoices # %d absolute path of the directory containing the configuration file chdir = %(vassal_dir) -virtualenv = /path-to-shared-subscribie-python/venv/ +virtualenv = /path-to-shared-subscribie-python/.venv/ wsgi-file = /path-to/subscribie.wsgi From e7633d9b41355819b67d444fc17746fcc69d3168 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Sat, 6 Jan 2024 22:56:56 +0000 Subject: [PATCH 07/20] Wip Fix #1039 use strictyaml for app settings --- .gitignore | 2 + pyproject.toml | 3 +- requirements.lock | 1 + settings.yaml.example | 83 +++++++++++++++++++ subscribie/__init__.py | 36 ++++---- .../anti_spam_subscribie_shop_names/run.py | 4 +- subscribie/api.py | 9 +- subscribie/auth.py | 6 +- subscribie/blueprints/admin/__init__.py | 24 +++--- subscribie/blueprints/admin/priceList.py | 7 +- subscribie/blueprints/admin/priceListRule.py | 6 +- subscribie/blueprints/subscriber/__init__.py | 3 +- subscribie/logger.py | 14 ++-- subscribie/models.py | 6 +- subscribie/receivers.py | 2 - subscribie/settings.py | 64 ++++++++++++++ 16 files changed, 206 insertions(+), 64 deletions(-) create mode 100644 settings.yaml.example create mode 100644 subscribie/settings.py diff --git a/.gitignore b/.gitignore index 46fa21fc..87413bab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +settings.yaml +settings.yml venv/ *.sql *.swp diff --git a/pyproject.toml b/pyproject.toml index 71f62400..9ed2cae4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,8 @@ dependencies = [ "scipy", "pandas", "scikit-learn", - "black>=23.12.1", + "black", + "strictyaml", ] readme = "README.md" requires-python = ">= 3.8" diff --git a/requirements.lock b/requirements.lock index 6c54a8d1..fd482c9d 100644 --- a/requirements.lock +++ b/requirements.lock @@ -69,6 +69,7 @@ scipy==1.11.4 six==1.16.0 smmap==5.0.1 sqlalchemy==2.0.23 +strictyaml==1.7.3 stripe==7.10.0 threadpoolctl==3.2.0 typing-extensions==4.9.0 diff --git a/settings.yaml.example b/settings.yaml.example new file mode 100644 index 00000000..2f418867 --- /dev/null +++ b/settings.yaml.example @@ -0,0 +1,83 @@ +--- +# Change FLASK_ENV: live for live +FLASK_ENV: development +# Cookie policies +# SESSION_COOKIE_SECURE: True +# SESSION_COOKIE_HTTPONLY: True +# SESSION_COOKIE_SAMESITE: None + +# Software as a service (SAAS) +SAAS_URL: https://subscribie.co.uk/ +# SAAS_API_KEY is to allow subscribie platform to send authenticated +# api requests to subscribie shops created by the shop builder. +SAAS_API_KEY: changeme +SAAS_ACTIVATE_ACCOUNT_PATH: /activate + +# For testing this repo in isolation, SUBSCRIBIE_REPO_DIRECTORY can be './' +# for production, SUBSCRIBIE_REPO_DIRECTORY should be wherever the repo +# is cloned to +SUBSCRIBIE_REPO_DIRECTORY: ./ +SQLALCHEMY_TRACK_MODIFICATIONS: False +SQLALCHEMY_DATABASE_URI: "sqlite:////tmp/data.db" +SECRET_KEY: "random string. e.g. echo -e 'from os import urandom\\nprint urandom(25)' | python" +DB_FULL_PATH: "/tmp/data.db" +MODULES_PATH: "./modules/" +TEMPLATE_BASE_DIR: "./subscribie/themes/" +THEME_NAME: "jesmond" +CUSTOM_PAGES_PATH: "./subscribie/custom_pages/" +UPLOADED_IMAGES_DEST: "./subscribie/static/" +UPLOADED_FILES_DEST: "./subscribie/uploads/" +# Default 50Mb upload limit +MAX_CONTENT_LENGTH: "52428800" +SUCCESS_REDIRECT_URL: "http://127.0.0.1:5000/complete_mandate" +THANKYOU_URL: "http://127.0.0.1:5000/thankyou" +EMAIL_LOGIN_FROM: "hello@example.com" +EMAIL_QUEUE_FOLDER: "/var/email-queue/" +SERVER_NAME: "127.0.0.1:5000" + + +PERMANENT_SESSION_LIFETIME: "1800" + +MAIL_DEFAULT_SENDER: "noreply@example.com" + +STRIPE_LIVE_PUBLISHABLE_KEY: "pk_live_changeme" +STRIPE_LIVE_SECRET_KEY: "sk_live_changeme" + +STRIPE_TEST_PUBLISHABLE_KEY: "pk_test_changeme" +STRIPE_TEST_SECRET_KEY: "sk_test_changeme" + +# Internal server where shop should send its stripe connect account id to. See https://github.com/Subscribie/subscribie/issues/352 +STRIPE_CONNECT_ACCOUNT_ANNOUNCER_HOST: http://127.0.0.1:8001 + +# For development: + +PYTHON_LOG_LEVEL: DEBUG + +PLAYWRIGHT_HOST: http://127.0.0.1:5000/ +PLAYWRIGHT_HEADLESS: true + +#rename shop variables +PATH_TO_SITES: "/path/to/sites/subscribie/" +PATH_TO_RENAME_SCRIPT: "/path/to/sites/subscribie/rename-shop.sh" +SUBSCRIBIE_DOMAIN: "subscriby.shop" + +PRIVATE_KEY: "/tmp/private.pem" +PUBLIC_KEY: "/tmp/public.pem" + +# Currencies +SUPPORTED_CURRENCIES: "GBP,USD,EUR" + +# Anti spam +ANTI_SPAM_SHOP_NAMES_MODEL_FULL_PATH: "/path/to/classifier.pkl" + + +# Optional +TELEGRAM_TOKEN: +TELEGRAM_CHAT_ID: +TELEGRAM_PYTHON_LOG_LEVEL: ERROR + + +# Environment Settings for tests +TEST_SHOP_OWNER_EMAIL_ISSUE_704: admin@example.com +TEST_SHOP_OWNER_LOGIN_URL: http://127.0.0.1:5000/auth/login + diff --git a/subscribie/__init__.py b/subscribie/__init__.py index 6049cfa3..7ef9edad 100644 --- a/subscribie/__init__.py +++ b/subscribie/__init__.py @@ -7,8 +7,7 @@ :copyright: (c) 2018 by Karma Computing Ltd """ -from dotenv import load_dotenv - +from subscribie.settings import settings from .logger import logger # noqa: F401 import logging import os @@ -44,8 +43,6 @@ from jinja2 import Template from .models import PaymentProvider, Person, Company, Module, Plan, PriceList -load_dotenv(verbose=True) - log = logging.getLogger(__name__) @@ -55,12 +52,12 @@ def seed_db(): def create_app(test_config=None): app = Flask(__name__, instance_relative_config=True) + app.config.update(settings) LANGUAGES = ["en", "de", "es", "fr", "hr"] - load_dotenv(verbose=True) PERMANENT_SESSION_LIFETIME = int(os.environ.pop("PERMANENT_SESSION_LIFETIME", 1800)) app.config.update(os.environ) app.config["PERMANENT_SESSION_LIFETIME"] = PERMANENT_SESSION_LIFETIME - app.config["MAX_CONTENT_LENGTH"] = int(os.getenv("MAX_CONTENT_LENGTH", 52428800)) + app.config["MAX_CONTENT_LENGTH"] = int(settings.get("MAX_CONTENT_LENGTH", 52428800)) if test_config is not None: app.config.update(test_config) @@ -180,21 +177,18 @@ def start_session(): # therefore needs its inital PriceLists created if len(price_lists) == 0: # Create defaul PriceList with zero rules for each suported currency - if os.getenv("SUPPORTED_CURRENCIES", False) is not False: - for currency in os.getenv("SUPPORTED_CURRENCIES").split(","): - log.debug( - f"Creating PriceList with zero rules for currency {currency}" # noqa: E501 - ) - price_list = PriceList( - currency=currency, name=f"Default {currency}" - ) # noqa: E501 - database.session.add(price_list) - database.session.commit() - log.debug( - f"Created PriceList with zero rules for currency {currency}" - ) - else: - log.debug("SUPPORTED_CURRENCIES is not set") + for currency in settings.get("SUPPORTED_CURRENCIES"): + log.debug( + f"Creating PriceList with zero rules for currency {currency}" # noqa: E501 + ) + price_list = PriceList( + currency=currency, name=f"Default {currency}" + ) # noqa: E501 + database.session.add(price_list) + database.session.commit() + log.debug( + f"Created PriceList with zero rules for currency {currency}" + ) # Ensure every plan has a PriceList attached for each supported currency plans = Plan.query.all() price_lists = ( diff --git a/subscribie/anti_spam_subscribie_shop_names/run.py b/subscribie/anti_spam_subscribie_shop_names/run.py index 0e98ae56..da28844c 100644 --- a/subscribie/anti_spam_subscribie_shop_names/run.py +++ b/subscribie/anti_spam_subscribie_shop_names/run.py @@ -1,9 +1,9 @@ import pickle from scipy.sparse import hstack from flask import Flask -import os +from subscribie.settings import settings -ANTI_SPAM_SHOP_NAMES_MODEL_FULL_PATH = os.getenv( +ANTI_SPAM_SHOP_NAMES_MODEL_FULL_PATH = settings.get( "ANTI_SPAM_SHOP_NAMES_MODEL_FULL_PATH", "./classifier.pkl" ) diff --git a/subscribie/api.py b/subscribie/api.py index 36313882..c77bc36b 100644 --- a/subscribie/api.py +++ b/subscribie/api.py @@ -5,11 +5,10 @@ from .auth import generate_login_url from .auth import get_magic_login_link import pydantic -from subscribie import schemas, database +from subscribie import settings, schemas, database import json import secrets from Crypto.Cipher import AES -import os import base64 log = logging.getLogger(__name__) @@ -22,7 +21,7 @@ def shop_name_taken(shop_name): from builder import Shop shop_name = shop_name.replace(" ", "").lower() - shop_name = f"https://{shop_name}.{os.getenv('SUBSCRIBIE_DOMAIN')}" + shop_name = f"https://{shop_name}.{settings.get('SUBSCRIBIE_DOMAIN')}" lookup = database.session.query(Shop).where(Shop.site_url == shop_name).all() log.debug(f"Running shop_name_taken lookup for: {lookup}") if len(lookup) == 0: @@ -56,7 +55,7 @@ def get_login_link(): return {"msg": "Could not generate login link"} -def encrypt_secret(data=None, key=os.getenv("SECRET_KEY")): +def encrypt_secret(data=None, key=settings.get("SECRET_KEY")): assert data is not None data = data.encode("utf-8") key = key[:16].encode("utf-8") @@ -66,7 +65,7 @@ def encrypt_secret(data=None, key=os.getenv("SECRET_KEY")): return nonce, ciphertext, tag -def decrypt_secret(data, key=os.getenv("SECRET_KEY")): +def decrypt_secret(data, key=settings.get("SECRET_KEY")): nonce = base64.b64decode(data.split(":")[0]) ciphertext = base64.b64decode(data.split(":")[1]) tag = base64.b64decode(data.split(":")[2]) diff --git a/subscribie/auth.py b/subscribie/auth.py index b6f0adf6..dd214d5b 100644 --- a/subscribie/auth.py +++ b/subscribie/auth.py @@ -1,6 +1,6 @@ import logging import functools - +from subscribie.settings import settings from flask import ( Blueprint, flash, @@ -544,9 +544,9 @@ def wrapped_view(**kwargs): def development_mode_only(view): @functools.wraps(view) def wrapped_view(**kwargs): - if os.getenv("FLASK_ENV") != "development": + if settings.get("FLASK_ENV") != "development": msg = { - "msg": f"Error. Only possible in development mode. Current FLASK_ENV mode is: {os.getenv('FLASK_ENV')}" # noqa: E501 + "msg": f"Error. Only possible in development mode. Current FLASK_ENV mode is: {settings.get('FLASK_ENV')}" # noqa: E501 } return jsonify(msg), 403 return view(**kwargs) diff --git a/subscribie/blueprints/admin/__init__.py b/subscribie/blueprints/admin/__init__.py index 428d4873..6da8eee8 100644 --- a/subscribie/blueprints/admin/__init__.py +++ b/subscribie/blueprints/admin/__init__.py @@ -1,7 +1,7 @@ import logging import threading import json -from dotenv import load_dotenv +from subscribie import settings from subscribie.database import database # noqa from flask import ( Blueprint, @@ -105,8 +105,6 @@ from .priceList import list_priceLists # noqa: F401 from .priceListRule import list_priceListRules # noqa: F401 -load_dotenv(verbose=True) # get environment variables from .env - def dec2pence(amount): """Take two decimal place string and convert to pence""" @@ -946,7 +944,7 @@ def set_stripe_livemode(): def stripe_connect(): setting = Setting.query.first() shop_activated = setting.shop_activated - SERVER_NAME = os.getenv("SERVER_NAME") + SERVER_NAME = settings.get("SERVER_NAME") saas_url = current_app.config.get("SAAS_URL") saas_activate_account_path = current_app.config.get("SAAS_ACTIVATE_ACCOUNT_PATH") saas_activate_account_url = ( @@ -1661,17 +1659,17 @@ def set_reply_to_email(): @admin.route("/rename-shop", methods=["GET"]) @login_required def rename_shop(): - SERVER_NAME = os.getenv("SERVER_NAME") + SERVER_NAME = settings.get("SERVER_NAME") return render_template("admin/settings/rename_shop.html", SERVER_NAME=SERVER_NAME) @admin.route("/rename-shop", methods=["POST"]) @login_required def rename_shop_post(): - PATH_TO_RENAME_SCRIPT = os.getenv("PATH_TO_RENAME_SCRIPT", False) - PATH_TO_SITES = os.getenv("PATH_TO_SITES", False) - SUBSCRIBIE_DOMAIN = os.getenv("SUBSCRIBIE_DOMAIN", False) - SERVER_NAME = os.getenv("SERVER_NAME") + PATH_TO_RENAME_SCRIPT = settings.get("PATH_TO_RENAME_SCRIPT", False) + PATH_TO_SITES = settings.get("PATH_TO_SITES", False) + SUBSCRIBIE_DOMAIN = settings.get("SUBSCRIBIE_DOMAIN", False) + SERVER_NAME = settings.get("SERVER_NAME") new_name = request.json["new_name"].replace(SUBSCRIBIE_DOMAIN, "").replace(".", "") NEW_DOMAIN = secure_filename(new_name + "." + SUBSCRIBIE_DOMAIN) @@ -1680,8 +1678,8 @@ def rename_shop_post(): if new_name.isalnum() is False: return {"msg": "Shop name can only contain letters and numbers"} - base_path = os.getenv("PATH_TO_SITES") - new_path = Path(os.getenv("PATH_TO_SITES") + f"{NEW_DOMAIN}") + base_path = settings.get("PATH_TO_SITES") + new_path = Path(settings.get("PATH_TO_SITES") + f"{NEW_DOMAIN}") if not new_path.startswith(base_path): log.error("Invalid path for shop rename") @@ -1691,7 +1689,7 @@ def rename_shop_post(): msg = "This name already exists" flash(msg) log.debug( - f"Attempt to rename site {NEW_DOMAIN} but dir {os.getenv('PATH_TO_SITES')} already exists" # noqa + f"Attempt to rename site {NEW_DOMAIN} but dir {settings.get('PATH_TO_SITES')} already exists" # noqa ) return {"msg": msg} else: @@ -1948,7 +1946,7 @@ def change_thank_you_url(): if request.form.get("default"): settings.custom_thank_you_url = None database.session.commit() - flash(f"Custom thank you url changed to default") + flash("Custom thank you url changed to default") return render_template( "admin/settings/custom_thank_you_page.html", form=form, diff --git a/subscribie/blueprints/admin/priceList.py b/subscribie/blueprints/admin/priceList.py index e1854ed8..f7395972 100644 --- a/subscribie/blueprints/admin/priceList.py +++ b/subscribie/blueprints/admin/priceList.py @@ -1,16 +1,17 @@ from . import admin +from subscribie.settings import settings from subscribie.auth import login_required from subscribie.models import PriceList, PriceListRule from subscribie.database import database from flask import render_template, request, url_for, redirect, flash import logging -import os + log = logging.getLogger(__name__) -if os.getenv("SUPPORTED_CURRENCIES", False) is not False: +if settings.get("SUPPORTED_CURRENCIES", False) is not False: supported_currencies = [] - for currency in os.getenv("SUPPORTED_CURRENCIES").split(","): + for currency in settings.get("SUPPORTED_CURRENCIES"): supported_currencies.append(currency) diff --git a/subscribie/blueprints/admin/priceListRule.py b/subscribie/blueprints/admin/priceListRule.py index 21f75b11..b360cb50 100644 --- a/subscribie/blueprints/admin/priceListRule.py +++ b/subscribie/blueprints/admin/priceListRule.py @@ -1,18 +1,18 @@ from . import admin +from subscribie.settings import settings from subscribie.auth import login_required from subscribie.models import PriceListRule from subscribie.database import database from flask import render_template, request, redirect, url_for, flash import logging -import os from datetime import datetime dog = 1 log = logging.getLogger(__name__) -if os.getenv("SUPPORTED_CURRENCIES", False) is not False: +if settings.get("SUPPORTED_CURRENCIES", False) is not False: supported_currencies = [] - for currency in os.getenv("SUPPORTED_CURRENCIES").split(","): + for currency in settings.get("SUPPORTED_CURRENCIES"): supported_currencies.append(currency) diff --git a/subscribie/blueprints/subscriber/__init__.py b/subscribie/blueprints/subscriber/__init__.py index 2512faf9..f51e231f 100644 --- a/subscribie/blueprints/subscriber/__init__.py +++ b/subscribie/blueprints/subscriber/__init__.py @@ -3,6 +3,7 @@ import functools import binascii import os +from subscribie.settings import settings from pathlib import Path from flask import ( Blueprint, @@ -145,7 +146,7 @@ def forgot_password(): ) ) company = Company.query.first() - FLASK_ENV = os.getenv("FLASK_ENV") + FLASK_ENV = settings.get("FLASK_ENV") if FLASK_ENV == "development": password_reset_url = ( "http://" + request.host + "/account/password-reset?token=" + token diff --git a/subscribie/logger.py b/subscribie/logger.py index 32fe659f..248149a5 100644 --- a/subscribie/logger.py +++ b/subscribie/logger.py @@ -1,17 +1,15 @@ import logging import coloredlogs +from subscribie.settings import settings from flask import has_request_context, request from logging.handlers import QueueHandler, QueueListener from subscribie.TelegramHTTPHandler import TelegramHTTPHandler -import os import sys import queue -from dotenv import load_dotenv -load_dotenv(verbose=True) -PYTHON_LOG_LEVEL = os.getenv("PYTHON_LOG_LEVEL", "DEBUG") -TELEGRAM_PYTHON_LOG_LEVEL = os.getenv("TELEGRAM_PYTHON_LOG_LEVEL", "ERROR") +PYTHON_LOG_LEVEL = settings.get("PYTHON_LOG_LEVEL", "DEBUG") +TELEGRAM_PYTHON_LOG_LEVEL = settings.get("TELEGRAM_PYTHON_LOG_LEVEL", "ERROR") logger = logging.getLogger() handler = logging.StreamHandler() # sys.stderr will be used by default @@ -52,13 +50,13 @@ def handle_exception(exc_type, exc_value, exc_traceback): # Telegram logging -if os.getenv("FLASK_ENV", None) != "development": +if settings.get("FLASK_ENV", None) != "development": # See https://docs.python.org/3/howto/logging-cookbook.html#dealing-with-handlers-that-block # noqa que = queue.Queue(-1) # no limit on size queue_handler = QueueHandler(que) - telegram_token = os.getenv("TELEGRAM_TOKEN", None) - telegram_chat_id = os.getenv("TELEGRAM_CHAT_ID", None) + telegram_token = settings.get("TELEGRAM_TOKEN", None) + telegram_chat_id = settings.get("TELEGRAM_CHAT_ID", None) telegramHandlerHost = "api.telegram.org" telegramHandlerUrl = ( diff --git a/subscribie/models.py b/subscribie/models.py index 5bb9a61a..0bea90f0 100644 --- a/subscribie/models.py +++ b/subscribie/models.py @@ -775,8 +775,10 @@ def calculatePrice( if rule.percent_discount: """ If I want a minimum sell price - Then I want a percent discount to apply if the value is equal to or greater than the minimum sell price - Otherwise, always apply the percent discount regardlrss of sell price. + Then I want a percent discount to apply if the value is + equal to or greater than the minimum sell price. + Otherwise, always apply the percent discount regardlrss + of sell price. """ if ( rule.has_min_sell_price diff --git a/subscribie/receivers.py b/subscribie/receivers.py index 773eced0..4be254df 100644 --- a/subscribie/receivers.py +++ b/subscribie/receivers.py @@ -7,10 +7,8 @@ from subscribie.notifications import subscriberPaymentFailedNotification from subscribie.models import Subscription, Document from subscribie.database import database -from dotenv import load_dotenv import sqlalchemy -load_dotenv(verbose=True) log = logging.getLogger(__name__) diff --git a/subscribie/settings.py b/subscribie/settings.py new file mode 100644 index 00000000..5870a8f2 --- /dev/null +++ b/subscribie/settings.py @@ -0,0 +1,64 @@ +from strictyaml import load, Map, Email, Str, Url, Int, Bool, Regex, CommaSeparated + +# Load application settings according to schema + +# Schema for Subscribie application settings +# See also https://hitchdev.com/strictyaml/ +schema = Map( + { + "FLASK_ENV": Str(), + "SAAS_URL": Url(), + "SAAS_API_KEY": Str(), + "SAAS_ACTIVATE_ACCOUNT_PATH": Str(), + "SUBSCRIBIE_REPO_DIRECTORY": Str(), + "SQLALCHEMY_TRACK_MODIFICATIONS": Bool(), + "SQLALCHEMY_DATABASE_URI": Regex("sqlite:////.*"), + "SECRET_KEY": Str(), + "DB_FULL_PATH": Str(), + "MODULES_PATH": Str(), + "TEMPLATE_BASE_DIR": Str(), + "THEME_NAME": Str(), + "CUSTOM_PAGES_PATH": Str(), + "UPLOADED_IMAGES_DEST": Str(), + "UPLOADED_FILES_DEST": Str(), + "MAX_CONTENT_LENGTH": Str(), + "SUCCESS_REDIRECT_URL": Str(), + "THANKYOU_URL": Str(), + "EMAIL_LOGIN_FROM": Str(), + "EMAIL_QUEUE_FOLDER": Str(), + "SERVER_NAME": Str(), + "PERMANENT_SESSION_LIFETIME": Int(), + "MAIL_DEFAULT_SENDER": Email(), + "STRIPE_LIVE_PUBLISHABLE_KEY": Regex("pk_live_..*"), + "STRIPE_LIVE_SECRET_KEY": Regex("sk_live_..*"), + "STRIPE_TEST_PUBLISHABLE_KEY": Regex("pk_test_..*"), + "STRIPE_TEST_SECRET_KEY": Regex("sk_test_..*"), + "STRIPE_CONNECT_ACCOUNT_ANNOUNCER_HOST": Url(), + "PYTHON_LOG_LEVEL": Str(), + "PLAYWRIGHT_HOST": Url(), + "PLAYWRIGHT_HEADLESS": Bool(), + "PATH_TO_SITES": Str(), + "PATH_TO_RENAME_SCRIPT": Str(), + "SUBSCRIBIE_DOMAIN": Str(), + "PRIVATE_KEY": Str(), + "PUBLIC_KEY": Str(), + "SUPPORTED_CURRENCIES": CommaSeparated(Str()), + "ANTI_SPAM_SHOP_NAMES_MODEL_FULL_PATH": Str(), + "TELEGRAM_TOKEN": Str(), + "TELEGRAM_CHAT_ID": Str(), + "TELEGRAM_PYTHON_LOG_LEVEL": Str(), + "TEST_SHOP_OWNER_EMAIL_ISSUE_704": Email(), + "TEST_SHOP_OWNER_LOGIN_URL": Url(), + } +) + + +def load_settings(): + with open("settings.yaml") as fp: + settings_string = fp.read() + settings = load(settings_string, schema) + return settings + + +# Load app setttings via strictyaml & schema +settings = load_settings().data From a0ac73a668868beb4795d656842cf36e157cd8de Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Sun, 7 Jan 2024 15:11:56 +0000 Subject: [PATCH 08/20] #1276 remove unused packages python-dotenv, update .gitignore --- .env.example | 88 ------------------------------- .gitignore | 11 +++- Makefile | 15 ------ pyproject.toml | 4 +- requirements-dev.lock | 78 +++++++++++++++++++++++++++ requirements.lock | 2 - requirements.txt | 33 ------------ setup.py | 42 --------------- systemd/system/subscribie.service | 20 ------- 9 files changed, 89 insertions(+), 204 deletions(-) delete mode 100644 .env.example delete mode 100644 Makefile create mode 100644 requirements-dev.lock delete mode 100644 requirements.txt delete mode 100644 setup.py delete mode 100644 systemd/system/subscribie.service diff --git a/.env.example b/.env.example deleted file mode 100644 index 29f5267b..00000000 --- a/.env.example +++ /dev/null @@ -1,88 +0,0 @@ -#Only values in uppercase are actually stored in the config object later on. So make sure to use uppercase letters for your config keys. - -# Change FLASK_ENV=live for live -FLASK_ENV=development - -# Software as a service (SAAS) -SAAS_URL=https://subscribie.co.uk/ -# SAAS_API_KEY is to allow subscribie platform to send authenticated -# api requests to subscribie shops created by the shop builder. -SAAS_API_KEY=changeme -SAAS_ACTIVATE_ACCOUNT_PATH=/activate - -# For testing this repo in isolation, SUBSCRIBIE_REPO_DIRECTORY can be './' -# for production, SUBSCRIBIE_REPO_DIRECTORY should be wherever the repo -# is cloned to -SUBSCRIBIE_REPO_DIRECTORY=./ -SQLALCHEMY_TRACK_MODIFICATIONS=False -SQLALCHEMY_DATABASE_URI="sqlite:////tmp/data.db" -SECRET_KEY="random string. e.g. echo -e 'from os import urandom\\nprint urandom(25)' | python" -DB_FULL_PATH="/tmp/data.db" -MODULES_PATH="./modules/" -TEMPLATE_BASE_DIR="./subscribie/themes/" -THEME_NAME="jesmond" -CUSTOM_PAGES_PATH="./subscribie/custom_pages/" -UPLOADED_IMAGES_DEST="./subscribie/static/" -UPLOADED_FILES_DEST="./subscribie/uploads/" -# Default 50Mb upload limit -MAX_CONTENT_LENGTH="52428800" -SUCCESS_REDIRECT_URL="http://127.0.0.1:5000/complete_mandate" -THANKYOU_URL="http://127.0.0.1:5000/thankyou" -EMAIL_LOGIN_FROM="hello@example.com" -EMAIL_QUEUE_FOLDER="/var/email-queue/" - -SERVER_NAME="127.0.0.1:5000" - -# Cookie policies -#SESSION_COOKIE_SECURE=True -#SESSION_COOKIE_HTTPONLY=True -#SESSION_COOKIE_SAMESITE=None - -PERMANENT_SESSION_LIFETIME="1800" - -MAIL_DEFAULT_SENDER="noreply@example.com" - -STRIPE_LIVE_PUBLISHABLE_KEY= -STRIPE_LIVE_SECRET_KEY= - -STRIPE_TEST_PUBLISHABLE_KEY= -STRIPE_TEST_SECRET_KEY= - -# Internal server where shop should send its stripe connect account id to. See https://github.com/Subscribie/subscribie/issues/352 -STRIPE_CONNECT_ACCOUNT_ANNOUNCER_HOST=http://127.0.0.1:8001 - -# For development: - -# Python log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL -# See https://docs.python.org/3/howto/logging.html -PYTHON_LOG_LEVEL=DEBUG - -# Playwright testing -PLAYWRIGHT_HOST=http://127.0.0.1:5000/ -PLAYWRIGHT_HEADLESS=true - -#rename shop variables -PATH_TO_SITES="/path/to/sites/subscribie/" -PATH_TO_RENAME_SCRIPT="/path/to/sites/subscribie/rename-shop.sh" -SUBSCRIBIE_DOMAIN="subscriby.shop" - -PRIVATE_KEY="/tmp/private.pem" -PUBLIC_KEY="/tmp/public.pem" - -# Currencies -SUPPORTED_CURRENCIES="GBP,USD,EUR" - -# Anti spam -ANTI_SPAM_SHOP_NAMES_MODEL_FULL_PATH="/path/to/classifier.pkl" - - -# Optional -TELEGRAM_TOKEN= -TELEGRAM_CHAT_ID= -TELEGRAM_PYTHON_LOG_LEVEL=ERROR - - -# Environment Settings for tests -TEST_SHOP_OWNER_EMAIL_ISSUE_704=admin@example.com -TEST_SHOP_OWNER_LOGIN_URL=http://127.0.0.1:5000/auth/login - diff --git a/.gitignore b/.gitignore index 87413bab..e7a2b58c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ settings.yaml settings.yml +*.pem venv/ -*.sql +.venv *.swp *.pyc __pycache__/ @@ -18,6 +19,7 @@ subscribie.egg-info/* build/* dist/* data.db +data.db* modules/ *.db *.pub @@ -30,6 +32,7 @@ node_modules *chromium.png *webkit.png .env +.env* .env.docker *.webm public @@ -44,3 +47,9 @@ tests/browser-automated-tests-playwright/worker* tests/browser-automated-tests-playwright/e2e/*-snapshots subscribie/static/* subscribie/custom_pages/* +playwright-report +tests/browser-automated-tests-playwright +.terraform +*.pkl +emails +*.bk diff --git a/Makefile b/Makefile deleted file mode 100644 index aced0021..00000000 --- a/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -.ONESHELL: - -default: install - -install: - virtualenv -p python3.6 venv - . venv/bin/activate - pip install . - cp .env.example .env - python -m pytest - -clean: - rm -rf venv - find -iname "*.pyc" -delete - diff --git a/pyproject.toml b/pyproject.toml index 9ed2cae4..864255b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "subscribie" version = "v0.1.181" description = "Collect recurring payments online - subscription payments collection automation" authors = [ - { name = "chrisjsimpson", email = "chris.j.simpson@live.co.uk" } + { name = "Karma Computing", email = "subscribie@karmacomputing.co.uk" } ] dependencies = [ "graphviz", @@ -24,13 +24,11 @@ dependencies = [ "Flask-SQLAlchemy", "Flask-Migrate", "Flask-Babel", - "python-dotenv", "pyjwt[crypto]", "py-auth-header-parser", "pydantic", "backoff", "coloredlogs", - "env-validate", "python-dateutil", "currency-symbols", "pycryptodome", diff --git a/requirements-dev.lock b/requirements-dev.lock new file mode 100644 index 00000000..0f134b89 --- /dev/null +++ b/requirements-dev.lock @@ -0,0 +1,78 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false + +-e file:. +alembic==1.13.1 +annotated-types==0.6.0 +babel==2.14.0 +backoff==2.2.1 +black==23.12.1 +blinker==1.7.0 +certifi==2023.11.17 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +coloredlogs==15.0.1 +cryptography==41.0.7 +currency-symbols==2.0.3 +dnspython==2.4.2 +email-validator==2.1.0.post1 +flask==3.0.0 +flask-babel==4.0.0 +flask-cors==4.0.0 +flask-mail==0.9.1 +flask-migrate==4.0.5 +flask-reuploaded==1.4.0 +flask-sqlalchemy==3.1.1 +flask-wtf==1.2.1 +gitdb==4.0.11 +gitpython==3.1.40 +graphlib==0.9.5 +graphviz==0.20.1 +greenlet==3.0.3 +humanfriendly==10.0 +idna==3.6 +iniconfig==2.0.0 +itsdangerous==2.1.2 +jinja2==3.1.2 +joblib==1.3.2 +mako==1.3.0 +markupsafe==2.1.3 +mypy-extensions==1.0.0 +numpy==1.26.2 +packaging==23.2 +pandas==2.1.4 +pathlib==1.0.1 +pathspec==0.12.1 +platformdirs==4.1.0 +pluggy==1.3.0 +py-auth-header-parser==1.0.2 +pycountry==23.12.11 +pycparser==2.21 +pycryptodome==3.19.0 +pydantic==2.5.3 +pydantic-core==2.14.6 +pyjwt==2.8.0 +pytest==7.4.3 +python-dateutil==2.8.2 +pytz==2023.3.post1 +requests==2.31.0 +scikit-learn==1.3.2 +scipy==1.11.4 +six==1.16.0 +smmap==5.0.1 +sqlalchemy==2.0.23 +strictyaml==1.7.3 +stripe==7.10.0 +threadpoolctl==3.2.0 +typing-extensions==4.9.0 +tzdata==2023.3 +urllib3==2.1.0 +werkzeug==3.0.1 +wheel==0.42.0 +wtforms==3.1.1 diff --git a/requirements.lock b/requirements.lock index fd482c9d..0f134b89 100644 --- a/requirements.lock +++ b/requirements.lock @@ -22,7 +22,6 @@ cryptography==41.0.7 currency-symbols==2.0.3 dnspython==2.4.2 email-validator==2.1.0.post1 -env-validate==0.5 flask==3.0.0 flask-babel==4.0.0 flask-cors==4.0.0 @@ -61,7 +60,6 @@ pydantic-core==2.14.6 pyjwt==2.8.0 pytest==7.4.3 python-dateutil==2.8.2 -python-dotenv==1.0.0 pytz==2023.3.post1 requests==2.31.0 scikit-learn==1.3.2 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e9afbec0..00000000 --- a/requirements.txt +++ /dev/null @@ -1,33 +0,0 @@ -graphviz -graphlib -wheel -Flask>=2,<3 -flask_cors -Flask-Reuploaded==0.3.2 -Flask-WTF==1.0.0 -email-validator==1.1.3 -Flask-Mail>=0.9.1 -requests==2.31.0 -blinker==1.4 -stripe -GitPython -pathlib -pytest -Flask-SQLAlchemy==2.5.1 -Flask-Migrate==3.0.1 -Flask-Babel==2.0.0 -python-dotenv==0.13.0 -pyjwt[crypto] -py-auth-header-parser==1.0.2 -pydantic==1.6.2 -backoff==1.10.0 -coloredlogs==15.0 -env-validate==0.4 -python-dateutil==2.8.2 -currency-symbols==2.0.1 -pycryptodome==3.14.1 -pycountry==22.3.5 -SQLAlchemy==1.4.43 -scipy -pandas -scikit-learn diff --git a/setup.py b/setup.py deleted file mode 100644 index 62e7bc52..00000000 --- a/setup.py +++ /dev/null @@ -1,42 +0,0 @@ -import setuptools - -setuptools.setup( - name="subscribie", - version="0.0.7", - author="Karma Computing", - author_email="subscribie@karmacomputing.co.uk", - desciption="Recurring subscription management and billing", - packages=setuptools.find_packages(), - include_package_data=True, - classifiers=( - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU Affero General Public License v3", - "Operating System :: OS Independent", - ), - install_requires=[ - "Flask>=1,<2", - "flask_cors", - "Flask-Reuploaded==0.3.2", - "flask_wtf", - "wtforms[email]", - "Flask-Mail>=0.9.1", - "requests", - "blinker", - "oauth2client", - "pyyaml", - "stripe", - "GitPython", - "pytest", - "tinycss", - "flask_sqlalchemy", - "flask_migrate", - "python-dotenv==0.13.0", - "click==7.1.2", - "pyjwt[crypto]", - "py-auth-header-parser==1.0.2", - "pydantic==1.6.2", - ], - entry_points=""" - [console_scripts] - """, -) diff --git a/systemd/system/subscribie.service b/systemd/system/subscribie.service deleted file mode 100644 index ec3c717a..00000000 --- a/systemd/system/subscribie.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=Subscribie uWSGI Emperor which manages vassals -After=network.target - -[Service] -# Remember to ensure enable-threads = true in your emperor config.ini -ExecStart=/path-to/venv/bin/uwsgi --ini emperor.ini -RuntimeDirectory=uwsgi -Restart=always -KillSignal=SIGQUIT -Type=notify -StandardError=syslog -NotifyAccess=all -WorkingDirectory=/path/to/subscribie/ - -[Install] -WantedBy=multi-user.target - -# See https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#systemd - From e2d8d0a2e5350dff050df300bde664bd66790cb9 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Sun, 7 Jan 2024 15:13:09 +0000 Subject: [PATCH 09/20] #1276 calculate migrations directory using pathlib/__file__ so migratinos work after python packaging --- subscribie/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/subscribie/__init__.py b/subscribie/__init__.py index 7ef9edad..d57bf89f 100644 --- a/subscribie/__init__.py +++ b/subscribie/__init__.py @@ -124,11 +124,14 @@ def start_session(): """Migrate database when app first boots""" log.info("Migrating database") - upgrade( - directory=Path( - current_app.config["SUBSCRIBIE_REPO_DIRECTORY"] + "/migrations" - ) + migrations_dir = ( + Path(os.path.join(os.path.dirname(__file__))) + .parent.absolute() + .joinpath("migrations") ) + log.info(f"migrations_dir is resolved to: {migrations_dir}") + log.info("Performing database migration (if any)") + upgrade(directory=migrations_dir) """Import any custom modules""" # Set custom modules path From fe5c4fdb51e0cca0f01ffcacc7aa60619617a728 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Sun, 7 Jan 2024 16:24:57 +0000 Subject: [PATCH 10/20] #1276 get theme and static folders from package if configured does not exist #1276 abort if configured modules dir does not exist --- subscribie/Template.py | 44 ++++++++++++++++++++++++++++++++++++++++++ subscribie/__init__.py | 7 +++++++ 2 files changed, 51 insertions(+) diff --git a/subscribie/Template.py b/subscribie/Template.py index 5eb7db5c..a4b2df41 100644 --- a/subscribie/Template.py +++ b/subscribie/Template.py @@ -1,5 +1,9 @@ import jinja2 from pathlib import Path +import logging +from flask import abort + +log = logging.getLogger(__name__) def load_theme(app): @@ -31,12 +35,52 @@ def load_theme(app): app.config["THEME_NAME"], ).resolve() + if themePath.exists() is False: + log.warning(f"Configured themePath ({themePath}) does not exist.") + + log.warning("Attempting theme path from built-in package (if exists)") + + themePath = ( + Path(__file__) + .parent.joinpath( + f'themes/theme-{app.config["THEME_NAME"]}/{app.config["THEME_NAME"]}' + ) + .resolve() + ) + if themePath.exists() is False: + abort( + f"Unable to determine valid themePath. Check TEMPLATE_BASE_DIR in settings and THEME_NAME. TEMPLATE_BASE_DIR is set to {TEMPLATE_BASE_DIR} and THEME_NAME: {THEME_NAME}" # noqa :E501 + ) + log.info(f"Default package themePath is set to {themePath} and exists.") + else: + log.info(f"Custom themePath is set to {themePath} and exists.") + staticFolder = Path( app.config["TEMPLATE_BASE_DIR"], "theme-" + app.config["THEME_NAME"], "static", ).resolve() + if staticFolder.exists() is False: + log.warning(f"Configured staticFolder ({staticFolder}) does not exist.") + + log.warning( + "Attempting to resolve staticFolder from built-in package (if exists)" + ) + + staticFolder = ( + Path(__file__) + .parent.joinpath(Path(f'themes/theme-{app.config["THEME_NAME"]}', "static")) + .resolve() + ) + if staticFolder.exists() is False: + abort( + f"Unable to determine valid staticFolder. Check TEMPLATE_BASE_DIR in settings and THEME_NAME. TEMPLATE_BASE_DIR is set to {TEMPLATE_BASE_DIR} and THEME_NAME: {THEME_NAME}" # noqa :E501 + ) + log.info(f"Default package staticFolder is set to {staticFolder} and exists.") + else: + log.info(f"Custom staticFolder is set to {staticFolder} and exists.") + # Set THEME_PATH app.config["THEME_PATH"] = themePath diff --git a/subscribie/__init__.py b/subscribie/__init__.py index d57bf89f..1f743fb7 100644 --- a/subscribie/__init__.py +++ b/subscribie/__init__.py @@ -24,6 +24,7 @@ current_app, Blueprint, request, + abort, ) from flask_babel import Babel, _ from subscribie.email import EmailMessageQueue @@ -135,6 +136,12 @@ def start_session(): """Import any custom modules""" # Set custom modules path + modulesPath = Path(app.config["MODULES_PATH"]) + if modulesPath.exists() is False: + msg = f"Configured MODULES_PATH '{modulesPath}' does not exist" + log.error(msg) + abort(msg) + sys.path.append(app.config["MODULES_PATH"]) modules = Module.query.all() log.info(f"sys.path contains: {sys.path}") From a165e5573c4f6ff9d11815689166e17797d0ed50 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Sun, 7 Jan 2024 17:16:14 +0000 Subject: [PATCH 11/20] #1276 load seed.sql from package if not found in cwd --- subscribie/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/subscribie/__init__.py b/subscribie/__init__.py index 1f743fb7..68ff3540 100644 --- a/subscribie/__init__.py +++ b/subscribie/__init__.py @@ -239,7 +239,14 @@ def page_error_500(e): def initdb(): """Initialize the database.""" click.echo("Init the db") - with open("seed.sql") as fp: + DBseedFile = Path("seed.sql") + if DBseedFile.exists() is False: + log.warning(f"DBseedFile does not exist in {os.getcwd()}") + DBseedFile = Path(Path(__file__).parent.parent, "seed.sql") + log.warning( + f"Attempting to load DBseedFile from package dir at {DBseedFile}" + ) + with open(DBseedFile) as fp: con = sqlite3.connect(app.config["DB_FULL_PATH"]) cur = con.cursor() # Check not already seeded From 64616cf1dc9ec9491926acfd1887d836b4b98af1 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Mon, 8 Jan 2024 07:34:09 +0000 Subject: [PATCH 12/20] #1276 use importlib.resources for datafiles import See https://importlib-resources.readthedocs.io/en/latest/using.html#using-importlib-resources --- subscribie/__init__.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/subscribie/__init__.py b/subscribie/__init__.py index 68ff3540..b985a02e 100644 --- a/subscribie/__init__.py +++ b/subscribie/__init__.py @@ -38,6 +38,7 @@ import importlib import urllib from pathlib import Path +from importlib.resources import files, as_file import sqlalchemy from flask_migrate import Migrate, upgrade import click @@ -125,14 +126,20 @@ def start_session(): """Migrate database when app first boots""" log.info("Migrating database") - migrations_dir = ( - Path(os.path.join(os.path.dirname(__file__))) - .parent.absolute() - .joinpath("migrations") - ) - log.info(f"migrations_dir is resolved to: {migrations_dir}") - log.info("Performing database migration (if any)") - upgrade(directory=migrations_dir) + migrations_dir_context = files("migrations") + # See: + # https://importlib-resources.readthedocs.io/en/latest/using.html#using-importlib-resources + # https://docs.python.org/3.11/library/importlib.resources.html#importlib.resources.as_file + # https://docs.python.org/3/whatsnew/3.12.html#importlib-resources + # https://bugs.python.org/issue45427 + # https://github.com/python/importlib_resources/pull/255 + with as_file(migrations_dir_context) as migrations_dir_tmp: + log.info(f"migrations_dir_tmp is {migrations_dir_tmp}") + log.info( + "https://docs.python.org/3.11/library/importlib.resources.html#importlib.resources.as_file" # noqa: E501 + ) + log.info("Performing database migration (if any)") + upgrade(directory=migrations_dir_tmp) """Import any custom modules""" # Set custom modules path @@ -242,7 +249,7 @@ def initdb(): DBseedFile = Path("seed.sql") if DBseedFile.exists() is False: log.warning(f"DBseedFile does not exist in {os.getcwd()}") - DBseedFile = Path(Path(__file__).parent.parent, "seed.sql") + DBseedFile = files("subscribie").parent.joinpath("seed.sql") log.warning( f"Attempting to load DBseedFile from package dir at {DBseedFile}" ) From 0556066cc8da4c5333d6dd2aa0b3fd5fdcdb6bc0 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Fri, 19 Jan 2024 22:11:07 +0000 Subject: [PATCH 13/20] #1276 update .envsubst.template for subscribie deployer --- .envsubst.template | 15 ++++----------- subscribie/settings.py | 1 + 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.envsubst.template b/.envsubst.template index bfa2bc1c..2aa409bb 100644 --- a/.envsubst.template +++ b/.envsubst.template @@ -1,6 +1,3 @@ -#Only values in uppercase are actually stored in the config object later on. So make sure to use uppercase letters for your config keys. - -# Change FLASK_ENV=live for live FLASK_ENV=${FLASK_ENV} # Software as a service (SAAS) @@ -39,6 +36,8 @@ SERVER_NAME=${SERVER_NAME} #SESSION_COOKIE_HTTPONLY=True #SESSION_COOKIE_SAMESITE=None +PERMANENT_SESSION_LIFETIME=${PERMANENT_SESSION_LIFETIME} + MAIL_DEFAULT_SENDER=${MAIL_DEFAULT_SENDER} # Supported currencies @@ -53,17 +52,9 @@ STRIPE_TEST_SECRET_KEY=${STRIPE_TEST_SECRET_KEY} # Internal server where shop should send its stripe connect account id to. See https://github.com/Subscribie/subscribie/issues/352 STRIPE_CONNECT_ACCOUNT_ANNOUNCER_HOST=${STRIPE_CONNECT_ACCOUNT_ANNOUNCER_HOST} -# Google oauth (signin / up with Google -GOOGLE_CLIENT_ID=???.apps.googleusercontent.com -GOOGLE_CLIENT_SECRET=??? -GOOGLE_REDIRECT_URI="http://127.0.0.1:5000/google-oauth2callback/" -GOOGLE_RESPONSE_TYPE=code -GOOGLE_SCOPE="email openid profile" # For development: -HONEYCOMB_API_KEY= - # Python log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL # See https://docs.python.org/3/howto/logging.html PYTHON_LOG_LEVEL=DEBUG @@ -80,6 +71,8 @@ SUBSCRIBIE_DOMAIN="subscriby.shop" PRIVATE_KEY="/tmp/private.pem" PUBLIC_KEY="/tmp/public.pem" +ANTI_SPAM_SHOP_NAMES_MODEL_FULL_PATH=/changeme + # Optional TELEGRAM_TOKEN= diff --git a/subscribie/settings.py b/subscribie/settings.py index 5870a8f2..27d4ef59 100644 --- a/subscribie/settings.py +++ b/subscribie/settings.py @@ -49,6 +49,7 @@ "TELEGRAM_PYTHON_LOG_LEVEL": Str(), "TEST_SHOP_OWNER_EMAIL_ISSUE_704": Email(), "TEST_SHOP_OWNER_LOGIN_URL": Url(), + "MAIL_SERVER": Str(), } ) From 4993430381d4b39448cf93f7c3c84e0ceca4b729 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Fri, 19 Jan 2024 22:27:24 +0000 Subject: [PATCH 14/20] Ref #1276 remove MAIN_SERVER setting not needed as using incron pickup submission --- subscribie/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/subscribie/settings.py b/subscribie/settings.py index 27d4ef59..5870a8f2 100644 --- a/subscribie/settings.py +++ b/subscribie/settings.py @@ -49,7 +49,6 @@ "TELEGRAM_PYTHON_LOG_LEVEL": Str(), "TEST_SHOP_OWNER_EMAIL_ISSUE_704": Email(), "TEST_SHOP_OWNER_LOGIN_URL": Url(), - "MAIL_SERVER": Str(), } ) From ef292ef8f8f67d51390299dd89a0da7ad7cb23ed Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Mon, 29 Jan 2024 22:26:45 +0000 Subject: [PATCH 15/20] downgrade stripe_api_key not set to a warning downgrade stripe connect account not setup to a warning not error --- subscribie/blueprints/admin/subscription.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subscribie/blueprints/admin/subscription.py b/subscribie/blueprints/admin/subscription.py index dbbb9508..66275181 100644 --- a/subscribie/blueprints/admin/subscription.py +++ b/subscribie/blueprints/admin/subscription.py @@ -24,9 +24,9 @@ def update_stripe_subscription_statuses(app): stripe.api_key = get_stripe_secret_key() connect_account = get_stripe_connect_account() if stripe.api_key is None: - log.error("Stripe api key not set refusing to update subscription statuses") + log.warning("Stripe api key not set refusing to update subscription statuses") # noqa: E501 if connect_account is None: - log.error( + log.warning( "Stripe connect account not set. Refusing to update subscription statuses" # noqa: E501 ) if stripe_connect_active(): From 6d8fa3c9b0c66dbaa77f6aa3244829194e2107df Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Tue, 30 Jan 2024 16:18:17 +0000 Subject: [PATCH 16/20] #1276 bump minimal python version --- .github/workflows/python-package.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 4b592cb2..b05f911c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8] + python-version: [3.11, 3.12] steps: - uses: actions/checkout@v2 diff --git a/pyproject.toml b/pyproject.toml index 864255b9..90e19d5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ "strictyaml", ] readme = "README.md" -requires-python = ">= 3.8" +requires-python = ">= 3.11" [project.scripts] From d6187cb35dc0abc8b1c136bfa3abadf9fcb3e6be Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Tue, 30 Jan 2024 16:18:43 +0000 Subject: [PATCH 17/20] tidy --- subscribie/blueprints/admin/priceListRule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subscribie/blueprints/admin/priceListRule.py b/subscribie/blueprints/admin/priceListRule.py index b360cb50..a95a4fee 100644 --- a/subscribie/blueprints/admin/priceListRule.py +++ b/subscribie/blueprints/admin/priceListRule.py @@ -1,5 +1,5 @@ from . import admin -from subscribie.settings import settings +from subscribie import settings from subscribie.auth import login_required from subscribie.models import PriceListRule from subscribie.database import database @@ -7,7 +7,7 @@ import logging from datetime import datetime -dog = 1 + log = logging.getLogger(__name__) if settings.get("SUPPORTED_CURRENCIES", False) is not False: From e8585e80afe954991ce8dca0c10acee7278c4619 Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Tue, 30 Jan 2024 16:48:58 +0000 Subject: [PATCH 18/20] #1276 python min version 3.12 update python-package pipeline --- .github/workflows/python-package.yml | 11 +++++++---- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b05f911c..c85aa9e1 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.11, 3.12] + python-version: [3.12] steps: - uses: actions/checkout@v2 @@ -30,7 +30,7 @@ jobs: run: | python -m pip install --upgrade pip pip install flake8 pytest coverage - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install -r requirements.lock - name: Lint with flake8 run: | true @@ -38,9 +38,12 @@ jobs: #flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide #flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Copy .env.example to .env + - name: Copy settings.yaml.example to settings.yaml run: | - cp .env.example .env + cp settings.yaml.example settings.yaml + - name: Create empty modules path + run: | + mkdir -p ./modules - name: Migrate database using flask db upgrade run: | pwd diff --git a/pyproject.toml b/pyproject.toml index 90e19d5b..72bd07b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ "strictyaml", ] readme = "README.md" -requires-python = ">= 3.11" +requires-python = ">= 3.12" [project.scripts] From ab1b54daa9c41945513fae489ba8a6160c00495d Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Tue, 30 Jan 2024 17:15:56 +0000 Subject: [PATCH 19/20] Fix #1282 include archived plans in calculating active subscribers --- subscribie/blueprints/admin/stats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/subscribie/blueprints/admin/stats.py b/subscribie/blueprints/admin/stats.py index 034ae9ab..06bec3f3 100644 --- a/subscribie/blueprints/admin/stats.py +++ b/subscribie/blueprints/admin/stats.py @@ -15,6 +15,7 @@ def get_number_of_active_subscribers(): .join(Plan, Subscription.sku_uuid == Plan.uuid) .join(PlanRequirements, Plan.id == PlanRequirements.plan_id) .filter(PlanRequirements.subscription == 1) + .execution_options(include_archived=True) ) # Check if their subscriptions are active for subscriber in subscribers_with_subscriptions: From aed69c1578024bbaa386f8f488a5eb4a037bb5fd Mon Sep 17 00:00:00 2001 From: chrisjsimpson Date: Wed, 31 Jan 2024 20:43:40 +0000 Subject: [PATCH 20/20] Fix #1282 Stats to include archived plans --- subscribie/blueprints/admin/stats.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/subscribie/blueprints/admin/stats.py b/subscribie/blueprints/admin/stats.py index 06bec3f3..82a7391e 100644 --- a/subscribie/blueprints/admin/stats.py +++ b/subscribie/blueprints/admin/stats.py @@ -52,6 +52,7 @@ def get_number_of_subscribers(): .join(Plan, Subscription.sku_uuid == Plan.uuid) .join(PlanRequirements, Plan.id == PlanRequirements.plan_id) .filter(PlanRequirements.subscription == 1) + .execution_options(include_archived=True) .count() ) return count @@ -64,6 +65,7 @@ def get_number_of_signups(): .join(Subscription) .join(Plan, Subscription.sku_uuid == Plan.uuid) .join(PlanRequirements, Plan.id == PlanRequirements.plan_id) + .execution_options(include_archived=True) .count() ) return count @@ -78,6 +80,7 @@ def get_number_of_one_off_purchases(): .join(PlanRequirements, Plan.id == PlanRequirements.plan_id) .filter(PlanRequirements.subscription == 0) .filter(PlanRequirements.instant_payment == 1) + .execution_options(include_archived=True) .count() ) return count @@ -88,6 +91,7 @@ def get_number_of_transactions_with_donations(): database.session.query(Person) .join(Transaction) .filter(Transaction.is_donation == 1) + .execution_options(include_archived=True) .count() ) return count