From 29786dc35ecb33d2a028c62c8ff5d9a554e390f8 Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Thu, 21 Nov 2024 14:47:55 +0200 Subject: [PATCH 01/12] feat(plugins): init plugin loader --- static/js/ConnectionsPanel.jsx | 5 +++ static/js/WebComponentLoader.jsx | 54 ++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 static/js/WebComponentLoader.jsx diff --git a/static/js/ConnectionsPanel.jsx b/static/js/ConnectionsPanel.jsx index 039b3eadf3..15d158cd9a 100644 --- a/static/js/ConnectionsPanel.jsx +++ b/static/js/ConnectionsPanel.jsx @@ -31,6 +31,7 @@ import { AddToSourceSheetBox } from './AddToSourceSheet'; import LexiconBox from './LexiconBox'; import AboutBox from './AboutBox'; import GuideBox from './GuideBox'; +import WebComponentLoader from './WebComponentLoader' import TranslationsBox from './TranslationsBox'; import ExtendedNotes from './ExtendedNotes'; import classNames from 'classnames'; @@ -348,6 +349,7 @@ class ConnectionsPanel extends Component { this.props.setConnectionsMode("Navigation")} /> this.props.setConnectionsMode("SidebarSearch")} /> this.props.setConnectionsMode("Translations")} count={resourcesButtonCounts.translations} /> + this.props.setConnectionsMode("Plugin")} /> } {showConnectionSummary ? @@ -675,6 +677,9 @@ class ConnectionsPanel extends Component { onSidebarSearchClick={this.props.onSidebarSearchClick} /> } + else if (this.props.mode === "Plugin") { + content = + } const marginless = ["Resources", "ConnectionsList", "Advanced Tools", "Share", "WebPages", "Topics", "manuscripts"].indexOf(this.props.mode) !== -1; let classes = classNames({ connectionsPanel: 1, textList: 1, marginless: marginless, fullPanel: this.props.fullPanel, singlePanel: !this.props.fullPanel }); diff --git a/static/js/WebComponentLoader.jsx b/static/js/WebComponentLoader.jsx new file mode 100644 index 0000000000..16c87dd417 --- /dev/null +++ b/static/js/WebComponentLoader.jsx @@ -0,0 +1,54 @@ +import React, { useState } from 'react'; + +function WebComponentLoader(props) { + const [link, setLink] = useState(''); + const [loaded, setLoaded] = useState(false); + const sref = props.sref; + + const repoToRawLink = (link) => { + const repo = link.split('github.com/')[1].split('/'); + const dateTimeStamp = new Date().getTime(); + return `https://${repo[0]}.github.io/${repo[1]}/plugin.js?rand=${dateTimeStamp}` + } + + let script = null; + + const handleClick = () => { + if (script) { + document.head.removeChild(script); + setLoaded(false); + } + if (link) { + script = document.createElement('script'); + script.src = repoToRawLink(link); + script.async = true; + script.onload = () => { + setLoaded(true); + }; + document.head.appendChild(script); + } + }; + + if (loaded) { + return ( +
+ + +
+ ); + } + + return ( +
+ setLink(e.target.value)} + /> + +
+ ); +} + +export default WebComponentLoader; From d6d172b6f715ffb057d7d173536f6575d266b3cb Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Fri, 22 Nov 2024 00:10:05 +0200 Subject: [PATCH 02/12] feat(plugins): Create sandbox for component dev --- plugins/__init__.py | 0 plugins/models.py | 3 +++ plugins/tests.py | 16 ++++++++++++++++ plugins/views.py | 32 ++++++++++++++++++++++++++++++++ sefaria/urls.py | 6 ++++++ static/js/WebComponentLoader.jsx | 16 +++++++++++----- 6 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 plugins/__init__.py create mode 100644 plugins/models.py create mode 100644 plugins/tests.py create mode 100644 plugins/views.py diff --git a/plugins/__init__.py b/plugins/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/models.py b/plugins/models.py new file mode 100644 index 0000000000..71a8362390 --- /dev/null +++ b/plugins/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/plugins/tests.py b/plugins/tests.py new file mode 100644 index 0000000000..501deb776c --- /dev/null +++ b/plugins/tests.py @@ -0,0 +1,16 @@ +""" +This file demonstrates writing tests using the unittest module. These will pass +when you run "manage.py test". + +Replace this with more appropriate tests for your application. +""" + +from django.test import TestCase + + +class SimpleTest(TestCase): + def test_basic_addition(self): + """ + Tests that 1 + 1 always equals 2. + """ + self.assertEqual(1 + 1, 2) diff --git a/plugins/views.py b/plugins/views.py new file mode 100644 index 0000000000..5d812200ed --- /dev/null +++ b/plugins/views.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +from django.http import HttpResponse +import requests + +import structlog +logger = structlog.get_logger(__name__) + + +def dev(request): + """ + Render the dev version of a plugin. + + @query_param request: Django request object + @query_param plugin_url: URL of the plugin + + This endpoint pulls the plugin from the plugin_url and updates the plugin's + custome element name to target. + """ + plugin_url = request.GET.get("plugin_url") + target = request.GET.get("target") + + custom_component_name = target + costum_component_class_name = (target[0].upper() + target[1:]).replace("-", "") + + content = requests.get(plugin_url) + plugin = content.text + + # replace all instances of the plugin's custom element name with the target + plugin = plugin.replace("sefaria-plugin", custom_component_name) + plugin = plugin.replace("SefariaPlugin", costum_component_class_name) + + return HttpResponse(plugin, content_type="text/javascript") diff --git a/sefaria/urls.py b/sefaria/urls.py index b73ae60733..50d82eca0b 100644 --- a/sefaria/urls.py +++ b/sefaria/urls.py @@ -12,6 +12,7 @@ import reader.views as reader_views import sefaria.views as sefaria_views import sourcesheets.views as sheets_views +import plugins.views as plugins_views import sefaria.gauth.views as gauth_views import django.contrib.auth.views as django_auth_views import api.views as api_views @@ -472,6 +473,11 @@ url(r'^(?P[^/]+)(/)?$', reader_views.catchall) ] +# Plugin API +urlpatterns += [ + url(r'^plugin/dev/?$', plugins_views.dev), +] + if DOWN_FOR_MAINTENANCE: # Keep admin accessible urlpatterns = [ diff --git a/static/js/WebComponentLoader.jsx b/static/js/WebComponentLoader.jsx index 16c87dd417..a052a5cfe3 100644 --- a/static/js/WebComponentLoader.jsx +++ b/static/js/WebComponentLoader.jsx @@ -3,15 +3,18 @@ import React, { useState } from 'react'; function WebComponentLoader(props) { const [link, setLink] = useState(''); const [loaded, setLoaded] = useState(false); + const [pluginName, setPluginName] = useState('sefaria-plugin'); const sref = props.sref; - const repoToRawLink = (link) => { + const repoToRawLink = (link, target) => { const repo = link.split('github.com/')[1].split('/'); - const dateTimeStamp = new Date().getTime(); - return `https://${repo[0]}.github.io/${repo[1]}/plugin.js?rand=${dateTimeStamp}` + const JsUrl = `https://${repo[0]}.github.io/${repo[1]}/plugin.js` + const middlewareLink = `/plugin/dev?target=${target}&plugin_url=${JsUrl}` + return middlewareLink } let script = null; + let rand = Math.floor(Math.random() * 1000); const handleClick = () => { if (script) { @@ -19,8 +22,10 @@ function WebComponentLoader(props) { setLoaded(false); } if (link) { + const target = `sefaria-plugin-${rand}` + setPluginName(target); script = document.createElement('script'); - script.src = repoToRawLink(link); + script.src = repoToRawLink(link, target); script.async = true; script.onload = () => { setLoaded(true); @@ -30,10 +35,11 @@ function WebComponentLoader(props) { }; if (loaded) { + const PluginElm = pluginName; return (
- +
); } From afb2f19957f678dc74112a51468b913536447853 Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Sun, 24 Nov 2024 21:15:01 +0200 Subject: [PATCH 03/12] chore(build): restart build --- plugins/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/models.py b/plugins/models.py index 71a8362390..ca200a0530 100644 --- a/plugins/models.py +++ b/plugins/models.py @@ -1,3 +1,2 @@ from django.db import models - # Create your models here. From 326cf7efb3570713cd46c95ad6ca65fab2824b20 Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Sun, 8 Dec 2024 13:40:14 +0200 Subject: [PATCH 04/12] chore(encrypted fields): add encrypted field for django models --- fields/encrypted.py | 69 +++++++++++++++++++++++++++++++++++++ sefaria/utils/encryption.py | 46 +++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 fields/encrypted.py create mode 100644 sefaria/utils/encryption.py diff --git a/fields/encrypted.py b/fields/encrypted.py new file mode 100644 index 0000000000..976ac2d8c0 --- /dev/null +++ b/fields/encrypted.py @@ -0,0 +1,69 @@ +import itertools + +import django.db +import django.db.models + +import cryptography.fernet + +from sefaria.utils.encryption import get_crypter + + +CRYPTER = get_crypter() + + +def encrypt_str(s): + # be sure to encode the string to bytes + return CRYPTER.encrypt(s.encode('utf-8')) + + +def decrypt_str(t): + # be sure to decode the bytes to a string + return CRYPTER.decrypt(t.encode('utf-8')).decode('utf-8') + + +def calc_encrypted_length(n): + # calculates the characters necessary to hold an encrypted string of + # n bytes + return len(encrypt_str('a' * n)) + + +class EncryptedMixin(object): + def to_python(self, value): + if value is None: + return value + + if isinstance(value, (bytes, str)): + if isinstance(value, bytes): + value = value.decode('utf-8') + try: + value = decrypt_str(value) + except cryptography.fernet.InvalidToken: + pass + + return super(EncryptedMixin, self).to_python(value) + + def from_db_value(self, value, *args, **kwargs): + return self.to_python(value) + + def get_db_prep_save(self, value, connection): + value = super(EncryptedMixin, self).get_db_prep_save(value, connection) + + if value is None: + return value + # decode the encrypted value to a unicode string, else this breaks in pgsql + return (encrypt_str(str(value))).decode('utf-8') + + def get_internal_type(self): + return "TextField" + + def deconstruct(self): + name, path, args, kwargs = super(EncryptedMixin, self).deconstruct() + + if 'max_length' in kwargs: + del kwargs['max_length'] + + return name, path, args, kwargs + + +class EncryptedCharField(EncryptedMixin, django.db.models.CharField): + pass diff --git a/sefaria/utils/encryption.py b/sefaria/utils/encryption.py new file mode 100644 index 0000000000..aefb3a1bea --- /dev/null +++ b/sefaria/utils/encryption.py @@ -0,0 +1,46 @@ +from django.conf import settings +from django.core import validators +from django.core.exceptions import ImproperlyConfigured +from django.utils import timezone +from django.utils.functional import cached_property +import cryptography.fernet + + +def parse_key(key): + """ + If the key is a string we need to ensure that it can be decoded + :param key: + :return: + """ + return cryptography.fernet.Fernet(key) + + +def get_crypter(configured_keys=None): + if not configured_keys: + configured_keys = getattr(settings, 'FIELD_ENCRYPTION_KEY', None) + + if configured_keys is None: + raise ImproperlyConfigured('FIELD_ENCRYPTION_KEY must be defined in settings') + + try: + # Allow the use of key rotation + if isinstance(configured_keys, (tuple, list)): + keys = [parse_key(k) for k in configured_keys] + else: + # else turn the single key into a list of one + keys = [parse_key(configured_keys), ] + except Exception as e: + raise ImproperlyConfigured(f'FIELD_ENCRYPTION_KEY defined incorrectly: {str(e)}') + + if len(keys) == 0: + raise ImproperlyConfigured('No keys defined in setting FIELD_ENCRYPTION_KEY') + + return cryptography.fernet.MultiFernet(keys) + +def encrypt_str_with_key(s, key): + cypher = get_crypter(key) + return cypher.encrypt(s.encode('utf-8')) + +def decrypt_str_with_key(t, key): + cypher = get_crypter(key) + return cypher.decrypt(t.encode('utf-8')).decode('utf-8') \ No newline at end of file From aae9997eb5142d6b4de18cc2f29326248a169c60 Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Sun, 8 Dec 2024 13:41:05 +0200 Subject: [PATCH 05/12] feat(plugin app): add plugin app --- plugins/admin.py | 9 +++++++++ plugins/apps.py | 6 ++++++ plugins/migrations/0001_initial.py | 28 ++++++++++++++++++++++++++++ plugins/migrations/__init__.py | 0 plugins/models.py | 24 +++++++++++++++++++++++- plugins/views.py | 29 +++++++++++++++++++++++++++++ sefaria/settings.py | 1 + sefaria/urls.py | 1 + 8 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 plugins/admin.py create mode 100644 plugins/apps.py create mode 100644 plugins/migrations/0001_initial.py create mode 100644 plugins/migrations/__init__.py diff --git a/plugins/admin.py b/plugins/admin.py new file mode 100644 index 0000000000..1bb7ab2380 --- /dev/null +++ b/plugins/admin.py @@ -0,0 +1,9 @@ +from .models import Plugin +from django.contrib import admin + +class PluginAdmin(admin.ModelAdmin): + list_display = ('name', 'url', 'secret') + search_fields = ('name',) + fields = ('name', 'description', 'url') + +admin.site.register(Plugin, PluginAdmin) \ No newline at end of file diff --git a/plugins/apps.py b/plugins/apps.py new file mode 100644 index 0000000000..55cbda88ed --- /dev/null +++ b/plugins/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class PluginsAppConfig(AppConfig): + name = "plugins" + verbose_name = "Plugins Management" \ No newline at end of file diff --git a/plugins/migrations/0001_initial.py b/plugins/migrations/0001_initial.py new file mode 100644 index 0000000000..936f73638d --- /dev/null +++ b/plugins/migrations/0001_initial.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-12-05 21:25 +from __future__ import unicode_literals + +from django.db import migrations, models +import fields.encrypted + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Plugin', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ('description', models.TextField()), + ('url', models.URLField()), + ('secret', fields.encrypted.EncryptedCharField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + ] diff --git a/plugins/migrations/__init__.py b/plugins/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/models.py b/plugins/models.py index ca200a0530..5d86600bd7 100644 --- a/plugins/models.py +++ b/plugins/models.py @@ -1,2 +1,24 @@ from django.db import models -# Create your models here. +from fields.encrypted import EncryptedCharField +from cryptography.fernet import Fernet + +class Plugin(models.Model): + name = models.CharField(max_length=100) + description = models.TextField() + url = models.URLField() + secret = EncryptedCharField(max_length=100) + created_at = models.DateTimeField(auto_now_add=True) + + # on create, generate a secret + def save(self, *args, **kwargs): + if not self.secret: + self.secret = self._generate_secret() + + super(Plugin, self).save(*args, **kwargs) + + def _generate_secret(self): + key = Fernet.generate_key() + return key.decode('utf-8') + + def __str__(self): + return self.name diff --git a/plugins/views.py b/plugins/views.py index 5d812200ed..9e955a4df9 100644 --- a/plugins/views.py +++ b/plugins/views.py @@ -3,6 +3,10 @@ import requests import structlog + +from plugins.models import Plugin +from sefaria.client.util import jsonResponse +from sefaria.utils.encryption import encrypt_str_with_key logger = structlog.get_logger(__name__) @@ -30,3 +34,28 @@ def dev(request): plugin = plugin.replace("SefariaPlugin", costum_component_class_name) return HttpResponse(plugin, content_type="text/javascript") + + +def get_user_plugin_secret(request, plugin_id): + """ + Get the secret for a user's plugin. + + @query_param request: Django request object + @query_param plugin_id: ID of the plugin + """ + + user = request.user + plugin = Plugin.objects.get(id=plugin_id) + + # encrypt the user id using the plugin secret + plugin_secret = plugin.secret + user_id = str(user.id) + + # encrypt the user id + encrypted_user_id = encrypt_str_with_key(user_id, plugin_secret) + + json_response = { + "encrypted_user_id": encrypted_user_id.decode('utf-8') + } + + return jsonResponse(json_response) \ No newline at end of file diff --git a/sefaria/settings.py b/sefaria/settings.py index 7eea25ff70..3f472db73a 100644 --- a/sefaria/settings.py +++ b/sefaria/settings.py @@ -150,6 +150,7 @@ 'webpack_loader', 'django_user_agents', 'rest_framework', + 'plugins.apps.PluginsAppConfig', #'easy_timezones' # Uncomment the next line to enable admin documentation: # 'django.contrib.admindocs', diff --git a/sefaria/urls.py b/sefaria/urls.py index 50d82eca0b..b2265aa4b7 100644 --- a/sefaria/urls.py +++ b/sefaria/urls.py @@ -476,6 +476,7 @@ # Plugin API urlpatterns += [ url(r'^plugin/dev/?$', plugins_views.dev), + url(r'^plugin/(?P.+)/user/?$', plugins_views.get_user_plugin_secret), ] if DOWN_FOR_MAINTENANCE: From 8da0882208803d36c1b5fe611b3c69ecbd00ba3e Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Sun, 8 Dec 2024 13:41:26 +0200 Subject: [PATCH 06/12] chore(requirements): add cryptography package --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c1219d9635..f26344a218 100644 --- a/requirements.txt +++ b/requirements.txt @@ -69,7 +69,7 @@ undecorated==0.3.0 unicodecsv==0.14.1 unidecode==1.1.1 user-agents==2.2.0 - +cryptography==44.0.0 #opentelemetry-distro #opentelemetry-exporter-otlp From b4511015b35bed63b67ba3a66e7e29b70217da1c Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Sun, 8 Dec 2024 13:42:03 +0200 Subject: [PATCH 07/12] feat(client): get encrypted user id on client --- static/js/WebComponentLoader.jsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/static/js/WebComponentLoader.jsx b/static/js/WebComponentLoader.jsx index a052a5cfe3..162df93842 100644 --- a/static/js/WebComponentLoader.jsx +++ b/static/js/WebComponentLoader.jsx @@ -13,6 +13,14 @@ function WebComponentLoader(props) { return middlewareLink } + const getPluginUser = (pluginId=1) => { + fetch(`/plugin/${pluginId}/user`) + .then(res => res.json()) + .then(data => { + console.log(data); + }); +} + let script = null; let rand = Math.floor(Math.random() * 1000); @@ -32,6 +40,9 @@ function WebComponentLoader(props) { }; document.head.appendChild(script); } + + getPluginUser(); + }; if (loaded) { From 69e52b73523e802ac0a097cb280c717162dd2cec Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Fri, 13 Dec 2024 00:27:15 +0200 Subject: [PATCH 08/12] ci(helm): semantic commit to prevent error --- plugins/admin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/admin.py b/plugins/admin.py index 1bb7ab2380..a7df63b6bd 100644 --- a/plugins/admin.py +++ b/plugins/admin.py @@ -6,4 +6,5 @@ class PluginAdmin(admin.ModelAdmin): search_fields = ('name',) fields = ('name', 'description', 'url') + admin.site.register(Plugin, PluginAdmin) \ No newline at end of file From 5c413aa4f05a1e4a20880a66ad64efb0fb260662 Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Fri, 13 Dec 2024 09:32:03 +0200 Subject: [PATCH 09/12] chore(helm): add FIELD_ENCRYPTION_KEY to helm chart --- .../templates/configmap/local-settings-file.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml index d12b5ee7f1..626d5c6bb2 100644 --- a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml +++ b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml @@ -153,6 +153,10 @@ data: SEFARIA_BOT_API_KEY = os.getenv("SEFARIA_BOT_API_KEY") + # Field Encryption + FIELD_ENCRYPTION_KEY_STR = os.getenv("FIELD_ENCRYPTION_KEY") + FIELD_ENCRYPTION_KEY = FIELD_ENCRYPTION_KEY_STR.encode('utf-8') + CLOUDFLARE_ZONE= os.getenv("CLOUDFLARE_ZONE") CLOUDFLARE_EMAIL= os.getenv("CLOUDFLARE_EMAIL") CLOUDFLARE_TOKEN= os.getenv("CLOUDFLARE_TOKEN") From a4dbb2978fc12bfdffdc6252e0f9a9751a193b4b Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Fri, 13 Dec 2024 10:42:21 +0200 Subject: [PATCH 10/12] chore(docker): don'e break on build on no key FIELD_ENCRYPTION_KEY --- sefaria/utils/encryption.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sefaria/utils/encryption.py b/sefaria/utils/encryption.py index aefb3a1bea..d8ddc1a958 100644 --- a/sefaria/utils/encryption.py +++ b/sefaria/utils/encryption.py @@ -1,10 +1,9 @@ from django.conf import settings -from django.core import validators from django.core.exceptions import ImproperlyConfigured -from django.utils import timezone -from django.utils.functional import cached_property import cryptography.fernet +import structlog +logger = structlog.get_logger(__name__) def parse_key(key): """ @@ -20,7 +19,8 @@ def get_crypter(configured_keys=None): configured_keys = getattr(settings, 'FIELD_ENCRYPTION_KEY', None) if configured_keys is None: - raise ImproperlyConfigured('FIELD_ENCRYPTION_KEY must be defined in settings') + logger.warning('FIELD_ENCRYPTION_KEY must be defined in settings') + return None try: # Allow the use of key rotation @@ -39,8 +39,10 @@ def get_crypter(configured_keys=None): def encrypt_str_with_key(s, key): cypher = get_crypter(key) - return cypher.encrypt(s.encode('utf-8')) + if cypher: + return cypher.encrypt(s.encode('utf-8')) def decrypt_str_with_key(t, key): cypher = get_crypter(key) - return cypher.decrypt(t.encode('utf-8')).decode('utf-8') \ No newline at end of file + if cypher: + return cypher.decrypt(t.encode('utf-8')).decode('utf-8') \ No newline at end of file From 293b9b7cb97d6bd631dd60904bb17d8c2cbd68f5 Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Wed, 1 Jan 2025 12:51:33 +0200 Subject: [PATCH 11/12] feat(plugins): Create plugin list in sidebar for active plugins --- .gitignore | 4 + fields/file_fields.py | 69 ++++++++++++ plugins/admin.py | 4 +- plugins/migrations/0002_plugin_image.py | 21 ++++ plugins/models.py | 11 ++ plugins/views.py | 16 +++ sefaria/urls.py | 1 + static/js/ConnectionsPanel.jsx | 5 +- static/js/WebComponentLoader.jsx | 71 ------------ static/js/components/plugins/PluginList.jsx | 60 +++++++++++ .../components/plugins/PluginsComponent.jsx | 39 +++++++ .../components/plugins/WebComponentLoader.jsx | 101 ++++++++++++++++++ static/js/sefaria/sefaria.js | 8 ++ 13 files changed, 334 insertions(+), 76 deletions(-) create mode 100644 fields/file_fields.py create mode 100644 plugins/migrations/0002_plugin_image.py delete mode 100644 static/js/WebComponentLoader.jsx create mode 100644 static/js/components/plugins/PluginList.jsx create mode 100644 static/js/components/plugins/PluginsComponent.jsx create mode 100644 static/js/components/plugins/WebComponentLoader.jsx diff --git a/.gitignore b/.gitignore index 658cbd1902..86d78f96f5 100644 --- a/.gitignore +++ b/.gitignore @@ -116,6 +116,10 @@ venv/ ##################################### client_secrets.json +# Google Cloud Storage # +######################## +google-cloud-secret* + # Webpack # ########### /static/bundles diff --git a/fields/file_fields.py b/fields/file_fields.py new file mode 100644 index 0000000000..492eeadf70 --- /dev/null +++ b/fields/file_fields.py @@ -0,0 +1,69 @@ +# file: myapp/fields.py + +from django.db.models.fields.files import ImageField, ImageFieldFile + +from sefaria.google_storage_manager import GoogleStorageManager + +class GCSImageFieldFile(ImageFieldFile): + """ + Minimal subclass of ImageFieldFile that stores files on Google Cloud Storage (GCS). + We override `save()` and `delete()` to call GoogleStorageManager. + """ + + @property + def url(self): + """ + Return the GCS URL we stored in `self.name`. + Django normally constructs the URL from default storage, but here + we already have the public URL in `self.name`. + """ + return self.name + + def save(self, name, content, save=True): + """ + 1) Upload file to GCS via GoogleStorageManager. + 2) Store the returned public URL in `self.name`. + 3) Optionally save the model field. + """ + public_url = GoogleStorageManager.upload_file( + from_file=content.file, # file-like object + to_filename=name, # use incoming name for simplicity + bucket_name=self.field.bucket_name + ) + self.name = public_url + self._committed = True + + if save: + setattr(self.instance, self.field.name, self) + self.instance.save(update_fields=[self.field.name]) + + def delete(self, save=True): + """ + Remove file from GCS (if exists), clear self.name, optionally save. + """ + if self.name: + # Extract the actual filename from the URL, then delete from GCS + filename = GoogleStorageManager.get_filename_from_url(self.name) + if filename: + GoogleStorageManager.delete_filename( + filename=filename, + bucket_name=self.field.bucket_name + ) + self.name = None + self._committed = False + + if save: + setattr(self.instance, self.field.name, self) + self.instance.save(update_fields=[self.field.name]) + + +class GCSImageField(ImageField): + """ + Minimal custom ImageField that uses GCSImageFieldFile for storage. + Stores the public GCS URL in the database instead of a local path. + """ + attr_class = GCSImageFieldFile + + def __init__(self, bucket_name=None, *args, **kwargs): + self.bucket_name = bucket_name or GoogleStorageManager.PROFILES_BUCKET + super().__init__(*args, **kwargs) diff --git a/plugins/admin.py b/plugins/admin.py index a7df63b6bd..6d2f6427cf 100644 --- a/plugins/admin.py +++ b/plugins/admin.py @@ -1,10 +1,8 @@ from .models import Plugin from django.contrib import admin - class PluginAdmin(admin.ModelAdmin): list_display = ('name', 'url', 'secret') search_fields = ('name',) - fields = ('name', 'description', 'url') - + fields = ('name', 'description', 'url', 'image') admin.site.register(Plugin, PluginAdmin) \ No newline at end of file diff --git a/plugins/migrations/0002_plugin_image.py b/plugins/migrations/0002_plugin_image.py new file mode 100644 index 0000000000..64fde5ec8d --- /dev/null +++ b/plugins/migrations/0002_plugin_image.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-12-31 12:16 +from __future__ import unicode_literals + +from django.db import migrations +import fields.file_fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='plugin', + name='image', + field=fields.file_fields.GCSImageField(blank=True, null=True, upload_to=''), + ), + ] diff --git a/plugins/models.py b/plugins/models.py index 5d86600bd7..014e7988d1 100644 --- a/plugins/models.py +++ b/plugins/models.py @@ -1,6 +1,7 @@ from django.db import models from fields.encrypted import EncryptedCharField from cryptography.fernet import Fernet +from fields.file_fields import GCSImageField class Plugin(models.Model): name = models.CharField(max_length=100) @@ -8,6 +9,7 @@ class Plugin(models.Model): url = models.URLField() secret = EncryptedCharField(max_length=100) created_at = models.DateTimeField(auto_now_add=True) + image = GCSImageField(blank=True, null=True) # on create, generate a secret def save(self, *args, **kwargs): @@ -16,6 +18,15 @@ def save(self, *args, **kwargs): super(Plugin, self).save(*args, **kwargs) + def to_dict(self): + return { + "id": self.id, + "name": self.name, + "description": self.description, + "url": self.url, + "image": self.image.url if self.image else None, + } + def _generate_secret(self): key = Fernet.generate_key() return key.decode('utf-8') diff --git a/plugins/views.py b/plugins/views.py index 9e955a4df9..43e3f5ef8e 100644 --- a/plugins/views.py +++ b/plugins/views.py @@ -58,4 +58,20 @@ def get_user_plugin_secret(request, plugin_id): "encrypted_user_id": encrypted_user_id.decode('utf-8') } + return jsonResponse(json_response) + + +def all_plugins(request): + """ + Get all plugins. + + @query_param request: Django request object + """ + + plugins = Plugin.objects.all() + + json_response = { + "plugins": [plugin.to_dict() for plugin in plugins] + } + return jsonResponse(json_response) \ No newline at end of file diff --git a/sefaria/urls.py b/sefaria/urls.py index 9a88371306..23ddef7bb7 100644 --- a/sefaria/urls.py +++ b/sefaria/urls.py @@ -480,6 +480,7 @@ urlpatterns += [ url(r'^plugin/dev/?$', plugins_views.dev), url(r'^plugin/(?P.+)/user/?$', plugins_views.get_user_plugin_secret), + url(r'^plugin/all?$', plugins_views.all_plugins), ] if DOWN_FOR_MAINTENANCE: diff --git a/static/js/ConnectionsPanel.jsx b/static/js/ConnectionsPanel.jsx index a83239dc24..d166fdef73 100644 --- a/static/js/ConnectionsPanel.jsx +++ b/static/js/ConnectionsPanel.jsx @@ -30,7 +30,7 @@ import { AddToSourceSheetBox } from './AddToSourceSheet'; import LexiconBox from './LexiconBox'; import AboutBox from './AboutBox'; import GuideBox from './GuideBox'; -import WebComponentLoader from './WebComponentLoader' +import PluginsComponent from './components/plugins/PluginsComponent'; import TranslationsBox from './TranslationsBox'; import ExtendedNotes from './ExtendedNotes'; import classNames from 'classnames'; @@ -677,7 +677,8 @@ class ConnectionsPanel extends Component { /> } else if (this.props.mode === "Plugin") { - content = + // put data here + content = } const marginless = ["Resources", "ConnectionsList", "Advanced Tools", "Share", "WebPages", "Topics", "manuscripts"].indexOf(this.props.mode) !== -1; diff --git a/static/js/WebComponentLoader.jsx b/static/js/WebComponentLoader.jsx deleted file mode 100644 index 162df93842..0000000000 --- a/static/js/WebComponentLoader.jsx +++ /dev/null @@ -1,71 +0,0 @@ -import React, { useState } from 'react'; - -function WebComponentLoader(props) { - const [link, setLink] = useState(''); - const [loaded, setLoaded] = useState(false); - const [pluginName, setPluginName] = useState('sefaria-plugin'); - const sref = props.sref; - - const repoToRawLink = (link, target) => { - const repo = link.split('github.com/')[1].split('/'); - const JsUrl = `https://${repo[0]}.github.io/${repo[1]}/plugin.js` - const middlewareLink = `/plugin/dev?target=${target}&plugin_url=${JsUrl}` - return middlewareLink - } - - const getPluginUser = (pluginId=1) => { - fetch(`/plugin/${pluginId}/user`) - .then(res => res.json()) - .then(data => { - console.log(data); - }); -} - - let script = null; - let rand = Math.floor(Math.random() * 1000); - - const handleClick = () => { - if (script) { - document.head.removeChild(script); - setLoaded(false); - } - if (link) { - const target = `sefaria-plugin-${rand}` - setPluginName(target); - script = document.createElement('script'); - script.src = repoToRawLink(link, target); - script.async = true; - script.onload = () => { - setLoaded(true); - }; - document.head.appendChild(script); - } - - getPluginUser(); - - }; - - if (loaded) { - const PluginElm = pluginName; - return ( -
- - -
- ); - } - - return ( -
- setLink(e.target.value)} - /> - -
- ); -} - -export default WebComponentLoader; diff --git a/static/js/components/plugins/PluginList.jsx b/static/js/components/plugins/PluginList.jsx new file mode 100644 index 0000000000..b0cc684d57 --- /dev/null +++ b/static/js/components/plugins/PluginList.jsx @@ -0,0 +1,60 @@ +import React, { useEffect, useState } from 'react'; +import WebComponentLoader from './WebComponentLoader'; +import sefaria from '../../sefaria/sefaria'; + +const PluginList = (props) => { + const [plugins, setPlugins] = useState([]); + const [pluginLink, setPluginLink] = useState(''); + const setShowPluginOptions = props.setShowPluginOptions; + + useEffect(() => { + const fetchPlugins = async () => { + try { + const data = await sefaria.getPlugins(); + setPlugins(data.plugins); + } catch (error) { + console.error('Error fetching plugins:', error); + } + }; + + fetchPlugins(); + }, []); + + const pluginButton = (plugin) => { + return ( + + ) + } + + const pluginMenu = ( +
+
+ + Active Plugins + +
+
+ {plugins.map((plugin) => ( pluginButton(plugin) ))} +
+
+ ); + + return ( +
+ {pluginLink ? ( + + ) : pluginMenu} +
+ ); +}; + +export default PluginList; \ No newline at end of file diff --git a/static/js/components/plugins/PluginsComponent.jsx b/static/js/components/plugins/PluginsComponent.jsx new file mode 100644 index 0000000000..adc8fb4351 --- /dev/null +++ b/static/js/components/plugins/PluginsComponent.jsx @@ -0,0 +1,39 @@ +import React, { useState } from 'react'; +import PluginList from './PluginList'; +import WebComponentLoader from './WebComponentLoader'; + +const PluginsComponent = (props) => { + const [showDevelopers, setShowDevelopers] = useState(false); + const [showPluginOptions, setShowPluginOptions] = useState(true); + + const toggleComponent = () => { + setShowDevelopers(!showDevelopers); + }; + + const pluginListCreateToggle = ( + {showDevelopers ? 'Plugin List' : 'Create'} + ) + + return ( +
+ {showPluginOptions && pluginListCreateToggle} + {showDevelopers ? : } +
+ ); +}; + +export default PluginsComponent; \ No newline at end of file diff --git a/static/js/components/plugins/WebComponentLoader.jsx b/static/js/components/plugins/WebComponentLoader.jsx new file mode 100644 index 0000000000..072bf482e9 --- /dev/null +++ b/static/js/components/plugins/WebComponentLoader.jsx @@ -0,0 +1,101 @@ +import React, { useState, useEffect } from 'react'; + +function WebComponentLoader(props) { + const [link, setLink] = useState(''); + const [loaded, setLoaded] = useState(false); + const [pluginName, setPluginName] = useState('sefaria-plugin'); + const sref = props.sref; + const pluginLink = props.pluginLink; + const isDeveloper = props.isDeveloper; + const setShowPluginOptions = props.setShowPluginOptions; + + useEffect(() => { + if (pluginLink) { + setLink(pluginLink); + loadPlugin(pluginLink); + } + }, [pluginLink]); + + const repoToRawLink = (link, target) => { + const repo = link.split('github.com/')[1].split('/'); + const JsUrl = `https://${repo[0]}.github.io/${repo[1]}/plugin.js`; + const middlewareLink = `/plugin/dev?target=${target}&plugin_url=${JsUrl}`; + return middlewareLink; + }; + + const getPluginUser = (pluginId = 1) => { + fetch(`/plugin/${pluginId}/user`) + .then(res => res.json()) + .then(data => { + console.log(data); + }); + }; + + let script = null; + let rand = Math.floor(Math.random() * 1000); + + const loadPlugin = (pluginLink) => { + setShowPluginOptions(false); + if (script) { + document.head.removeChild(script); + setLoaded(false); + } + if (pluginLink) { + const target = `sefaria-plugin-${rand}`; + setPluginName(target); + script = document.createElement('script'); + script.src = repoToRawLink(pluginLink, target); + script.async = true; + script.onload = () => { + setLoaded(true); + }; + document.head.appendChild(script); + } + + getPluginUser(); + }; + + if (loaded) { + const PluginElm = pluginName; + return ( +
+ {isDeveloper && } + +
+ ); + } + else if (isDeveloper) { + return ( +
+ setLink(e.target.value)} + /> + +
+ ); + } + else { + return ( +
+

Loading...

+
+ ); + } +} + +export default WebComponentLoader; diff --git a/static/js/sefaria/sefaria.js b/static/js/sefaria/sefaria.js index 520fde8d89..a970d46edd 100644 --- a/static/js/sefaria/sefaria.js +++ b/static/js/sefaria/sefaria.js @@ -2494,6 +2494,14 @@ _media: {}, store: Sefaria._profiles }); }, + _plugins: {}, + getPlugins: () => { + return Sefaria._cachedApiPromise({ + url: Sefaria.apiHost + "/plugin/all", + key: "plugins", + store: Sefaria._plugins + }); + }, userHistory: {loaded: false, items: []}, loadUserHistory: function (limit, callback) { const skip = Sefaria.userHistory.items.length; From 15229439ccfe700e53a4a6b4cc2aebfccd3ba7f2 Mon Sep 17 00:00:00 2001 From: Akiva Berger Date: Wed, 1 Jan 2025 16:03:51 +0200 Subject: [PATCH 12/12] feat(plugins): add scroll to ref feature --- .../components/plugins/WebComponentLoader.jsx | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/static/js/components/plugins/WebComponentLoader.jsx b/static/js/components/plugins/WebComponentLoader.jsx index 072bf482e9..0875ec7ab0 100644 --- a/static/js/components/plugins/WebComponentLoader.jsx +++ b/static/js/components/plugins/WebComponentLoader.jsx @@ -48,6 +48,7 @@ function WebComponentLoader(props) { script.async = true; script.onload = () => { setLoaded(true); + addEventListenerToPlugin(target); }; document.head.appendChild(script); } @@ -55,12 +56,32 @@ function WebComponentLoader(props) { getPluginUser(); }; + const addEventListenerToPlugin = (target) => { + const pluginElement = document.querySelector(target); + if (pluginElement) { + pluginElement.addEventListener('scrollToRef', (event) => { + scrollToRef(event.detail.sref); + }); + } + }; + + const scrollToRef = (sref) => { + if (sref) { + const query = `div[data-ref="${sref}"]`; + const element = document.querySelectorAll(query)[0]; + if (element) { + element.scrollIntoView(); + element.parentElement.parentElement.parentElement.parentElement.parentElement.scrollBy(0, -40); + } + } + }; + if (loaded) { const PluginElm = pluginName; return (
{isDeveloper && } - +
); }