From 1bdc95b71361eb75520ccd9636c6d013e79cdbe6 Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Sun, 1 Oct 2023 19:05:59 -0500 Subject: [PATCH 01/15] Implement new providers unified callback-based API --- dialect/preferences.py | 4 +- dialect/providers/__init__.py | 14 +- dialect/providers/base.py | 371 ++++++------- dialect/providers/bing.py | 187 ++++--- dialect/providers/google.py | 697 ++++++++++++++++-------- dialect/providers/libretrans.py | 233 ++++---- dialect/providers/lingva.py | 166 +++--- dialect/providers/yandex.py | 200 +++++-- dialect/widgets/provider_preferences.py | 38 +- dialect/window.py | 504 +++++++---------- 10 files changed, 1348 insertions(+), 1066 deletions(-) diff --git a/dialect/preferences.py b/dialect/preferences.py index 0bc4e27b..7e2de213 100644 --- a/dialect/preferences.py +++ b/dialect/preferences.py @@ -8,7 +8,7 @@ from dialect.define import RES_PATH from dialect.settings import Settings -from dialect.providers import ProvidersListModel, MODULES, TTS +from dialect.providers import ProviderFeature, ProvidersListModel, MODULES, TTS from dialect.widgets import ProviderPreferences @@ -90,7 +90,7 @@ def _provider_has_settings(self, name): if not name: return False - if MODULES[name].change_instance or MODULES[name].api_key_supported: + if ProviderFeature.INSTANCES in MODULES[name].features or ProviderFeature.API_KEY in MODULES[name].features: return True return False diff --git a/dialect/providers/__init__.py b/dialect/providers/__init__.py index d50bbadf..ffbeed21 100644 --- a/dialect/providers/__init__.py +++ b/dialect/providers/__init__.py @@ -7,17 +7,21 @@ from gi.repository import Gio, GObject +from dialect.providers.base import ProviderCapability, ProviderFeature, ProviderError, ProviderErrorCode # noqa + + MODULES = {} TRANSLATORS = {} TTS = {} for _importer, modname, _ispkg in pkgutil.iter_modules(__path__): - if modname != 'base': + if modname != 'base' and not modname.startswith('_'): modclass = importlib.import_module('dialect.providers.' + modname).Provider MODULES[modclass.name] = modclass - if modclass.translation: - TRANSLATORS[modclass.name] = modclass - if modclass.tts: - TTS[modclass.name] = modclass + if modclass.capabilities: + if ProviderCapability.TRANSLATION in modclass.capabilities: + TRANSLATORS[modclass.name] = modclass + if ProviderCapability.TTS in modclass.capabilities: + TTS[modclass.name] = modclass def check_translator_availability(provider_name): diff --git a/dialect/providers/base.py b/dialect/providers/base.py index f4c5bbb7..84dc9510 100644 --- a/dialect/providers/base.py +++ b/dialect/providers/base.py @@ -6,38 +6,93 @@ import json import logging import urllib.parse +import threading +from enum import Enum, Flag, auto +from typing import Callable from gi.repository import GLib, Gio, Soup from dialect.define import APP_ID from dialect.languages import get_lang_name, normalize_lang_code +from dialect.session import Session + + +class ProviderCapability(Flag): + TRANSLATION = auto() + """ If it provides translation """ + TTS = auto() + """ If it provides text-to-speech """ + DEFINITIONS = auto() + """ If it provides dictionary definitions """ + + +class ProviderFeature(Flag): + INSTANCES = auto() + """ If it supports changing the instance url """ + API_KEY = auto() + """ If the api key is supported but not necessary """ + API_KEY_REQUIRED = auto() + """ If the api key is required for the provider to work """ + DETECTION = auto() + """ If it supports detecting text language (Auto translation) """ + MISTAKES = auto() + """ If it supports showing translation mistakes """ + PRONUNCIATION = auto() + """ If it supports showing translation pronunciation """ + SUGGESTIONS = auto() + """ If it supports sending translation suggestions to the service """ + + +class ProviderErrorCode(Enum): + UNEXPECTED = auto() + NETWORK = auto + EMPTY = auto() + API_KEY_REQUIRED = auto() + API_KEY_INVALID = auto() + INVALID_LANG_CODE = auto() + BATCH_SIZE_EXCEEDED = auto() + CHARACTERS_LIMIT_EXCEEDED = auto() + SERVICE_LIMIT_REACHED = auto() + TRANSLATION_FAILED = auto() + TTS_FAILED = auto() + + +class ProviderError: + """Helper error handing class to be passed between callbacks""" + + def __init__(self, code: ProviderErrorCode, message: str = '') -> None: + self.code = code # Serves for quick error matching + self.message = message # More detailed error info if needed + + +class Translation: + text = None + extra_data = { + 'possible-mistakes': [None, None], + 'src-pronunciation': None, + 'dest-pronunciation': None, + } + + def __init__(self, text, extra_data): + self.text = text + self.extra_data = extra_data class BaseProvider: - __provider_type__ = '' - """ The type of engine used by the provider - str or dict if you want to use diferent engines per feature - """ name = '' """ Module name for itern use, like settings storing """ prettyname = '' """ Module name for UI display """ - translation = False - """ If it provides translation """ - tts = False - """ If it provides text-to-speech """ - definitions = False - """ If it provides dict definitions """ - change_instance = False - """ If it supports changing the instance url """ - api_key_supported = False - """ If it supports setting api keys """ + capabilities: ProviderCapability | None = None + """ Provider capabilities, translation, tts, etc """ + features: ProviderFeature | None = None + """ Provider features """ defaults = { 'instance_url': '', 'api_key': '', 'src_langs': ['en', 'fr', 'es', 'de'], - 'dest_langs': ['fr', 'es', 'de', 'en'] + 'dest_langs': ['fr', 'es', 'de', 'en'], } def __init__(self): @@ -52,22 +107,17 @@ def __init__(self): self.chars_limit = -1 """ Translation char limit """ - self.detection = False - """ If it supports deteting text language (Auto translation) """ - self.mistakes = False - """ If it supports showing translation mistakes """ - self.pronunciation = False - """ If it supports showing translation pronunciation """ - self.suggestions = False - """ If it supports sending translation suggestions to the service """ - self.api_key_required = False - """ If the api key is required for the provider to work """ + self.history = [] """ Here we save the translation history """ # GSettings self.settings = Gio.Settings(f'{APP_ID}.translator', f'/app/drey/Dialect/translators/{self.name}/') + """ + Provider settings helpers and properties + """ + @property def instance_url(self): return self.settings.get_string('instance-url') or self.defaults['instance_url'] @@ -112,9 +162,13 @@ def dest_langs(self, dest_langs): def reset_dest_langs(self): self.dest_langs = [] + """ + General provider helpers + """ + @staticmethod def format_url(url: str, path: str = '', params: dict = {}, http: bool = False): - """ Formats a given url with path with the https protocol """ + """Formats a given url with path with the https protocol""" if not path.startswith('/'): path = '/' + path @@ -130,7 +184,7 @@ def format_url(url: str, path: str = '', params: dict = {}, http: bool = False): return protocol + url + path + params_str def add_lang(self, original_code, name=None, trans=True, tts=False): - """ Add lang supported by provider """ + """Add lang supported by provider""" code = normalize_lang_code(original_code) # Get normalized lang code @@ -148,7 +202,7 @@ def add_lang(self, original_code, name=None, trans=True, tts=False): self._languages_names[code] = name def denormalize_lang(self, *codes): - """ Get denormalized lang code if available """ + """Get denormalized lang code if available""" if len(codes) == 1: return self._nonstandard_langs.get(codes[0], codes[0]) @@ -159,7 +213,7 @@ def denormalize_lang(self, *codes): return tuple(result) def get_lang_name(self, code): - """ Get language name """ + """Get language name""" name = get_lang_name(code) # Try getting translated name from Dialect if name is None: # Get name from provider if available @@ -167,51 +221,70 @@ def get_lang_name(self, code): return name + """ + Providers API methods + """ -class LocalProvider(BaseProvider): - """ Base class for providers using the local threaded engine """ - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def init_trans(self): - pass - - def init_tts(self): - pass - - def init_def(self): - pass + @staticmethod + def validate_instance(url: str, on_done: Callable[[bool], None], on_fail: Callable[[ProviderError], None]): + raise NotImplementedError() + + def validate_api_key(self, key: str, on_done: Callable[[bool], None], on_fail: Callable[[ProviderError], None]): + raise NotImplementedError() + + def init_trans(self, on_done: Callable, on_fail: Callable[[ProviderError], None]): + on_done() + + def init_tts(self, on_done: Callable, on_fail: Callable[[ProviderError], None]): + on_done() + + def translate( + self, + text: str, + src: str, + dest: str, + on_done: Callable[[Translation, str | None], None], + on_fail: Callable[[ProviderError], None], + ): + raise NotImplementedError() + + def suggest( + self, + text: str, + src: str, + dest: str, + suggestion: str, + on_done: Callable[[bool], None], + on_fail: Callable[[ProviderError], None], + ): + raise NotImplementedError() + + def speech( + self, + text: str, + language: str, + on_done: Callable[[io.BytesIO], None], + on_fail: Callable[[ProviderError], None], + ): + raise NotImplementedError() - def translate(self, text: str, src: str, dest: str): - pass - def suggest(self, text: str, src: str, dest: str, suggestion: str) -> bool: - pass +class LocalProvider(BaseProvider): + """Base class for providers needing local threaded helpers""" - def download_speech(self, text: str, language: str, file: io.BytesIO): - pass + def launch_thread(self, callback: Callable, *args): + threading.Thread(target=callback, args=args, daemon=True).start() class SoupProvider(BaseProvider): - """ Base class for providers using the libsoup engine """ - - trans_init_requests = [] - """ List of request to do before using the provider """ - tts_init_requests = [] - """ List of request to do before using the provider """ - def_init_requests = [] - """ List of request to do before using the provider """ + """Base class for providers needing libsoup helpers""" def __init__(self, **kwargs): super().__init__(**kwargs) - self.error = '' - """ Loading error when initializing """ - @staticmethod def encode_data(data) -> GLib.Bytes | None: - """ Convert dict to JSON and bytes """ + """Convert dict to JSON and bytes""" data_glib_bytes = None try: data_bytes = json.dumps(data).encode('utf-8') @@ -222,14 +295,18 @@ def encode_data(data) -> GLib.Bytes | None: @staticmethod def read_data(data: bytes) -> dict: - """ Get JSON data from bytes """ - return json.loads( - data - ) if data else {} + """Get JSON data from bytes""" + return json.loads(data) if data else {} + + @staticmethod + def read_response(session: Session, result: Gio.AsyncResult) -> dict: + """Get JSON data from session result""" + response = session.get_response(session, result) + return SoupProvider.read_data(response) @staticmethod - def create_request(method: str, url: str, data={}, headers: dict = {}, form: bool = False) -> Soup.Message: - """ Helper for creating Soup.Message """ + def create_message(method: str, url: str, data={}, headers: dict = {}, form: bool = False) -> Soup.Message: + """Helper for creating libsoup's message""" if form and data: form_data = Soup.form_encode_hash(data) @@ -247,130 +324,58 @@ def create_request(method: str, url: str, data={}, headers: dict = {}, form: boo return message @staticmethod - def format_validate_instance(url: str) -> Soup.Message: - pass + def send_and_read(message: Soup.Message, callback: Callable[[Session, Gio.AsyncResult], None]): + """Helper method for libsoup's send_and_read_async + Useful when priority and cancellable is not needed""" + Session.get().send_and_read_async(message, 0, None, callback) @staticmethod - def validate_instance(data: bytes) -> bool: - pass - - def format_validate_api_key(self, api_key: str) -> Soup.Message: - pass - - def validate_api_key(self, data: bytes): - pass - - def format_translation(self, text: str, src: str, dest: str) -> Soup.Message: - pass - - def get_translation(self, data: bytes): - pass - - def format_suggestion(self, text: str, src: str, dest: str, suggestion: str) -> Soup.Message: - pass - - def get_suggestion(self, data: bytes) -> bool: - pass - - def format_speech(self, text: str, language: str) -> Soup.Message: - pass - - def get_speech(self, data: bytes, file: io.BytesIO): - pass - - -class ProviderError(Exception): - """Base Exception for Translator related errors.""" - - def __init__(self, cause, message='Translator Error'): - self.cause = cause - self.message = message - super().__init__(self.message) - - def __str__(self): - return f'{self.message}: {self.cause}' - - -class ApiKeyRequired(ProviderError): - """Exception raised when API key is required.""" - - def __init__(self, cause, message='API Key Required'): - self.cause = cause - self.message = message - super().__init__(self.cause, self.message) - - -class InvalidApiKey(ProviderError): - """Exception raised when an invalid API key is found.""" - - def __init__(self, cause, message='Invalid API Key'): - self.cause = cause - self.message = message - super().__init__(self.cause, self.message) - - -class InvalidLangCode(ProviderError): - """Exception raised when an invalid lang code is sent.""" - - def __init__(self, cause, message='Invalid Lang Code'): - self.cause = cause - self.message = message - super().__init__(self.cause, self.message) - - -class BatchSizeExceeded(ProviderError): - """Exception raised when the batch size limit has been exceeded.""" - - def __init__(self, cause, message='Batch Size Exceeded'): - self.cause = cause - self.message = message - super().__init__(self.cause, self.message) - - -class CharactersLimitExceeded(ProviderError): - """Exception raised when the char limit has been exceeded.""" - - def __init__(self, cause, message='Characters Limit Exceeded'): - self.cause = cause - self.message = message - super().__init__(self.cause, self.message) - - -class ServiceLimitReached(ProviderError): - """Exception raised when the service limit has been reached.""" - - def __init__(self, cause, message='Service Limit Reached'): - self.cause = cause - self.message = message - super().__init__(self.cause, self.message) + def check_known_errors(data: dict) -> None | ProviderError: + """Checks data for possible response errors and return a found error if any + This should be implemented by subclases""" + return None + @staticmethod + def process_response( + session: Session, + result: Gio.AsyncResult, + on_continue: Callable[[dict], None], + on_fail: Callable[[ProviderError], None], + check_common: bool = True, + ): + """Helper method for the most common workflow for processing soup responses + + Checks for soup errors, then checks for common errors on data and calls on_fail + if any, otherwise calls on_continue where the provider will finish the process. + """ -class TranslationError(ProviderError): - """Exception raised when translation fails.""" + try: + data = SoupProvider.read_response(session, result) - def __init__(self, cause, message='Translation has failed'): - self.cause = cause - self.message = message - super().__init__(self.cause, self.message) + if check_common: + error = SoupProvider.check_known_errors(data) + if error: + on_fail(error) + return + on_continue(data) -class TextToSpeechError(ProviderError): - """Exception raised when tts fails.""" + except Exception as exc: + logging.warning(exc) + on_fail(ProviderError(ProviderErrorCode.NETWORK, str(exc))) - def __init__(self, cause, message='Text to Speech has failed'): - self.cause = cause - self.message = message - super().__init__(self.message) + @staticmethod + def send_and_read_and_process_response( + message: Soup.Message, + on_continue: Callable[[dict], None], + on_fail: Callable[[ProviderError], None], + check_common: bool = True, + ): + """Helper packaging send_and_read and process_response + Avoids implementors having to deal with many callbacks.""" -class Translation: - text = None - extra_data = { - 'possible-mistakes': None, - 'src-pronunciation': None, - 'dest-pronunciation': None, - } + def on_response(session: Session, result: Gio.AsyncResult): + SoupProvider.process_response(session, result, on_continue, on_fail, check_common) - def __init__(self, text, extra_data): - self.text = text - self.extra_data = extra_data + SoupProvider.send_and_read(message, on_response) diff --git a/dialect/providers/bing.py b/dialect/providers/bing.py index aa7bf4f0..4810dd97 100644 --- a/dialect/providers/bing.py +++ b/dialect/providers/bing.py @@ -7,43 +7,38 @@ from bs4 import BeautifulSoup from dialect.providers.base import ( - ProviderError, SoupProvider, Translation, TranslationError + ProviderCapability, + ProviderFeature, + ProviderError, + ProviderErrorCode, + SoupProvider, + Translation, ) +from dialect.session import Session class Provider(SoupProvider): - __provider_type__ = 'soup' - name = 'bing' prettyname = 'Bing' - translation = True - tts = False - definitions = False - change_instance = False - api_key_supported = False + + capabilities = ProviderCapability.TRANSLATION + features = ProviderFeature.DETECTION | ProviderFeature.PRONUNCIATION + defaults = { 'instance_url': '', 'api_key': '', 'src_langs': ['en', 'fr', 'es', 'de'], - 'dest_langs': ['fr', 'es', 'de', 'en'] + 'dest_langs': ['fr', 'es', 'de', 'en'], } - trans_init_requests = [ - 'parse_html' - ] - - _headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', - 'Accept': '*/*' - } + _headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', 'Accept': '*/*'} def __init__(self, **kwargs): super().__init__(**kwargs) self.chars_limit = 1000 # Web UI limit - self.detection = True - self.pronunciation = True + # Session vars self._key = '' self._token = '' self._ig = '' @@ -64,93 +59,103 @@ def translate_url(self): } return self.format_url('www.bing.com', '/ttranslatev3', params) - def format_parse_html_init(self): - return self.create_request('GET', self.html_url, headers=self._headers) - - def parse_html_init(self, data): - if data: + def init_trans(self, on_done, on_fail): + def on_response(session, result): try: - soup = BeautifulSoup(data, 'html.parser') + data = Session.get_response(session, result) + if data: + try: + soup = BeautifulSoup(data, 'html.parser') - # Get Langs - langs = soup.find('optgroup', {'id': 't_tgtAllLang'}) - for child in langs.findChildren(): - if child.name == 'option': - self.languages.append(child['value']) + # Get Langs + langs = soup.find('optgroup', {'id': 't_tgtAllLang'}) + for child in langs.findChildren(): + if child.name == 'option': + self.languages.append(child['value']) - # Get IID - iid = soup.find('div', {'id': 'rich_tta'}) - self._iid = iid['data-iid'] + # Get IID + iid = soup.find('div', {'id': 'rich_tta'}) + self._iid = iid['data-iid'] - # Decode response bytes - data = data.decode('utf-8') + # Decode response bytes + data = data.decode('utf-8') - # Look for abuse prevention data - params = re.findall("var params_AbusePreventionHelper = \[(.*?)\];", data)[0] - abuse_params = params.replace('"', '').split(',') - self._key = abuse_params[0] - self._token = abuse_params[1] + # Look for abuse prevention data + params = re.findall("var params_AbusePreventionHelper = \[(.*?)\];", data)[0] # noqa + abuse_params = params.replace('"', '').split(',') + self._key = abuse_params[0] + self._token = abuse_params[1] - # Look for IG - self._ig = re.findall("IG:\"(.*?)\",", data)[0] + # Look for IG + self._ig = re.findall("IG:\"(.*?)\",", data)[0] - except Exception as exc: - self.error = 'Failed parsing HTML from bing.com' - logging.warning(self.error, str(exc)) + on_done() - else: - self.error = 'Could not get HTML from bing.com' - logging.warning(self.error) - - def format_translation(self, text, src, dest): - data = { - 'fromLang': 'auto-detect', - 'text': text, - 'to': dest, - 'token': self._token, - 'key': self._key - } + except Exception as exc: + error = 'Failed parsing HTML from bing.com' + logging.warning(error, exc) + on_fail(ProviderError(ProviderErrorCode.NETWORK, error)) - if src != 'auto': - data['fromLang'] = src + else: + on_fail(ProviderError(ProviderErrorCode.EMPTY, 'Could not get HTML from bing.com')) - return self.create_request('POST', self.translate_url, data, self._headers, True) - - def get_translation(self, data): - self._count += 1 # Increment requests count - - data = self.read_data(data) - self._check_errors(data) - - try: - data = data[0] - detected = None - pronunciation = None + except Exception as exc: + logging.warning(exc) + on_fail(ProviderError(ProviderErrorCode.NETWORK, str(exc))) - if 'translations' in data: + # Message request to get bing's website html + message = self.create_message('GET', self.html_url, headers=self._headers) + # Do async request + self.send_and_read(message, on_response) - if 'detectedLanguage' in data: - detected = data['detectedLanguage']['language'] + def translate(self, text, src, dest, on_done, on_fail): + def on_response(data): + try: + data = data[0] + detected = None + pronunciation = None + + if 'translations' in data: + if 'detectedLanguage' in data: + detected = data['detectedLanguage']['language'] + + if 'transliteration' in data['translations'][0]: + pronunciation = data['translations'][0]['transliteration']['text'] + + translation = Translation( + data['translations'][0]['text'], + { + 'possible-mistakes': [None, None], + 'src-pronunciation': None, + 'dest-pronunciation': pronunciation, + }, + ) + on_done(translation, detected) - if 'transliteration' in data['translations'][0]: - pronunciation = data['translations'][0]['transliteration']['text'] + except Exception as exc: + logging.warning(exc) + on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, str(exc))) - result = Translation( - data['translations'][0]['text'], - { - 'possible-mistakes': None, - 'src-pronunciation': None, - 'dest-pronunciation': pronunciation, - } - ) - return (result, detected) + # Increment requests count + self._count += 1 - except Exception as exc: - raise TranslationError(str(exc)) + # Form data + data = { + 'fromLang': 'auto-detect' if src == 'auto' else src, + 'text': text, + 'to': dest, + 'token': self._token, + 'key': self._key, + } + # Request message + message = self.create_message('POST', self.translate_url, data, self._headers, True) + # Do async request + self.send_and_read_and_process_response(message, on_response, on_fail) - def _check_errors(self, data): + @staticmethod + def check_known_errors(data): if not data: - raise ProviderError('Request empty') + return ProviderError(ProviderErrorCode.EMPTY, 'Response is empty!') if 'errorMessage' in data: error = data['errorMessage'] @@ -158,4 +163,6 @@ def _check_errors(self, data): match code: case _: - raise ProviderError(error) + return ProviderError(ProviderErrorCode.UNEXPECTED, error) + + return None diff --git a/dialect/providers/google.py b/dialect/providers/google.py index 64c80b25..e33b4e5d 100644 --- a/dialect/providers/google.py +++ b/dialect/providers/google.py @@ -4,13 +4,22 @@ import html import json +import logging import random import re +from tempfile import NamedTemporaryFile from typing import List + from gtts import gTTS, lang from dialect.providers.base import ( - LocalProvider, ProviderError, SoupProvider, Translation, TranslationError, TextToSpeechError + LocalProvider, + ProviderCapability, + ProviderError, + ProviderErrorCode, + ProviderFeature, + SoupProvider, + Translation, ) RPC_ID = 'MkEWBc' @@ -18,91 +27,217 @@ # Predefined URLs used to make google translate requests. TRANSLATE_RPC = '{host}/_/TranslateWebserverUi/data/batchexecute' -DEFAULT_SERVICE_URLS = ('translate.google.ac', 'translate.google.ad', 'translate.google.ae', - 'translate.google.al', 'translate.google.am', 'translate.google.as', - 'translate.google.at', 'translate.google.az', 'translate.google.ba', - 'translate.google.be', 'translate.google.bf', 'translate.google.bg', - 'translate.google.bi', 'translate.google.bj', 'translate.google.bs', - 'translate.google.bt', 'translate.google.by', 'translate.google.ca', - 'translate.google.cat', 'translate.google.cc', 'translate.google.cd', - 'translate.google.cf', 'translate.google.cg', 'translate.google.ch', - 'translate.google.ci', 'translate.google.cl', 'translate.google.cm', - 'translate.google.cn', 'translate.google.co.ao', 'translate.google.co.bw', - 'translate.google.co.ck', 'translate.google.co.cr', 'translate.google.co.id', - 'translate.google.co.il', 'translate.google.co.in', 'translate.google.co.jp', - 'translate.google.co.ke', 'translate.google.co.kr', 'translate.google.co.ls', - 'translate.google.co.ma', 'translate.google.co.mz', 'translate.google.co.nz', - 'translate.google.co.th', 'translate.google.co.tz', 'translate.google.co.ug', - 'translate.google.co.uk', 'translate.google.co.uz', 'translate.google.co.ve', - 'translate.google.co.vi', 'translate.google.co.za', 'translate.google.co.zm', - 'translate.google.co.zw', 'translate.google.com.af', 'translate.google.com.ag', - 'translate.google.com.ai', 'translate.google.com.ar', 'translate.google.com.au', - 'translate.google.com.bd', 'translate.google.com.bh', 'translate.google.com.bn', - 'translate.google.com.bo', 'translate.google.com.br', 'translate.google.com.bz', - 'translate.google.com.co', 'translate.google.com.cu', 'translate.google.com.cy', - 'translate.google.com.do', 'translate.google.com.ec', 'translate.google.com.eg', - 'translate.google.com.et', 'translate.google.com.fj', 'translate.google.com.gh', - 'translate.google.com.gi', 'translate.google.com.gt', 'translate.google.com.hk', - 'translate.google.com.jm', 'translate.google.com.kh', 'translate.google.com.kw', - 'translate.google.com.lb', 'translate.google.com.ly', 'translate.google.com.mm', - 'translate.google.com.mt', 'translate.google.com.mx', 'translate.google.com.my', - 'translate.google.com.na', 'translate.google.com.ng', 'translate.google.com.ni', - 'translate.google.com.np', 'translate.google.com.om', 'translate.google.com.pa', - 'translate.google.com.pe', 'translate.google.com.pg', 'translate.google.com.ph', - 'translate.google.com.pk', 'translate.google.com.pr', 'translate.google.com.py', - 'translate.google.com.qa', 'translate.google.com.sa', 'translate.google.com.sb', - 'translate.google.com.sg', 'translate.google.com.sl', 'translate.google.com.sv', - 'translate.google.com.tj', 'translate.google.com.tr', 'translate.google.com.tw', - 'translate.google.com.ua', 'translate.google.com.uy', 'translate.google.com.vc', - 'translate.google.com.vn', 'translate.google.com', 'translate.google.cv', - 'translate.google.cz', 'translate.google.de', 'translate.google.dj', - 'translate.google.dk', 'translate.google.dm', 'translate.google.dz', - 'translate.google.ee', 'translate.google.es', 'translate.google.fi', - 'translate.google.fm', 'translate.google.fr', 'translate.google.ga', - 'translate.google.ge', 'translate.google.gg', 'translate.google.gl', - 'translate.google.gm', 'translate.google.gp', 'translate.google.gr', - 'translate.google.gy', 'translate.google.hn', 'translate.google.hr', - 'translate.google.ht', 'translate.google.hu', 'translate.google.ie', - 'translate.google.im', 'translate.google.iq', 'translate.google.is', - 'translate.google.it', 'translate.google.je', 'translate.google.jo', - 'translate.google.kg', 'translate.google.ki', 'translate.google.kz', - 'translate.google.la', 'translate.google.li', 'translate.google.lk', - 'translate.google.lt', 'translate.google.lu', 'translate.google.lv', - 'translate.google.md', 'translate.google.me', 'translate.google.mg', - 'translate.google.mk', 'translate.google.ml', 'translate.google.mn', - 'translate.google.ms', 'translate.google.mu', 'translate.google.mv', - 'translate.google.mw', 'translate.google.ne', 'translate.google.nl', - 'translate.google.no', 'translate.google.nr', 'translate.google.nu', - 'translate.google.pl', 'translate.google.pn', 'translate.google.ps', - 'translate.google.pt', 'translate.google.ro', 'translate.google.rs', - 'translate.google.ru', 'translate.google.rw', 'translate.google.sc', - 'translate.google.se', 'translate.google.sh', 'translate.google.si', - 'translate.google.sk', 'translate.google.sm', 'translate.google.sn', - 'translate.google.so', 'translate.google.sr', 'translate.google.st', - 'translate.google.td', 'translate.google.tg', 'translate.google.tk', - 'translate.google.tl', 'translate.google.tm', 'translate.google.tn', - 'translate.google.to', 'translate.google.tt', 'translate.google.us', - 'translate.google.vg', 'translate.google.vu', 'translate.google.ws') +DEFAULT_SERVICE_URLS = ( + 'translate.google.ac', + 'translate.google.ad', + 'translate.google.ae', + 'translate.google.al', + 'translate.google.am', + 'translate.google.as', + 'translate.google.at', + 'translate.google.az', + 'translate.google.ba', + 'translate.google.be', + 'translate.google.bf', + 'translate.google.bg', + 'translate.google.bi', + 'translate.google.bj', + 'translate.google.bs', + 'translate.google.bt', + 'translate.google.by', + 'translate.google.ca', + 'translate.google.cat', + 'translate.google.cc', + 'translate.google.cd', + 'translate.google.cf', + 'translate.google.cg', + 'translate.google.ch', + 'translate.google.ci', + 'translate.google.cl', + 'translate.google.cm', + 'translate.google.cn', + 'translate.google.co.ao', + 'translate.google.co.bw', + 'translate.google.co.ck', + 'translate.google.co.cr', + 'translate.google.co.id', + 'translate.google.co.il', + 'translate.google.co.in', + 'translate.google.co.jp', + 'translate.google.co.ke', + 'translate.google.co.kr', + 'translate.google.co.ls', + 'translate.google.co.ma', + 'translate.google.co.mz', + 'translate.google.co.nz', + 'translate.google.co.th', + 'translate.google.co.tz', + 'translate.google.co.ug', + 'translate.google.co.uk', + 'translate.google.co.uz', + 'translate.google.co.ve', + 'translate.google.co.vi', + 'translate.google.co.za', + 'translate.google.co.zm', + 'translate.google.co.zw', + 'translate.google.com.af', + 'translate.google.com.ag', + 'translate.google.com.ai', + 'translate.google.com.ar', + 'translate.google.com.au', + 'translate.google.com.bd', + 'translate.google.com.bh', + 'translate.google.com.bn', + 'translate.google.com.bo', + 'translate.google.com.br', + 'translate.google.com.bz', + 'translate.google.com.co', + 'translate.google.com.cu', + 'translate.google.com.cy', + 'translate.google.com.do', + 'translate.google.com.ec', + 'translate.google.com.eg', + 'translate.google.com.et', + 'translate.google.com.fj', + 'translate.google.com.gh', + 'translate.google.com.gi', + 'translate.google.com.gt', + 'translate.google.com.hk', + 'translate.google.com.jm', + 'translate.google.com.kh', + 'translate.google.com.kw', + 'translate.google.com.lb', + 'translate.google.com.ly', + 'translate.google.com.mm', + 'translate.google.com.mt', + 'translate.google.com.mx', + 'translate.google.com.my', + 'translate.google.com.na', + 'translate.google.com.ng', + 'translate.google.com.ni', + 'translate.google.com.np', + 'translate.google.com.om', + 'translate.google.com.pa', + 'translate.google.com.pe', + 'translate.google.com.pg', + 'translate.google.com.ph', + 'translate.google.com.pk', + 'translate.google.com.pr', + 'translate.google.com.py', + 'translate.google.com.qa', + 'translate.google.com.sa', + 'translate.google.com.sb', + 'translate.google.com.sg', + 'translate.google.com.sl', + 'translate.google.com.sv', + 'translate.google.com.tj', + 'translate.google.com.tr', + 'translate.google.com.tw', + 'translate.google.com.ua', + 'translate.google.com.uy', + 'translate.google.com.vc', + 'translate.google.com.vn', + 'translate.google.com', + 'translate.google.cv', + 'translate.google.cz', + 'translate.google.de', + 'translate.google.dj', + 'translate.google.dk', + 'translate.google.dm', + 'translate.google.dz', + 'translate.google.ee', + 'translate.google.es', + 'translate.google.fi', + 'translate.google.fm', + 'translate.google.fr', + 'translate.google.ga', + 'translate.google.ge', + 'translate.google.gg', + 'translate.google.gl', + 'translate.google.gm', + 'translate.google.gp', + 'translate.google.gr', + 'translate.google.gy', + 'translate.google.hn', + 'translate.google.hr', + 'translate.google.ht', + 'translate.google.hu', + 'translate.google.ie', + 'translate.google.im', + 'translate.google.iq', + 'translate.google.is', + 'translate.google.it', + 'translate.google.je', + 'translate.google.jo', + 'translate.google.kg', + 'translate.google.ki', + 'translate.google.kz', + 'translate.google.la', + 'translate.google.li', + 'translate.google.lk', + 'translate.google.lt', + 'translate.google.lu', + 'translate.google.lv', + 'translate.google.md', + 'translate.google.me', + 'translate.google.mg', + 'translate.google.mk', + 'translate.google.ml', + 'translate.google.mn', + 'translate.google.ms', + 'translate.google.mu', + 'translate.google.mv', + 'translate.google.mw', + 'translate.google.ne', + 'translate.google.nl', + 'translate.google.no', + 'translate.google.nr', + 'translate.google.nu', + 'translate.google.pl', + 'translate.google.pn', + 'translate.google.ps', + 'translate.google.pt', + 'translate.google.ro', + 'translate.google.rs', + 'translate.google.ru', + 'translate.google.rw', + 'translate.google.sc', + 'translate.google.se', + 'translate.google.sh', + 'translate.google.si', + 'translate.google.sk', + 'translate.google.sm', + 'translate.google.sn', + 'translate.google.so', + 'translate.google.sr', + 'translate.google.st', + 'translate.google.td', + 'translate.google.tg', + 'translate.google.tk', + 'translate.google.tl', + 'translate.google.tm', + 'translate.google.tn', + 'translate.google.to', + 'translate.google.tt', + 'translate.google.us', + 'translate.google.vg', + 'translate.google.vu', + 'translate.google.ws', +) class Provider(LocalProvider, SoupProvider): - __provider_type__ = { - 'translation': 'soup', - 'tts': 'local' - } - name = 'google' prettyname = 'Google' - translation = True - tts = True - definitions = False - change_instance = False - api_key_supported = False + + capabilities = ProviderCapability.TRANSLATION | ProviderCapability.TTS + features = ProviderFeature.DETECTION | ProviderFeature.MISTAKES | ProviderFeature.PRONUNCIATION + defaults = { 'instance_url': '', 'api_key': '', 'src_langs': ['en', 'fr', 'es', 'de'], - 'dest_langs': ['fr', 'es', 'de', 'en'] + 'dest_langs': ['fr', 'es', 'de', 'en'], } _service_urls = DEFAULT_SERVICE_URLS @@ -110,49 +245,150 @@ class Provider(LocalProvider, SoupProvider): 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', 'Referer': 'https://translate.google.com', } - _src_lang = None - _dest_lang = None def __init__(self, **kwargs): super().__init__(**kwargs) self.chars_limit = 2000 - self.detection = True - self.mistakes = True - self.pronunciation = True - # Populate languages + def init_trans(self, on_done, on_fail): languages = [ - 'af', 'sq', 'am', 'ar', 'hy', 'az', 'eu', 'be', 'bn', 'bs', 'bg', 'ca', - 'ceb', 'ny', 'zh-CN', 'zh-TW', 'co', 'hr', 'cs', 'da', 'nl', 'en', 'eo', - 'et', 'tl', 'fi', 'fr', 'fy', 'gl', 'ka', 'de', 'el', 'gu', 'ht', 'ha', - 'haw', 'iw', 'hi', 'hmn', 'hu', 'is', 'ig', 'id', 'ga', 'it', 'ja', 'jw', - 'kn', 'kk', 'km', 'rw', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'lt', 'lb', - 'mk', 'mg', 'ms', 'ml', 'mt', 'mi', 'mr', 'mn', 'my', 'ne', 'no', 'or', - 'ps', 'fa', 'pl', 'pt', 'pa', 'ro', 'ru', 'sm', 'gd', 'sr', 'st', 'sn', - 'sd', 'si', 'sk', 'sl', 'so', 'es', 'su', 'sw', 'sv', 'tg', 'ta', 'tt', - 'te', 'th', 'tr', 'tk', 'uk', 'ur', 'ug', 'uz', 'vi', 'cy', 'xh', 'yi', - 'yo', 'zu' + 'af', + 'sq', + 'am', + 'ar', + 'hy', + 'az', + 'eu', + 'be', + 'bn', + 'bs', + 'bg', + 'ca', + 'ceb', + 'ny', + 'zh-CN', + 'zh-TW', + 'co', + 'hr', + 'cs', + 'da', + 'nl', + 'en', + 'eo', + 'et', + 'tl', + 'fi', + 'fr', + 'fy', + 'gl', + 'ka', + 'de', + 'el', + 'gu', + 'ht', + 'ha', + 'haw', + 'iw', + 'hi', + 'hmn', + 'hu', + 'is', + 'ig', + 'id', + 'ga', + 'it', + 'ja', + 'jw', + 'kn', + 'kk', + 'km', + 'rw', + 'ko', + 'ku', + 'ky', + 'lo', + 'la', + 'lv', + 'lt', + 'lb', + 'mk', + 'mg', + 'ms', + 'ml', + 'mt', + 'mi', + 'mr', + 'mn', + 'my', + 'ne', + 'no', + 'or', + 'ps', + 'fa', + 'pl', + 'pt', + 'pa', + 'ro', + 'ru', + 'sm', + 'gd', + 'sr', + 'st', + 'sn', + 'sd', + 'si', + 'sk', + 'sl', + 'so', + 'es', + 'su', + 'sw', + 'sv', + 'tg', + 'ta', + 'tt', + 'te', + 'th', + 'tr', + 'tk', + 'uk', + 'ur', + 'ug', + 'uz', + 'vi', + 'cy', + 'xh', + 'yi', + 'yo', + 'zu', ] for code in languages: self.add_lang(code) - def init_tts(self): + on_done() + + def init_tts(self, on_done, on_fail): for code in lang.tts_langs().keys(): self.add_lang(code, trans=False, tts=True) - self.loaded = True + on_done() @staticmethod def _build_rpc_request(text: str, src: str, dest: str): - return json.dumps([[ + return json.dumps( [ - RPC_ID, - json.dumps([[text, src, dest, True], [None]], separators=(',', ':')), - None, - 'generic', + [ + [ + RPC_ID, + json.dumps([[text, src, dest, True], [None]], separators=(',', ':')), + None, + 'generic', + ], + ] ], - ]], separators=(',', ':')) + separators=(',', ':'), + ) def _pick_service_url(self): if len(self._service_urls) == 1: @@ -173,129 +409,138 @@ def translate_url(self): return self.format_url(url, params=params) - def format_translation(self, text, src, dest): - data = { - 'f.req': self._build_rpc_request(text, src, dest), - } - self._src_lang = src - self._dest_lang = dest - return self.create_request('POST', self.translate_url, data, self._headers, True) - - def get_translation(self, data): - try: - token_found = False - square_bracket_counts = [0, 0] - resp = '' - data = data.decode('utf-8') - - for line in data.split('\n'): - token_found = token_found or f'"{RPC_ID}"' in line[:30] - if not token_found: - continue - - is_in_string = False - for index, char in enumerate(line): - if char == '\"' and line[max(0, index - 1)] != '\\': - is_in_string = not is_in_string - if not is_in_string: - if char == '[': - square_bracket_counts[0] += 1 - elif char == ']': - square_bracket_counts[1] += 1 - - resp += line - if square_bracket_counts[0] == square_bracket_counts[1]: - break - - data = json.loads(resp) - parsed = json.loads(data[0][2]) - translated_parts = None - translated = None - try: - translated_parts = list( - map( - lambda part: TranslatedPart( - part[0] if len(part) > 0 else '', - part[1] if len(part) >= 2 else [] - ), - parsed[1][0][0][5] - ) - ) - except TypeError: - translated_parts = [ - TranslatedPart( - parsed[1][0][1][0], - [parsed[1][0][0][0], parsed[1][0][1][0]] - ) - ] - - first_iter = True - translated = "" - for part in translated_parts: - if not part.text.isspace() and not first_iter: - translated += " " - if first_iter: - first_iter = False - translated += part.text - - src = None + def translate(self, text, src_lang, dest_lang, on_done, on_fail): + def on_response(session, result): try: - src = parsed[1][-1][1] - except (IndexError, TypeError): - pass - - if not src == self._src_lang: - raise TranslationError('source language mismatch') + data = session.get_response(session, result) - if src == 'auto': try: - if parsed[0][2] in self.languages: - src = parsed[0][2] - except (IndexError, TypeError): - pass + token_found = False + square_bracket_counts = [0, 0] + resp = '' + data = data.decode('utf-8') + + for line in data.split('\n'): + token_found = token_found or f'"{RPC_ID}"' in line[:30] + if not token_found: + continue + + is_in_string = False + for index, char in enumerate(line): + if char == '\"' and line[max(0, index - 1)] != '\\': + is_in_string = not is_in_string + if not is_in_string: + if char == '[': + square_bracket_counts[0] += 1 + elif char == ']': + square_bracket_counts[1] += 1 + + resp += line + if square_bracket_counts[0] == square_bracket_counts[1]: + break + + data = json.loads(resp) + parsed = json.loads(data[0][2]) + translated_parts = None + translated = None + try: + translated_parts = list( + map( + lambda part: TranslatedPart( + part[0] if len(part) > 0 else '', part[1] if len(part) >= 2 else [] + ), + parsed[1][0][0][5], + ) + ) + except TypeError: + translated_parts = [ + TranslatedPart(parsed[1][0][1][0], [parsed[1][0][0][0], parsed[1][0][1][0]]) + ] + + first_iter = True + translated = "" + for part in translated_parts: + if not part.text.isspace() and not first_iter: + translated += " " + if first_iter: + first_iter = False + translated += part.text + + src = None + try: + src = parsed[1][-1][1] + except (IndexError, TypeError): + pass + + if not src == src_lang: + on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, 'source language mismatch')) + return + + if src == 'auto': + try: + if parsed[0][2] in self.languages: + src = parsed[0][2] + except (IndexError, TypeError): + pass + + dest = None + try: + dest = parsed[1][-1][2] + except (IndexError, TypeError): + pass + + if not dest == dest_lang: + on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, 'destination language mismatch')) + return + + origin_pronunciation = None + try: + origin_pronunciation = parsed[0][0] + except (IndexError, TypeError): + pass + + pronunciation = None + try: + pronunciation = parsed[1][0][0][1] + except (IndexError, TypeError): + pass + + mistake = None + try: + mistake = parsed[0][1][0][0][1] + # Convert to pango markup + mistake = mistake.replace('', '').replace('', '') + except (IndexError, TypeError): + pass + + result = Translation( + translated, + { + 'possible-mistakes': [mistake, self._strip_html_tags(mistake)], + 'src-pronunciation': origin_pronunciation, + 'dest-pronunciation': pronunciation, + }, + ) + on_done(result, src) - dest = None - try: - dest = parsed[1][-1][2] - except (IndexError, TypeError): - pass + except Exception as exc: + logging.warning(exc) + on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, str(exc))) - if not dest == self._dest_lang: - raise TranslationError('destination language mismatch') + except Exception as exc: + logging.warning(exc) + on_fail(ProviderError(ProviderErrorCode.NETWORK, str(exc))) - origin_pronunciation = None - try: - origin_pronunciation = parsed[0][0] - except (IndexError, TypeError): - pass + # Form data + data = { + 'f.req': self._build_rpc_request(text, src_lang, dest_lang), + } - pronunciation = None - try: - pronunciation = parsed[1][0][0][1] - except (IndexError, TypeError): - pass + # Request message + message = self.create_message('POST', self.translate_url, data, self._headers, True) - mistake = None - try: - mistake = parsed[0][1][0][0][1] - # Convert to pango markup - mistake = mistake.replace('', '').replace('', '') - except (IndexError, TypeError): - pass - - result = Translation( - translated, - { - 'possible-mistakes': [mistake, self._strip_html_tags(mistake)], - 'src-pronunciation': origin_pronunciation, - 'dest-pronunciation': pronunciation, - } - ) - return (result, src) - except TranslationError as e: - raise e - except Exception as e: - raise ProviderError(str(e)) + # Do async request + self.send_and_read(message, on_response) def _strip_html_tags(self, text): """Strip html tags""" @@ -307,14 +552,20 @@ def _strip_html_tags(self, text): escaped = html.escape(tags_removed) return escaped - def download_speech(self, text, language, file): - try: - tts = gTTS(text, lang=language, lang_check=False) - tts.write_to_fp(file) - file.seek(0) + def speech(self, text, language, on_done, on_fail): + def work(): + try: + file = NamedTemporaryFile() + tts = gTTS(text, lang=language, lang_check=False) + tts.write_to_fp(file) + file.seek(0) + + on_done(file) + except Exception as exc: + logging.warning(exc) + on_fail(ProviderError(ProviderErrorCode.TTS_FAILED, str(exc))) - except Exception as exc: - raise TextToSpeechError(exc) from exc + self.launch_thread(work) class TranslatedPart: diff --git a/dialect/providers/libretrans.py b/dialect/providers/libretrans.py index 8117c1e3..73411798 100644 --- a/dialect/providers/libretrans.py +++ b/dialect/providers/libretrans.py @@ -5,55 +5,50 @@ import logging from dialect.providers.base import ( - ApiKeyRequired, BatchSizeExceeded, CharactersLimitExceeded, - InvalidLangCode, InvalidApiKey, ProviderError, SoupProvider, Translation, - TranslationError + ProviderCapability, + ProviderFeature, + ProviderErrorCode, + ProviderError, + SoupProvider, + Translation, ) class Provider(SoupProvider): - __provider_type__ = 'soup' - name = 'libretranslate' prettyname = 'LibreTranslate' - translation = True - tts = False - definitions = False - change_instance = True - api_key_supported = True + + capabilities = ProviderCapability.TRANSLATION + features = ProviderFeature.INSTANCES | ProviderFeature.DETECTION | ProviderFeature.PRONUNCIATION + defaults = { 'instance_url': 'libretranslate.de', 'api_key': '', 'src_langs': ['en', 'fr', 'es', 'de'], - 'dest_langs': ['fr', 'es', 'de', 'en'] + 'dest_langs': ['fr', 'es', 'de', 'en'], } - trans_init_requests = [ - 'languages', - 'settings' - ] - def __init__(self, **kwargs): super().__init__(**kwargs) self.chars_limit = 0 - self.detection = True - self.api_key_supported = False # For LT conditional api keys @staticmethod - def format_validate_instance(url): - url = Provider.format_url(url, '/spec') - return Provider.create_request('GET', url) + def validate_instance(url, on_done, on_fail): + def on_response(data): + valid = False - @staticmethod - def validate_instance(data): - data = Provider.read_data(data) - valid = False + try: + valid = data['info']['title'] == 'LibreTranslate' + except: # noqa + pass - if data and data is not None: - valid = data['info']['title'] == 'LibreTranslate' + on_done(valid) - return valid + # Message request to LT API spec endpoint + message = Provider.create_message('GET', Provider.format_url(url, '/spec')) + # Do async request + Provider.send_and_read_and_process_response(message, on_response, on_fail, False) @property def frontend_settings_url(self): @@ -73,75 +68,115 @@ def suggest_url(self): @property def translate_url(self): - return self.format_url(self.instance_url, '/translate') - - def format_languages_init(self): - return self.create_request('GET', self.lang_url) - - def languages_init(self, data): - try: - data = self.read_data(data) - self._check_errors(data) - for lang in data: - self.add_lang(lang['code'], lang['name']) - except Exception as exc: - logging.warning(exc) - self.error = str(exc) - - def format_settings_init(self): - return self.create_request('GET', self.frontend_settings_url) - - def settings_init(self, data): - try: - data = self.read_data(data) - self._check_errors(data) - self.suggestions = data.get('suggestions', False) - self.api_key_supported = data.get('apiKeys', False) - self.api_key_required = data.get('keyRequired', False) - self.chars_limit = data.get('charLimit', 0) - except Exception as exc: - logging.warning(exc) - self.error = str(exc) - - def format_validate_api_key(self, api_key): + self.format_url(self.instance_url, '/translate') + + def init_trans(self, on_done, on_fail): + def check_finished(): + self._init_count -= 1 + + if self._init_count == 0: + if self._init_error: + on_fail(self._init_error) + else: + on_done() + + def on_failed(error): + self._init_error = error + check_finished() + + def on_languages_response(data): + try: + for lang in data: + self.add_lang(lang['code'], lang['name']) + + check_finished() + + except Exception as exc: + logging.warning(exc) + on_failed(ProviderError(ProviderErrorCode.UNEXPECTED, str(exc))) + + def on_settings_response(data): + try: + if data.get('suggestions', False): + self.features ^= ProviderFeature.SUGGESTIONS + if data.get('apiKeys', False): + self.features ^= ProviderFeature.API_KEY + if data.get('keyRequired', False): + self.features ^= ProviderFeature.API_KEY_REQUIRED + + self.chars_limit = data.get('charLimit', 0) + + check_finished() + + except Exception as exc: + logging.warning(exc) + on_failed(ProviderError(ProviderErrorCode.UNEXPECTED, str(exc))) + + # Keep state of multiple request + self._init_count = 2 + self._init_error = None + + # Request messages + languages_message = self.create_message('GET', self.lang_url) + settings_message = self.create_message('GET', self.frontend_settings_url) + + # Do async requests + self.send_and_read_and_process_response(languages_message, on_languages_response, on_failed) + self.send_and_read_and_process_response(settings_message, on_settings_response, on_failed) + + def validate_api_key(self, key, on_done, on_fail): + def on_response(data): + valid = False + try: + valid = 'confidence' in data[0] + except: # noqa + pass + + on_done(valid) + + # Form data data = { 'q': 'hello', - 'source': 'en', - 'target': 'es', - 'api_key': api_key, + 'api_key': key, } - return self.create_request('POST', self.translate_url, data) - - def validate_api_key(self, data): - self.get_translation(data) - def format_translation(self, text, src, dest): + # Request message + message = self.create_message('POST', self.detect_url, data, form=True) + # Do async request + self.send_and_read_and_process_response(message, on_response, on_fail) + + def translate(self, text, src, dest, on_done, on_fail): + def on_response(data): + detected = data.get('detectedLanguage', {}).get('language', None) + translation = Translation( + data['translatedText'], + { + 'possible-mistakes': [None, None], + 'src-pronunciation': None, + 'dest-pronunciation': None, + }, + ) + on_done(translation, detected) + + # Request body data = { 'q': text, 'source': src, 'target': dest, } - if self.api_key and self.api_key_supported: + if self.api_key and ProviderFeature.API_KEY in self.features: data['api_key'] = self.api_key - return self.create_request('POST', self.translate_url, data) + # Request message + message = self.create_message('POST', self.translate_url, data) + # Do async request + self.send_and_read_and_process_response(message, on_response, on_fail) - def get_translation(self, data): - data = self.read_data(data) - self._check_errors(data) - detected = data.get('detectedLanguage', {}).get('language', None) - translation = Translation( - data['translatedText'], - { - 'possible-mistakes': None, - 'src-pronunciation': None, - 'dest-pronunciation': None, - }, - ) + def suggest(self, text, src, dest, suggestion, on_done, on_fail): + def on_response(data): + on_done(data.get('success', False)) - return (translation, detected) - - def format_suggestion(self, text, src, dest, suggestion): + # Form data data = { 'q': text, 'source': src, @@ -151,31 +186,29 @@ def format_suggestion(self, text, src, dest, suggestion): if self.api_key and self.api_key_supported: data['api_key'] = self.api_key - return self.create_request('POST', self.suggest_url, data, form=True) - - def get_suggestion(self, data): - data = self.read_data(data) - self._check_errors(data) - return data.get('success', False) + # Request message + message = self.create_message('POST', self.suggest_url, data, form=True) + # Do async request + self.send_and_read_and_process_response(message, on_response, on_fail) - def _check_errors(self, data): - """Raises a proper Exception if an error is found in the data.""" + @staticmethod + def check_known_errors(data): if not data: - raise ProviderError('Request empty') + return ProviderError(ProviderErrorCode.EMPTY, 'Response is empty!') if 'error' in data: error = data['error'] if error == 'Please contact the server operator to obtain an API key': - raise ApiKeyRequired(error) + return ProviderError(ProviderErrorCode.API_KEY_REQUIRED, error) elif error == 'Invalid API key': - raise InvalidApiKey(error) + return ProviderError(ProviderErrorCode.API_KEY_INVALID, error) elif 'is not supported' in error: - raise InvalidLangCode(error) + return ProviderError(ProviderErrorCode.INVALID_LANG_CODE, error) elif 'exceeds text limit' in error: - raise BatchSizeExceeded(error) + return ProviderError(ProviderErrorCode.BATCH_SIZE_EXCEEDED, error) elif 'exceeds character limit' in error: - raise CharactersLimitExceeded(error) + return ProviderError(ProviderErrorCode.CHARACTERS_LIMIT_EXCEEDED, error) elif 'Cannot translate text' in error or 'format is not supported' in error: - raise TranslationError(error) + return ProviderError(ProviderErrorCode.TRANSLATION_FAILED, error) else: - raise ProviderError(error) + return ProviderError(ProviderErrorCode.UNEXPECTED, error) diff --git a/dialect/providers/lingva.py b/dialect/providers/lingva.py index f568d50a..3e986f1f 100644 --- a/dialect/providers/lingva.py +++ b/dialect/providers/lingva.py @@ -3,59 +3,55 @@ # SPDX-License-Identifier: GPL-3.0-or-later import logging +from tempfile import NamedTemporaryFile import urllib from dialect.providers.base import ( - InvalidLangCode, SoupProvider, ProviderError, TextToSpeechError, Translation + ProviderCapability, + ProviderError, + ProviderErrorCode, + ProviderFeature, + SoupProvider, + Translation, ) class Provider(SoupProvider): - __provider_type__ = 'soup' - name = 'lingva' prettyname = 'Lingva Translate' - translation = True - tts = True - definitions = False - change_instance = True - api_key_supported = False + + capabilities = ProviderCapability.TRANSLATION | ProviderCapability.TTS + features = ( + ProviderFeature.INSTANCES | ProviderFeature.DETECTION | ProviderFeature.MISTAKES | ProviderFeature.PRONUNCIATION + ) + defaults = { 'instance_url': 'lingva.ml', 'api_key': '', 'src_langs': ['en', 'fr', 'es', 'de'], - 'dest_langs': ['fr', 'es', 'de', 'en'] + 'dest_langs': ['fr', 'es', 'de', 'en'], } - trans_init_requests = [ - 'languages' - ] - tts_init_requests = [ - 'languages' - ] - def __init__(self, **kwargs): super().__init__(**kwargs) self.chars_limit = 5000 - self.detection = True - self.mistakes = True - self.pronunciation = True - - @staticmethod - def format_validate_instance(url): - url = Provider.format_url(url, '/api/v1/en/es/hello') - return Provider.create_request('GET', url) @staticmethod - def validate_instance(data): - data = Provider.read_data(data) - valid = False + def validate_instance(url, on_done, on_fail): + def on_response(data): + valid = False + try: + valid = 'translation' in data + except: # noqa + pass - if data and 'translation' in data: - valid = True + on_done(valid) - return valid + # Lingva translation endpoint + message = Provider.create_message('GET', Provider.format_url(url, '/api/v1/en/es/hello')) + # Do async request + Provider.send_and_read_and_process_response(message, on_response, on_fail, False) @property def lang_url(self): @@ -69,71 +65,91 @@ def translate_url(self): def speech_url(self): return self.format_url(self.instance_url, '/api/v1/audio/{lang}/{text}') - def format_languages_init(self): - return self.create_request('GET', self.lang_url) - - def languages_init(self, data): - try: - data = self.read_data(data) - self._check_errors(data) + def init(self, on_done, on_fail): + def on_response(data): if 'languages' in data: for lang in data['languages']: if lang['code'] != 'auto': self.add_lang(lang['code'], lang['name'], tts=True) + on_done() else: - self.error = 'No langs found on server.' - except Exception as exc: - logging.warning(exc) - self.error = str(exc) - - def format_translation(self, text, src, dest): + on_fail(ProviderError(ProviderErrorCode.UNEXPECTED, 'No langs found in server.')) + + # Languages message request + message = self.create_message('GET', self.lang_url) + # Do async request + self.send_and_read_and_process_response(message, on_response, on_fail) + + def init_trans(self, on_done, on_fail): + self.init(on_done, on_fail) + + def init_tts(self, on_done, on_fail): + self.init(on_done, on_fail) + + def translate(self, text, src, dest, on_done, on_fail): + def on_response(data): + try: + detected = data['info'].get('detectedSource', None) + mistakes = data['info'].get('typo', None) + src_pronunciation = data['info']['pronunciation'].get('query', None) + dest_pronunciation = data['info']['pronunciation'].get('translation', None) + + translation = Translation( + data['translation'], + { + 'possible-mistakes': [mistakes, mistakes], + 'src-pronunciation': src_pronunciation, + 'dest-pronunciation': dest_pronunciation, + }, + ) + + on_done(translation, detected) + + except Exception as exc: + error = 'Failed reading the translation data' + logging.warning(error, exc) + on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, error)) + + # Format url query data text = urllib.parse.quote(text, safe='') url = self.translate_url.format(text=text, src=src, dest=dest) - return self.create_request('GET', url) - def get_translation(self, data): - data = self.read_data(data) - self._check_errors(data) + # Request message + message = self.create_message('GET', url) - detected = data['info'].get('detectedSource', None) - mistakes = data['info'].get('typo', None) - src_pronunciation = data['info']['pronunciation'].get('query', None) - dest_pronunciation = data['info']['pronunciation'].get('translation', None) + # Do async request + self.send_and_read_and_process_response(message, on_response, on_fail) - translation = Translation( - data['translation'], - { - 'possible-mistakes': [mistakes, mistakes], - 'src-pronunciation': src_pronunciation, - 'dest-pronunciation': dest_pronunciation, - }, - ) + def speech(self, text, language, on_done, on_fail): + def on_response(data): + if 'audio' in data: + file = NamedTemporaryFile() + audio = bytearray(data['audio']) + file.write(audio) + file.seek(0) - return (translation, detected) + on_done(file) + else: + on_fail(ProviderError(ProviderErrorCode.TTS_FAILED, 'No audio was found.')) - def format_speech(self, text, language): + # Format url query data url = self.speech_url.format(text=text, lang=language) - return self.create_request('GET', url) - def get_speech(self, data, file): - data = self.read_data(data) - self._check_errors(data) + # Request message + message = self.create_message('GET', url) - if 'audio' in data: - audio = bytearray(data['audio']) - file.write(audio) - file.seek(0) - else: - raise TextToSpeechError('No audio was found') + # Do async request + self.send_and_read_and_process_response(message, on_response, on_fail) - def _check_errors(self, data): + @staticmethod + def check_known_errors(data): """Raises a proper Exception if an error is found in the data.""" if not data: - raise ProviderError('Request empty') + return ProviderError(ProviderErrorCode.EMPTY, 'Response is empty!') if 'error' in data: error = data['error'] if error == 'Invalid target language' or error == 'Invalid source language': - raise InvalidLangCode(error) + return ProviderError(ProviderErrorCode.INVALID_LANG_CODE, error) else: - raise ProviderError(error) + return ProviderError(ProviderErrorCode.UNEXPECTED, error) diff --git a/dialect/providers/yandex.py b/dialect/providers/yandex.py index 73d997a7..87cca347 100644 --- a/dialect/providers/yandex.py +++ b/dialect/providers/yandex.py @@ -4,25 +4,27 @@ from uuid import uuid4 from dialect.providers.base import ( - ProviderError, SoupProvider, Translation, TranslationError + ProviderCapability, + ProviderError, + ProviderErrorCode, + ProviderFeature, + SoupProvider, + Translation, ) class Provider(SoupProvider): - __provider_type__ = 'soup' - name = 'yandex' prettyname = 'Yandex' - translation = True - tts = False - definitions = False - change_instance = False - api_key_supported = False + + capabilities = ProviderCapability.TRANSLATION + features = ProviderFeature.DETECTION + defaults = { 'instance_url': '', 'api_key': '', 'src_langs': ['en', 'fr', 'es', 'de'], - 'dest_langs': ['fr', 'es', 'de', 'en'] + 'dest_langs': ['fr', 'es', 'de', 'en'], } _headers = { @@ -32,59 +34,153 @@ class Provider(SoupProvider): def __init__(self, **kwargs): super().__init__(**kwargs) - self.languages = [ - 'af', 'sq', 'am', 'ar', 'hy', 'az', 'ba', 'eu', 'be', 'bn', 'bs', 'bg', 'my', - 'ca', 'ceb', 'zh', 'cv', 'hr', 'cs', 'da', 'nl', 'sjn', 'emj', 'en', 'eo', - 'et', 'fi', 'fr', 'gl', 'ka', 'de', 'el', 'gu', 'ht', 'he', 'mrj', 'hi', - 'hu', 'is', 'id', 'ga', 'it', 'ja', 'jv', 'kn', 'kk', 'kazlat', 'km', 'ko', - 'ky', 'lo', 'la', 'lv', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mi', 'mr', - 'mhr', 'mn', 'ne', 'no', 'pap', 'fa', 'pl', 'pt', 'pa', 'ro', 'ru', 'gd', 'sr', - 'si', 'sk', 'sl', 'es', 'su', 'sw', 'sv', 'tl', 'tg', 'ta', 'tt', 'te', 'th', 'tr', - 'udm', 'uk', 'ur', 'uz', 'uzbcyr', 'vi', 'cy', 'xh', 'sah', 'yi', 'zu' - ] self.chars_limit = 10000 - self.detection = True self._uuid = str(uuid4()).replace('-', '') + def init_trans(self, on_done, on_fail): + self.languages = [ + 'af', + 'sq', + 'am', + 'ar', + 'hy', + 'az', + 'ba', + 'eu', + 'be', + 'bn', + 'bs', + 'bg', + 'my', + 'ca', + 'ceb', + 'zh', + 'cv', + 'hr', + 'cs', + 'da', + 'nl', + 'sjn', + 'emj', + 'en', + 'eo', + 'et', + 'fi', + 'fr', + 'gl', + 'ka', + 'de', + 'el', + 'gu', + 'ht', + 'he', + 'mrj', + 'hi', + 'hu', + 'is', + 'id', + 'ga', + 'it', + 'ja', + 'jv', + 'kn', + 'kk', + 'kazlat', + 'km', + 'ko', + 'ky', + 'lo', + 'la', + 'lv', + 'lt', + 'lb', + 'mk', + 'mg', + 'ms', + 'ml', + 'mt', + 'mi', + 'mr', + 'mhr', + 'mn', + 'ne', + 'no', + 'pap', + 'fa', + 'pl', + 'pt', + 'pa', + 'ro', + 'ru', + 'gd', + 'sr', + 'si', + 'sk', + 'sl', + 'es', + 'su', + 'sw', + 'sv', + 'tl', + 'tg', + 'ta', + 'tt', + 'te', + 'th', + 'tr', + 'udm', + 'uk', + 'ur', + 'uz', + 'uzbcyr', + 'vi', + 'cy', + 'xh', + 'sah', + 'yi', + 'zu', + ] + + on_done() + @property def translate_url(self): path = f'/api/v1/tr.json/translate?id={self._uuid}-0-0&srv=android' return self.format_url('translate.yandex.net', path) - def format_translation(self, text, src, dest): - data = { - 'lang': dest, - 'text': text - } - if src != 'auto': - data['lang'] = f'{src}-{dest}' - - return self.create_request('POST', self.translate_url, data, self._headers, True) - - def get_translation(self, data): - data = self.read_data(data) - detected = None - - if 'code' in data and data['code'] == 200: + def translate(self, text, src, dest, on_done, on_fail): + def on_response(data): + detected = None + if 'code' in data and data['code'] == 200: + if 'lang' in data: + detected = data['lang'].split('-')[0] + + if 'text' in data: + result = Translation( + data['text'][0], + { + 'possible-mistakes': None, + 'src-pronunciation': None, + 'dest-pronunciation': None, + }, + ) + on_done(result, detected) + + else: + on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, 'Translation failed')) - if 'lang' in data: - detected = data['lang'].split('-')[0] + else: + error = data['message'] if 'message' in data else '' + on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, error)) - if 'text' in data: - result = Translation( - data['text'][0], - { - 'possible-mistakes': None, - 'src-pronunciation': None, - 'dest-pronunciation': None, - } - ) - return (result, detected) + # Form data + data = {'lang': dest, 'text': text} + if src != 'auto': + data['lang'] = f'{src}-{dest}' - else: - raise TranslationError('Translation failed') + # Request message + message = self.create_message('POST', self.translate_url, data, self._headers, True) - else: - error = data['message'] if 'message' in data else '' - raise ProviderError(error) + # Do async request + self.send_and_read_and_process_response(message, on_response, on_fail) diff --git a/dialect/widgets/provider_preferences.py b/dialect/widgets/provider_preferences.py index 1dcdb4a3..4509c2fb 100644 --- a/dialect/widgets/provider_preferences.py +++ b/dialect/widgets/provider_preferences.py @@ -9,6 +9,7 @@ from dialect.define import RES_PATH from dialect.session import Session +from dialect.providers import ProviderCapability, ProviderFeature @Gtk.Template(resource_path=f'{RES_PATH}/provider-preferences.ui') @@ -40,9 +41,9 @@ def __init__(self, providers, scope, **kwargs): self.title.props.subtitle = self.provider.prettyname - self.translation = self.provider.translation - self.tts = self.provider.tts - self.definitions = self.provider.definitions + self.translation = ProviderCapability.TRANSLATION in self.provider.capabilities + self.tts = ProviderCapability.TTS in self.provider.capabilities + self.definitions = ProviderCapability.DEFINITIONS in self.provider.capabilities # Check what entries to show self._check_settings() @@ -58,8 +59,8 @@ def _on_parent(self, _view, _pspec): self.get_root().parent.connect('notify::translator-loading', self._on_translator_loading) def _check_settings(self): - self.instance_entry.props.visible = self.provider.change_instance - self.api_key_entry.props.visible = self.provider.api_key_supported + self.instance_entry.props.visible = ProviderFeature.INSTANCES in self.provider.features + self.api_key_entry.props.visible = ProviderFeature.API_KEY in self.provider.features @Gtk.Template.Callback() def _on_back(self, _button): @@ -69,14 +70,7 @@ def _on_back(self, _button): @Gtk.Template.Callback() def _on_instance_apply(self, _row): """ Called on self.instance_entry::apply signal """ - def on_validation_response(session, result): - valid = False - try: - data = Session.get_response(session, result) - valid = self.provider.validate_instance(data) - except Exception as exc: - logging.error(exc) - + def on_done(valid): if valid: self.provider.instance_url = self.new_instance_url self.provider.reset_src_langs() @@ -109,8 +103,8 @@ def on_validation_response(session, result): self.instance_stack.props.visible_child_name = 'spinner' self.instance_spinner.start() - validation = self.provider.format_validate_instance(self.new_instance_url) - Session.get().send_and_read_async(validation, 0, None, on_validation_response) + # TODO: Use on_fail to notify network error + self.provider.validate_instance(self.new_instance_url, on_done, lambda _: on_done(False)) else: self.instance_entry.remove_css_class('error') @@ -133,15 +127,7 @@ def _on_reset_instance(self, _button): @Gtk.Template.Callback() def _on_api_key_apply(self, _row): """ Called on self.api_key_entry::apply signal """ - def on_validation_response(session, result): - valid = False - try: - data = Session.get_response(session, result) - self.provider.validate_api_key(data) - valid = True - except Exception as exc: - logging.error(exc) - + def on_done(valid): if valid: self.provider.api_key = self.new_api_key self.api_key_entry.remove_css_class('error') @@ -169,8 +155,8 @@ def on_validation_response(session, result): self.api_key_stack.props.visible_child_name = 'spinner' self.api_key_spinner.start() - validation = self.provider.format_validate_api_key(self.new_api_key) - Session.get().send_and_read_async(validation, 0, None, on_validation_response) + # TODO: Use on_fail to notify network error + self.provider.validate_api_key(self.new_api_key, on_done, lambda _: on_done(False)) else: self.api_key_entry.remove_css_class('error') diff --git a/dialect/window.py b/dialect/window.py index 33c3e291..743d8208 100644 --- a/dialect/window.py +++ b/dialect/window.py @@ -5,16 +5,13 @@ import logging import random -import threading -from tempfile import NamedTemporaryFile from gi.repository import Adw, Gdk, Gio, GLib, GObject, Gst, Gtk from dialect.define import APP_ID, PROFILE, RES_PATH, TRANS_NUMBER from dialect.languages import LanguagesListModel -from dialect.providers import TRANSLATORS, TTS -from dialect.providers.base import ApiKeyRequired, InvalidApiKey, ProviderError -from dialect.session import Session, ResponseError +from dialect.providers import TRANSLATORS, TTS, ProviderFeature, ProviderError, ProviderErrorCode +from dialect.providers.base import BaseProvider from dialect.settings import Settings from dialect.shortcuts import DialectShortcutsWindow from dialect.widgets import LangSelector, ThemeSwitcher @@ -81,7 +78,7 @@ class DialectWindow(Adw.ApplicationWindow): launch = True # Providers objects - provider = { + provider: dict[str, BaseProvider] = { 'trans': None, 'tts': None } @@ -122,7 +119,6 @@ def __init__(self, text, langs, **kwargs): bus = self.player.get_bus() bus.add_signal_watch() bus.connect('message', self.on_gst_message) - self.player_event = threading.Event() # An event for letting us know when Gst is done playing # Setup window self.setup_actions() @@ -203,7 +199,7 @@ def setup(self): # Load translator self.load_translator() - # Get languages available for speech + # Load text to speech self.load_tts() def setup_selectors(self): @@ -251,64 +247,56 @@ def setup_translation(self): self.toggle_voice_spinner(True) - def _check_provider_type(self, provider_type, context): - if isinstance(provider_type, dict): - return provider_type[context] - return provider_type - def load_translator(self): - def on_loaded(errors): - if errors or self.provider['trans'].error: - # Show error view - if self.provider['trans'].error: - self.loading_failed(self.provider['trans'].error) - else: - self.loading_failed(errors, True) - self.translator_loading = False + def on_done(): + # Mistakes support + if ProviderFeature.MISTAKES not in self.provider['trans'].features: + self.mistakes.props.reveal_child = False + + # Suggestions support + self.ui_suggest_cancel(None, None) + if ProviderFeature.SUGGESTIONS not in self.provider['trans'].features: + self.edit_btn.props.visible = False else: - # Supported features - if not self.provider['trans'].mistakes: - self.mistakes.props.reveal_child = False + self.edit_btn.props.visible = True - self.ui_suggest_cancel(None, None) - if not self.provider['trans'].suggestions: - self.edit_btn.props.visible = False - else: - self.edit_btn.props.visible = True + # Pronunciation support + if ProviderFeature.PRONUNCIATION not in self.provider['trans'].features: + self.src_pron_revealer.props.reveal_child = False + self.dest_pron_revealer.props.reveal_child = False + self.app.lookup_action('pronunciation').props.enabled = False + else: + self.app.lookup_action('pronunciation').props.enabled = True + + # Update langs + self.src_lang_model.set_langs(self.provider['trans'].languages) + self.dest_lang_model.set_langs(self.provider['trans'].languages) + + # Update selected langs + set_auto = Settings.get().src_auto and ProviderFeature.DETECTION in self.provider['trans'].features + src_lang = self.provider['trans'].languages[0] + if self.src_langs and self.src_langs[0] in self.provider['trans'].languages: + src_lang = self.src_langs[0] + self.src_lang_selector.selected = 'auto' if set_auto else src_lang + + dest_lang = self.provider['trans'].languages[1] + if self.dest_langs and self.dest_langs[0] in self.provider['trans'].languages: + dest_lang = self.dest_langs[0] + self.dest_lang_selector.selected = dest_lang - if not self.provider['trans'].pronunciation: - self.src_pron_revealer.props.reveal_child = False - self.dest_pron_revealer.props.reveal_child = False - self.app.lookup_action('pronunciation').props.enabled = False - else: - self.app.lookup_action('pronunciation').props.enabled = True - - # Update langs - self.src_lang_model.set_langs(self.provider['trans'].languages) - self.dest_lang_model.set_langs(self.provider['trans'].languages) - - # Update selected langs - set_auto = Settings.get().src_auto and self.provider['trans'].detection - src_lang = self.provider['trans'].languages[0] - if self.src_langs and self.src_langs[0] in self.provider['trans'].languages: - src_lang = self.src_langs[0] - self.src_lang_selector.selected = 'auto' if set_auto else src_lang - - dest_lang = self.provider['trans'].languages[1] - if self.dest_langs and self.dest_langs[0] in self.provider['trans'].languages: - dest_lang = self.dest_langs[0] - self.dest_lang_selector.selected = dest_lang - - # Update chars limit - if self.provider['trans'].chars_limit == -1: # -1 means unlimited - self.char_counter.props.label = '' - else: - count = f"{str(self.src_buffer.get_char_count())}/{self.provider['trans'].chars_limit}" - self.char_counter.props.label = count + # Update chars limit + if self.provider['trans'].chars_limit == -1: # -1 means unlimited + self.char_counter.props.label = '' + else: + count = f"{str(self.src_buffer.get_char_count())}/{self.provider['trans'].chars_limit}" + self.char_counter.props.label = count - self.translator_loading = False + self.translator_loading = False - self.check_apikey() + self.check_apikey() + + def on_fail(error: ProviderError): + self.loading_failed(error) provider = Settings.get().active_translator @@ -317,6 +305,13 @@ def on_loaded(errors): # Translator object self.provider['trans'] = TRANSLATORS[provider]() + # Get saved languages + self.src_langs = self.provider['trans'].src_langs + self.dest_langs = self.provider['trans'].dest_langs + # Do provider init + self.provider['trans'].init_trans(on_done, on_fail) + + # Connect to provider settings changes self.provider['trans'].settings.connect( 'changed::instance-url', self._on_provider_changed, self.provider['trans'].name ) @@ -324,31 +319,13 @@ def on_loaded(errors): 'changed::api-key', self._on_provider_changed, self.provider['trans'].name ) - # Get saved languages - self.src_langs = self.provider['trans'].src_langs - self.dest_langs = self.provider['trans'].dest_langs - - # Make the init requests required to use the translator - if self.provider['trans'].trans_init_requests: - requests = [] - for name in self.provider['trans'].trans_init_requests: - message = getattr(self.provider['trans'], f'format_{name}_init')() - callback = getattr(self.provider['trans'], f'{name}_init') - requests.append([message, callback]) - Session.get().multiple(requests, on_loaded) - else: - on_loaded('') - def check_apikey(self): - def on_response(session, result): - try: - data = Session.get_response(session, result) - self.provider['trans'].validate_api_key(data) + def on_done(valid): + if valid: self.main_stack.props.visible_child_name = 'translate' - except InvalidApiKey as exc: - logging.warning(exc) + else: self.key_page.props.title = _('The provided API key is invalid') - if self.provider['trans'].api_key_required: + if ProviderFeature.API_KEY_REQUIRED in self.provider['trans'].features: self.key_page.props.description = _('Please set a valid API key in the preferences.') else: self.key_page.props.description = _( @@ -357,18 +334,17 @@ def on_response(session, result): self.rmv_key_btn.props.visible = True self.error_api_key_btn.props.visible = True self.main_stack.props.visible_child_name = 'api-key' - except ProviderError as exc: - logging.warning(exc) - self.loading_failed(str(exc)) - except Exception as exc: - logging.warning(exc) - self.loading_failed(str(exc), True) - if self.provider['trans'].api_key_supported: + def on_fail(error: ProviderError): + self.loading_failed(error) + + if ProviderFeature.API_KEY in self.provider['trans'].features: if self.provider['trans'].api_key: - validation = self.provider['trans'].format_validate_api_key(self.provider['trans'].api_key) - Session.get().send_and_read_async(validation, 0, None, on_response) - elif not self.provider['trans'].api_key and self.provider['trans'].api_key_required: + self.provider['trans'].validate_api_key(self.provider['trans'].api_key, on_done, on_fail) + elif ( + not self.provider['trans'].api_key + and ProviderFeature.API_KEY_REQUIRED in self.provider['trans'].features + ): self.key_page.props.title = _('API key is required to use the service') self.key_page.props.description = _('Please set an API key in the preferences.') self.main_stack.props.visible_child_name = 'api-key' @@ -377,7 +353,7 @@ def on_response(session, result): else: self.main_stack.props.visible_child_name = 'translate' - def loading_failed(self, details='', network=False): + def loading_failed(self, error: ProviderError): self.main_stack.props.visible_child_name = 'error' service = self.provider['trans'].prettyname @@ -385,25 +361,25 @@ def loading_failed(self, details='', network=False): title = _('Failed loading the translation service') description = _('Please report this in the Dialect bug tracker if the issue persists.') - if self.provider['trans'].change_instance: + if ProviderFeature.INSTANCES in self.provider['trans'].features: description = _(( 'Failed loading "{url}", check if the instance address is correct or report in the Dialect bug tracker' ' if the issue persists.' )) description = description.format(url=url) - if network: + if error.code == ProviderErrorCode.NETWORK: title = _('Couldn’t connect to the translation service') description = _('We can’t connect to the server. Please check for network issues.') - if self.provider['trans'].change_instance: + if ProviderFeature.INSTANCES in self.provider['trans'].features: description = _(( 'We can’t connect to the {service} instance "{url}".\n' 'Please check for network issues or if the address is correct.' )) description = description.format(service=service, url=url) - if details: - description = description + '\n\n' + details + '' + if error.message: + description = description + '\n\n' + error.message + '' self.error_page.props.title = title self.error_page.props.description = description @@ -412,77 +388,53 @@ def loading_failed(self, details='', network=False): def retry_load_translator(self, _button): self.load_translator() + @Gtk.Template.Callback() + def remove_key_and_reload(self, _button): + self.provider['trans'].reset_api_key() + self.load_translator() + @Gtk.Template.Callback() def on_stack_page_change(self, _stack, _param): if self.main_stack.props.visible_child_name == 'translate' and self.launch: # Page being set to "Translate" means the translator is fully ready # We can now translate as per CLI parameters - self.launch = False # Prevent reoccurance of CLI parameter translation + self.launch = False # Prevent reoccurrence of CLI parameter translation if self.launch_text != '': - self.translate(self.launch_text, self.launch_langs['src'], self.launch_langs['dest']) - - @Gtk.Template.Callback() - def remove_key_and_reload(self, _button): - self.provider['trans'].reset_api_key() - self.load_translator() + self.translate(self.launch_text, self.launch_langs['src'], self.launch_langs['dest']) def load_tts(self): - # TTS object + def on_done(): + self.download_speech() + + def on_fail(_error: ProviderError): + self.on_listen_failed() + + # TTS name provider = Settings.get().active_tts + # Check if TTS is disabled if provider != '': self.src_voice_btn.props.visible = True self.dest_voice_btn.props.visible = True + # TTS Object self.provider['tts'] = TTS[provider]() + self.provider['tts'].init_tts(on_done, on_fail) + + # Connect to provider settings changes self.provider['tts'].settings.connect( 'changed::instance-url', self._on_provider_changed, self.provider['tts'].name ) self.provider['tts'].settings.connect( 'changed::api-key', self._on_provider_changed, self.provider['tts'].name ) - - match self._check_provider_type(self.provider['tts'].__provider_type__, 'tts'): - case 'local': - threading.Thread( - target=self._load_local_tts, - daemon=True - ).start() - case 'soup': - # Make the init requests required to use the provider - requests = [] - if self.provider['tts'].tts_init_requests: - for name in self.provider['tts'].tts_init_requests: - message = getattr(self.provider['tts'], f'format_{name}_init')() - callback = getattr(self.provider['tts'], f'{name}_init') - requests.append([message, callback]) - Session.get().multiple(requests, self._on_tts_loaded) - else: - self._on_tts_loaded('') else: self.provider['tts'] = None self.src_voice_btn.props.visible = False self.dest_voice_btn.props.visible = False - def _load_local_tts(self): - try: - self.provider['tts'].init_tts() - except ProviderError as exc: - logging.error('Error: ' + str(exc)) - self.on_listen_failed() - else: - # Download speech if we are retrying - self.download_speech() - - def _on_tts_loaded(self, errors): - if errors or self.provider['tts'].error: - self.on_listen_failed() - else: - # Download speech if we are retrying - self.download_speech() - def on_listen_failed(self): self.src_voice_btn.props.child = self.src_voice_warning self.src_voice_spinner.stop() @@ -795,6 +747,21 @@ def ui_suggest(self, _action, _param): self.dest_text.props.editable = True def ui_suggest_ok(self, _action, _param): + def on_done(success): + self.dest_toolbar_stack.props.visible_child_name = 'default' + + if success: + self.send_notification(_('New translation has been suggested!')) + else: + self.send_notification(_('Suggestion failed.')) + + self.dest_text.props.editable = False + + def on_fail(error: ProviderError): + self.dest_toolbar_stack.props.visible_child_name = 'default' + self.send_notification(_('Suggestion failed.')) + self.dest_text.props.editable = False + dest_text = self.dest_buffer.get_text( self.dest_buffer.get_start_iter(), self.dest_buffer.get_end_iter(), @@ -805,13 +772,15 @@ def ui_suggest_ok(self, _action, _param): self.provider['trans'].history[self.current_history]['Languages'][0], self.provider['trans'].history[self.current_history]['Languages'][1] ) - message = self.provider['trans'].format_suggestion( + + self.provider['trans'].suggest( self.provider['trans'].history[self.current_history]['Text'][0], src, dest, - dest_text + dest_text, + on_done, + on_fail ) - Session.get().send_and_read_async(message, 0, None, self.on_suggest_response) self.before_suggest = None @@ -820,23 +789,7 @@ def ui_suggest_cancel(self, _action, _param): if self.before_suggest is not None: self.dest_buffer.props.text = self.before_suggest self.before_suggest = None - self.dest_text.props.editable = False - - def on_suggest_response(self, session, result): - success = False - self.dest_toolbar_stack.props.visible_child_name = 'default' - try: - data = Session.get_response(session, result) - success = self.provider['trans'].get_suggestion(data) - except Exception as exc: - logging.error(exc) - - if success: - self.send_notification(_('New translation has been suggested!')) - else: - self.send_notification(_('Suggestion failed.')) - - self.dest_text.props.editable = False + self.dest_text.props.editable = False def ui_src_voice(self, _action, _param): src_text = self.src_buffer.get_text( @@ -867,80 +820,44 @@ def _pre_speech(self, text, lang, called_from): 'called_from': called_from } - if not self.provider['tts'].error: - self.download_speech() - else: - self.load_tts() + self.download_speech() def on_gst_message(self, _bus, message): if message.type == Gst.MessageType.EOS: self.player.set_state(Gst.State.NULL) - self.player_event.set() elif message.type == Gst.MessageType.ERROR: self.player.set_state(Gst.State.NULL) - self.player_event.set() logging.error('Some error occurred while trying to play.') def download_speech(self): - if self.current_speech: - match self._check_provider_type(self.provider['tts'].__provider_type__, 'tts'): - case 'local': - threading.Thread( - target=self._download_local_tts, - daemon=True - ).start() - case 'soup': - lang = self.provider['tts'].denormalize_lang(self.current_speech['lang']) - message = self.provider['tts'].format_speech(self.current_speech['text'], lang) - Session.get().send_and_read_async( - message, - 0, - None, - self._on_tts_downloaded - ) - else: - self.toggle_voice_spinner(False) - self.voice_loading = False + def on_done(file): + try: + self._play_audio(file.name) + file.close() + except Exception as exc: + logging.error(exc) + self.on_listen_failed() + else: + self.toggle_voice_spinner(False) + finally: + self.voice_loading = False + self.current_speech = {} - def _download_local_tts(self): - """ Downlaod and play speech from local provider """ - try: - with NamedTemporaryFile() as file_to_play: - lang = self.provider['tts'].denormalize_lang(self.current_speech['lang']) - self.provider['tts'].download_speech(self.current_speech['text'], lang, file_to_play) - self._play_audio(file_to_play.name) - except Exception as exc: - logging.error(exc) - logging.error('Audio download failed.') - GLib.idle_add(self.on_listen_failed) - else: - GLib.idle_add(self.toggle_voice_spinner, False) - finally: - self.voice_loading = False - self.current_speech = {} - - def _on_tts_downloaded(self, session, result): - """ Write and play speech from libsoup provider """ - try: - data = Session.get_response(session, result) - with NamedTemporaryFile() as file_to_play: - self.provider['tts'].get_speech(data, file_to_play) - self._play_audio(file_to_play.name) - except Exception as exc: - logging.error(exc) + def on_fail(_error: ProviderError): self.on_listen_failed() + self.toggle_voice_spinner(False) + + if self.current_speech: + lang = self.provider['tts'].denormalize_lang(self.current_speech['lang']) + self.provider['tts'].speech(self.current_speech['text'], lang, on_done, on_fail) else: self.toggle_voice_spinner(False) - finally: self.voice_loading = False - self.current_speech = {} def _play_audio(self, path): uri = 'file://' + path self.player.set_property('uri', uri) self.player.set_state(Gst.State.PLAYING) - if self._check_provider_type(self.provider['tts'].__provider_type__, 'tts') == 'local': - self.player_event.wait() @Gtk.Template.Callback() def _on_key_event(self, _button, keyval, _keycode, state): @@ -1025,7 +942,7 @@ def on_dest_text_changed(self, buffer): sensitive = buffer.get_char_count() != 0 self.lookup_action('copy').props.enabled = sensitive self.lookup_action('suggest').set_enabled( - self.provider['trans'].suggestions + ProviderFeature.SUGGESTIONS in self.provider['trans'].features and sensitive ) if not self.voice_loading and self.provider['tts']: @@ -1106,77 +1023,48 @@ def translation(self, _action=None, _param=None): self.ongoing_trans = True src, dest = self.provider['trans'].denormalize_lang(src_language, dest_language) - message = self.provider['trans'].format_translation( - src_text, src, dest - ) - - Session.get().send_and_read_async( - message, - 0, - None, - self.on_translation_response, - (src_text, src_language, dest_language) + self.provider['trans'].translate( + src_text, + src, + dest, + self.on_translation_success, + self.on_translation_fail ) else: - self.trans_failed = False self.trans_mistakes = [None, None] self.trans_src_pron = None self.trans_dest_pron = None self.dest_buffer.props.text = '' if not self.ongoing_trans: - self.translation_done() - - def on_translation_response(self, session, result, original): - error = '' - dest_text = '' - - self.trans_mistakes = [None, None] - self.trans_src_pron = None - self.trans_dest_pron = None - try: - data = Session.get_response(session, result) - (translation, lang) = self.provider['trans'].get_translation(data) - - if lang and self.src_lang_selector.selected == 'auto': - if Settings.get().src_auto: - self.src_lang_selector.set_insight(lang) - else: - self.src_lang_selector.selected = lang - - dest_text = translation.text - self.trans_mistakes = translation.extra_data['possible-mistakes'] - self.trans_src_pron = translation.extra_data['src-pronunciation'] - self.trans_dest_pron = translation.extra_data['dest-pronunciation'] - self.trans_failed = False - - self.dest_buffer.props.text = dest_text - - # Finally, everything is saved in history - self.add_history_entry( - original[0], - original[1], - original[2], - dest_text - ) - except ResponseError as exc: - logging.error(exc) - error = 'network' - self.trans_failed = True - except InvalidApiKey as exc: - logging.error(exc) - error = 'invalid-api' - self.trans_failed = True - except ApiKeyRequired as exc: - logging.error(exc) - error = 'api-required' - self.trans_failed = True - except Exception as exc: - logging.error(exc) - self.trans_failed = True + self.translation_finish() + + def on_translation_success(self, translation, lang): + self.trans_warning.props.visible = False + + if lang and self.src_lang_selector.selected == 'auto': + if Settings.get().src_auto: + self.src_lang_selector.set_insight(lang) + else: + self.src_lang_selector.selected = lang + + self.dest_buffer.props.text = translation.text + + self.trans_mistakes = translation.extra_data['possible-mistakes'] + self.trans_src_pron = translation.extra_data['src-pronunciation'] + self.trans_dest_pron = translation.extra_data['dest-pronunciation'] + + # FIXME: Make history work again + # Finally, everything is saved in history + """self.add_history_entry( + original[0], + original[1], + original[2], + dest_text + )""" # Mistakes - if self.provider['trans'].mistakes and not self.trans_mistakes == [None, None]: + if ProviderFeature.MISTAKES in self.provider['trans'].features and not self.trans_mistakes == [None, None]: self.mistakes_label.set_markup(_('Did you mean: ') + f'{self.trans_mistakes[0]}') self.mistakes.props.reveal_child = True elif self.mistakes.props.reveal_child: @@ -1184,7 +1072,7 @@ def on_translation_response(self, session, result, original): # Pronunciation reveal = Settings.get().show_pronunciation - if self.provider['trans'].pronunciation: + if ProviderFeature.PRONUNCIATION in self.provider['trans'].features: if self.trans_src_pron is not None and self.trans_mistakes == [None, None]: self.src_pron_label.props.label = self.trans_src_pron self.src_pron_revealer.props.reveal_child = reveal @@ -1197,34 +1085,20 @@ def on_translation_response(self, session, result, original): elif self.dest_pron_revealer.props.reveal_child: self.dest_pron_revealer.props.reveal_child = False - self.translation_failed(self.trans_failed, error) - self.ongoing_trans = False if self.next_trans: self.translation() else: - self.translation_done() + self.translation_finish() - def translation_loading(self): - self.trans_spinner.show() - self.trans_spinner.start() - self.dest_box.props.sensitive = False - self.langs_button_box.props.sensitive = False - - def translation_done(self): - self.trans_spinner.stop() - self.trans_spinner.hide() - self.dest_box.props.sensitive = True - self.langs_button_box.props.sensitive = True + def on_translation_fail(self, error: ProviderError): + self.trans_warning.props.visible = True + self.lookup_action('copy').props.enabled = False + self.lookup_action('listen-src').props.enabled = False + self.lookup_action('listen-dest').props.enabled = False - def translation_failed(self, failed, error=''): - if failed: - self.trans_warning.show() - self.lookup_action('copy').props.enabled = False - self.lookup_action('listen-src').props.enabled = False - self.lookup_action('listen-dest').props.enabled = False - - if error == 'network': + match error.code: + case ProviderErrorCode.NETWORK: self.send_notification( _('Translation failed, check for network issues'), action={ @@ -1232,15 +1106,15 @@ def translation_failed(self, failed, error=''): 'name': 'win.translation', } ) - elif error == 'invalid-api': + case ProviderErrorCode.API_KEY_INVALID: self.send_notification( - _('The provided API key is invalid'), + _('Translation failed, check for network issues'), action={ - 'label': _('Preferences'), - 'name': 'app.preferences', + 'label': _('Retry'), + 'name': 'win.translation', } ) - elif error == 'api-required': + case ProviderErrorCode.API_KEY_REQUIRED: self.send_notification( _('API key is required to use the service'), action={ @@ -1248,16 +1122,26 @@ def translation_failed(self, failed, error=''): 'name': 'app.preferences', } ) - else: + case _: self.send_notification( _('Translation failed'), action={ 'label': _('Retry'), 'name': 'win.translation', } - ) - else: - self.trans_warning.hide() + ) + + def translation_loading(self): + self.trans_spinner.show() + self.trans_spinner.start() + self.dest_box.props.sensitive = False + self.langs_button_box.props.sensitive = False + + def translation_finish(self): + self.trans_spinner.stop() + self.trans_spinner.hide() + self.dest_box.props.sensitive = True + self.langs_button_box.props.sensitive = True def reload_translator(self): self.translator_loading = True From d11d84946b9e7b0bbe7ad4d10d0ca44abd19263b Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Sun, 1 Oct 2023 19:27:14 -0500 Subject: [PATCH 02/15] provider: Improve Translation class --- dialect/providers/base.py | 20 +++++++++++--------- dialect/providers/bing.py | 9 +++------ dialect/providers/google.py | 10 ++++------ dialect/providers/libretrans.py | 13 +++---------- dialect/providers/lingva.py | 9 ++------- dialect/providers/yandex.py | 11 ++--------- dialect/window.py | 22 +++++++++++----------- 7 files changed, 36 insertions(+), 58 deletions(-) diff --git a/dialect/providers/base.py b/dialect/providers/base.py index 84dc9510..03c2e36c 100644 --- a/dialect/providers/base.py +++ b/dialect/providers/base.py @@ -66,16 +66,18 @@ def __init__(self, code: ProviderErrorCode, message: str = '') -> None: class Translation: - text = None - extra_data = { - 'possible-mistakes': [None, None], - 'src-pronunciation': None, - 'dest-pronunciation': None, - } - def __init__(self, text, extra_data): + def __init__( + self, + text: str, + detected: None | str = None, + mistakes: tuple[None | str, None | str] = (None, None), + pronunciation: tuple[None | str, None | str] = (None, None), + ): self.text = text - self.extra_data = extra_data + self.detected = detected + self.mistakes = mistakes + self.pronunciation = pronunciation class BaseProvider: @@ -243,7 +245,7 @@ def translate( text: str, src: str, dest: str, - on_done: Callable[[Translation, str | None], None], + on_done: Callable[[Translation], None], on_fail: Callable[[ProviderError], None], ): raise NotImplementedError() diff --git a/dialect/providers/bing.py b/dialect/providers/bing.py index 4810dd97..4d4feee6 100644 --- a/dialect/providers/bing.py +++ b/dialect/providers/bing.py @@ -124,13 +124,10 @@ def on_response(data): translation = Translation( data['translations'][0]['text'], - { - 'possible-mistakes': [None, None], - 'src-pronunciation': None, - 'dest-pronunciation': pronunciation, - }, + detected, + pronunciation=(None, pronunciation) ) - on_done(translation, detected) + on_done(translation) except Exception as exc: logging.warning(exc) diff --git a/dialect/providers/google.py b/dialect/providers/google.py index e33b4e5d..62daf767 100644 --- a/dialect/providers/google.py +++ b/dialect/providers/google.py @@ -515,13 +515,11 @@ def on_response(session, result): result = Translation( translated, - { - 'possible-mistakes': [mistake, self._strip_html_tags(mistake)], - 'src-pronunciation': origin_pronunciation, - 'dest-pronunciation': pronunciation, - }, + src, + (mistake, self._strip_html_tags(mistake)), + (origin_pronunciation, pronunciation), ) - on_done(result, src) + on_done(result) except Exception as exc: logging.warning(exc) diff --git a/dialect/providers/libretrans.py b/dialect/providers/libretrans.py index 73411798..3ccbe243 100644 --- a/dialect/providers/libretrans.py +++ b/dialect/providers/libretrans.py @@ -40,7 +40,7 @@ def on_response(data): try: valid = data['info']['title'] == 'LibreTranslate' - except: # noqa + except: # noqa pass on_done(valid) @@ -148,15 +148,8 @@ def on_response(data): def translate(self, text, src, dest, on_done, on_fail): def on_response(data): detected = data.get('detectedLanguage', {}).get('language', None) - translation = Translation( - data['translatedText'], - { - 'possible-mistakes': [None, None], - 'src-pronunciation': None, - 'dest-pronunciation': None, - }, - ) - on_done(translation, detected) + translation = Translation(data['translatedText'], detected) + on_done(translation) # Request body data = { diff --git a/dialect/providers/lingva.py b/dialect/providers/lingva.py index 3e986f1f..c9b83dd2 100644 --- a/dialect/providers/lingva.py +++ b/dialect/providers/lingva.py @@ -95,15 +95,10 @@ def on_response(data): dest_pronunciation = data['info']['pronunciation'].get('translation', None) translation = Translation( - data['translation'], - { - 'possible-mistakes': [mistakes, mistakes], - 'src-pronunciation': src_pronunciation, - 'dest-pronunciation': dest_pronunciation, - }, + data['translation'], detected, (mistakes, mistakes), (src_pronunciation, dest_pronunciation) ) - on_done(translation, detected) + on_done(translation) except Exception as exc: error = 'Failed reading the translation data' diff --git a/dialect/providers/yandex.py b/dialect/providers/yandex.py index 87cca347..682889a4 100644 --- a/dialect/providers/yandex.py +++ b/dialect/providers/yandex.py @@ -157,15 +157,8 @@ def on_response(data): detected = data['lang'].split('-')[0] if 'text' in data: - result = Translation( - data['text'][0], - { - 'possible-mistakes': None, - 'src-pronunciation': None, - 'dest-pronunciation': None, - }, - ) - on_done(result, detected) + translation = Translation(data['text'][0], detected) + on_done(translation) else: on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, 'Translation failed')) diff --git a/dialect/window.py b/dialect/window.py index 743d8208..44bf2f52 100644 --- a/dialect/window.py +++ b/dialect/window.py @@ -11,7 +11,7 @@ from dialect.define import APP_ID, PROFILE, RES_PATH, TRANS_NUMBER from dialect.languages import LanguagesListModel from dialect.providers import TRANSLATORS, TTS, ProviderFeature, ProviderError, ProviderErrorCode -from dialect.providers.base import BaseProvider +from dialect.providers.base import BaseProvider, Translation from dialect.settings import Settings from dialect.shortcuts import DialectShortcutsWindow from dialect.widgets import LangSelector, ThemeSwitcher @@ -1031,7 +1031,7 @@ def translation(self, _action=None, _param=None): self.on_translation_fail ) else: - self.trans_mistakes = [None, None] + self.trans_mistakes = (None, None) self.trans_src_pron = None self.trans_dest_pron = None self.dest_buffer.props.text = '' @@ -1039,20 +1039,20 @@ def translation(self, _action=None, _param=None): if not self.ongoing_trans: self.translation_finish() - def on_translation_success(self, translation, lang): + def on_translation_success(self, translation: Translation): self.trans_warning.props.visible = False - if lang and self.src_lang_selector.selected == 'auto': + if translation.detected and self.src_lang_selector.selected == 'auto': if Settings.get().src_auto: - self.src_lang_selector.set_insight(lang) + self.src_lang_selector.set_insight(translation.detected) else: - self.src_lang_selector.selected = lang + self.src_lang_selector.selected = translation.detected self.dest_buffer.props.text = translation.text - self.trans_mistakes = translation.extra_data['possible-mistakes'] - self.trans_src_pron = translation.extra_data['src-pronunciation'] - self.trans_dest_pron = translation.extra_data['dest-pronunciation'] + self.trans_mistakes = translation.mistakes + self.trans_src_pron = translation.pronunciation[0] + self.trans_dest_pron = translation.pronunciation[1] # FIXME: Make history work again # Finally, everything is saved in history @@ -1064,7 +1064,7 @@ def on_translation_success(self, translation, lang): )""" # Mistakes - if ProviderFeature.MISTAKES in self.provider['trans'].features and not self.trans_mistakes == [None, None]: + if ProviderFeature.MISTAKES in self.provider['trans'].features and not self.trans_mistakes == (None, None): self.mistakes_label.set_markup(_('Did you mean: ') + f'{self.trans_mistakes[0]}') self.mistakes.props.reveal_child = True elif self.mistakes.props.reveal_child: @@ -1073,7 +1073,7 @@ def on_translation_success(self, translation, lang): # Pronunciation reveal = Settings.get().show_pronunciation if ProviderFeature.PRONUNCIATION in self.provider['trans'].features: - if self.trans_src_pron is not None and self.trans_mistakes == [None, None]: + if self.trans_src_pron is not None and self.trans_mistakes == (None, None): self.src_pron_label.props.label = self.trans_src_pron self.src_pron_revealer.props.reveal_child = reveal elif self.src_pron_revealer.props.reveal_child: From c7074c147bf7db786c2f882b2902fd4a62e0edef Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Sun, 1 Oct 2023 19:56:55 -0500 Subject: [PATCH 03/15] providers: Organize modules --- dialect/providers/__init__.py | 19 ++- dialect/providers/base.py | 118 +----------------- dialect/providers/local.py | 15 +++ dialect/providers/{ => modules}/bing.py | 2 +- dialect/providers/{ => modules}/google.py | 4 +- dialect/providers/{ => modules}/libretrans.py | 2 +- dialect/providers/{ => modules}/lingva.py | 2 +- dialect/providers/{ => modules}/yandex.py | 2 +- dialect/providers/soup.py | 116 +++++++++++++++++ 9 files changed, 147 insertions(+), 133 deletions(-) create mode 100644 dialect/providers/local.py rename dialect/providers/{ => modules}/bing.py (99%) rename dialect/providers/{ => modules}/google.py (99%) rename dialect/providers/{ => modules}/libretrans.py (99%) rename dialect/providers/{ => modules}/lingva.py (99%) rename dialect/providers/{ => modules}/yandex.py (98%) create mode 100644 dialect/providers/soup.py diff --git a/dialect/providers/__init__.py b/dialect/providers/__init__.py index ffbeed21..0dce1b35 100644 --- a/dialect/providers/__init__.py +++ b/dialect/providers/__init__.py @@ -8,20 +8,19 @@ from gi.repository import Gio, GObject from dialect.providers.base import ProviderCapability, ProviderFeature, ProviderError, ProviderErrorCode # noqa - +from dialect.providers import modules MODULES = {} TRANSLATORS = {} TTS = {} -for _importer, modname, _ispkg in pkgutil.iter_modules(__path__): - if modname != 'base' and not modname.startswith('_'): - modclass = importlib.import_module('dialect.providers.' + modname).Provider - MODULES[modclass.name] = modclass - if modclass.capabilities: - if ProviderCapability.TRANSLATION in modclass.capabilities: - TRANSLATORS[modclass.name] = modclass - if ProviderCapability.TTS in modclass.capabilities: - TTS[modclass.name] = modclass +for _importer, modname, _ispkg in pkgutil.iter_modules(modules.__path__): + modclass = importlib.import_module('dialect.providers.modules.' + modname).Provider + MODULES[modclass.name] = modclass + if modclass.capabilities: + if ProviderCapability.TRANSLATION in modclass.capabilities: + TRANSLATORS[modclass.name] = modclass + if ProviderCapability.TTS in modclass.capabilities: + TTS[modclass.name] = modclass def check_translator_availability(provider_name): diff --git a/dialect/providers/base.py b/dialect/providers/base.py index 03c2e36c..3d13df09 100644 --- a/dialect/providers/base.py +++ b/dialect/providers/base.py @@ -3,18 +3,14 @@ # SPDX-License-Identifier: GPL-3.0-or-later import io -import json -import logging import urllib.parse -import threading from enum import Enum, Flag, auto from typing import Callable -from gi.repository import GLib, Gio, Soup +from gi.repository import Gio from dialect.define import APP_ID from dialect.languages import get_lang_name, normalize_lang_code -from dialect.session import Session class ProviderCapability(Flag): @@ -269,115 +265,3 @@ def speech( on_fail: Callable[[ProviderError], None], ): raise NotImplementedError() - - -class LocalProvider(BaseProvider): - """Base class for providers needing local threaded helpers""" - - def launch_thread(self, callback: Callable, *args): - threading.Thread(target=callback, args=args, daemon=True).start() - - -class SoupProvider(BaseProvider): - """Base class for providers needing libsoup helpers""" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - @staticmethod - def encode_data(data) -> GLib.Bytes | None: - """Convert dict to JSON and bytes""" - data_glib_bytes = None - try: - data_bytes = json.dumps(data).encode('utf-8') - data_glib_bytes = GLib.Bytes.new(data_bytes) - except Exception as exc: - logging.warning(exc) - return data_glib_bytes - - @staticmethod - def read_data(data: bytes) -> dict: - """Get JSON data from bytes""" - return json.loads(data) if data else {} - - @staticmethod - def read_response(session: Session, result: Gio.AsyncResult) -> dict: - """Get JSON data from session result""" - response = session.get_response(session, result) - return SoupProvider.read_data(response) - - @staticmethod - def create_message(method: str, url: str, data={}, headers: dict = {}, form: bool = False) -> Soup.Message: - """Helper for creating libsoup's message""" - - if form and data: - form_data = Soup.form_encode_hash(data) - message = Soup.Message.new_from_encoded_form(method, url, form_data) - else: - message = Soup.Message.new(method, url) - if data and not form: - data = SoupProvider.encode_data(data) - message.set_request_body_from_bytes('application/json', data) - if headers: - for name, value in headers.items(): - message.get_request_headers().append(name, value) - if 'User-Agent' not in headers: - message.get_request_headers().append('User-Agent', 'Dialect App') - return message - - @staticmethod - def send_and_read(message: Soup.Message, callback: Callable[[Session, Gio.AsyncResult], None]): - """Helper method for libsoup's send_and_read_async - Useful when priority and cancellable is not needed""" - Session.get().send_and_read_async(message, 0, None, callback) - - @staticmethod - def check_known_errors(data: dict) -> None | ProviderError: - """Checks data for possible response errors and return a found error if any - This should be implemented by subclases""" - return None - - @staticmethod - def process_response( - session: Session, - result: Gio.AsyncResult, - on_continue: Callable[[dict], None], - on_fail: Callable[[ProviderError], None], - check_common: bool = True, - ): - """Helper method for the most common workflow for processing soup responses - - Checks for soup errors, then checks for common errors on data and calls on_fail - if any, otherwise calls on_continue where the provider will finish the process. - """ - - try: - data = SoupProvider.read_response(session, result) - - if check_common: - error = SoupProvider.check_known_errors(data) - if error: - on_fail(error) - return - - on_continue(data) - - except Exception as exc: - logging.warning(exc) - on_fail(ProviderError(ProviderErrorCode.NETWORK, str(exc))) - - @staticmethod - def send_and_read_and_process_response( - message: Soup.Message, - on_continue: Callable[[dict], None], - on_fail: Callable[[ProviderError], None], - check_common: bool = True, - ): - """Helper packaging send_and_read and process_response - - Avoids implementors having to deal with many callbacks.""" - - def on_response(session: Session, result: Gio.AsyncResult): - SoupProvider.process_response(session, result, on_continue, on_fail, check_common) - - SoupProvider.send_and_read(message, on_response) diff --git a/dialect/providers/local.py b/dialect/providers/local.py new file mode 100644 index 00000000..a350c32e --- /dev/null +++ b/dialect/providers/local.py @@ -0,0 +1,15 @@ +# Copyright 2023 Mufeed Ali +# Copyright 2023 Rafael Mardojai CM +# SPDX-License-Identifier: GPL-3.0-or-later + +import threading +from typing import Callable + +from dialect.providers.base import BaseProvider + + +class LocalProvider(BaseProvider): + """Base class for providers needing local threaded helpers""" + + def launch_thread(self, callback: Callable, *args): + threading.Thread(target=callback, args=args, daemon=True).start() diff --git a/dialect/providers/bing.py b/dialect/providers/modules/bing.py similarity index 99% rename from dialect/providers/bing.py rename to dialect/providers/modules/bing.py index 4d4feee6..77417276 100644 --- a/dialect/providers/bing.py +++ b/dialect/providers/modules/bing.py @@ -11,9 +11,9 @@ ProviderFeature, ProviderError, ProviderErrorCode, - SoupProvider, Translation, ) +from dialect.providers.soup import SoupProvider from dialect.session import Session diff --git a/dialect/providers/google.py b/dialect/providers/modules/google.py similarity index 99% rename from dialect/providers/google.py rename to dialect/providers/modules/google.py index 62daf767..d5261309 100644 --- a/dialect/providers/google.py +++ b/dialect/providers/modules/google.py @@ -13,14 +13,14 @@ from gtts import gTTS, lang from dialect.providers.base import ( - LocalProvider, ProviderCapability, ProviderError, ProviderErrorCode, ProviderFeature, - SoupProvider, Translation, ) +from dialect.providers.local import LocalProvider +from dialect.providers.soup import SoupProvider RPC_ID = 'MkEWBc' diff --git a/dialect/providers/libretrans.py b/dialect/providers/modules/libretrans.py similarity index 99% rename from dialect/providers/libretrans.py rename to dialect/providers/modules/libretrans.py index 3ccbe243..c74773b3 100644 --- a/dialect/providers/libretrans.py +++ b/dialect/providers/modules/libretrans.py @@ -9,9 +9,9 @@ ProviderFeature, ProviderErrorCode, ProviderError, - SoupProvider, Translation, ) +from dialect.providers.soup import SoupProvider class Provider(SoupProvider): diff --git a/dialect/providers/lingva.py b/dialect/providers/modules/lingva.py similarity index 99% rename from dialect/providers/lingva.py rename to dialect/providers/modules/lingva.py index c9b83dd2..8608c9cf 100644 --- a/dialect/providers/lingva.py +++ b/dialect/providers/modules/lingva.py @@ -11,9 +11,9 @@ ProviderError, ProviderErrorCode, ProviderFeature, - SoupProvider, Translation, ) +from dialect.providers.soup import SoupProvider class Provider(SoupProvider): diff --git a/dialect/providers/yandex.py b/dialect/providers/modules/yandex.py similarity index 98% rename from dialect/providers/yandex.py rename to dialect/providers/modules/yandex.py index 682889a4..38a28af8 100644 --- a/dialect/providers/yandex.py +++ b/dialect/providers/modules/yandex.py @@ -8,9 +8,9 @@ ProviderError, ProviderErrorCode, ProviderFeature, - SoupProvider, Translation, ) +from dialect.providers.soup import SoupProvider class Provider(SoupProvider): diff --git a/dialect/providers/soup.py b/dialect/providers/soup.py new file mode 100644 index 00000000..854d70cc --- /dev/null +++ b/dialect/providers/soup.py @@ -0,0 +1,116 @@ +# Copyright 2022 Mufeed Ali +# Copyright 2022 Rafael Mardojai CM +# SPDX-License-Identifier: GPL-3.0-or-later + +import json +import logging +from typing import Callable + +from gi.repository import GLib, Gio, Soup + +from dialect.providers.base import BaseProvider, ProviderError, ProviderErrorCode +from dialect.session import Session + +class SoupProvider(BaseProvider): + """Base class for providers needing libsoup helpers""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @staticmethod + def encode_data(data) -> GLib.Bytes | None: + """Convert dict to JSON and bytes""" + data_glib_bytes = None + try: + data_bytes = json.dumps(data).encode('utf-8') + data_glib_bytes = GLib.Bytes.new(data_bytes) + except Exception as exc: + logging.warning(exc) + return data_glib_bytes + + @staticmethod + def read_data(data: bytes) -> dict: + """Get JSON data from bytes""" + return json.loads(data) if data else {} + + @staticmethod + def read_response(session: Session, result: Gio.AsyncResult) -> dict: + """Get JSON data from session result""" + response = session.get_response(session, result) + return SoupProvider.read_data(response) + + @staticmethod + def create_message(method: str, url: str, data={}, headers: dict = {}, form: bool = False) -> Soup.Message: + """Helper for creating libsoup's message""" + + if form and data: + form_data = Soup.form_encode_hash(data) + message = Soup.Message.new_from_encoded_form(method, url, form_data) + else: + message = Soup.Message.new(method, url) + if data and not form: + data = SoupProvider.encode_data(data) + message.set_request_body_from_bytes('application/json', data) + if headers: + for name, value in headers.items(): + message.get_request_headers().append(name, value) + if 'User-Agent' not in headers: + message.get_request_headers().append('User-Agent', 'Dialect App') + return message + + @staticmethod + def send_and_read(message: Soup.Message, callback: Callable[[Session, Gio.AsyncResult], None]): + """Helper method for libsoup's send_and_read_async + Useful when priority and cancellable is not needed""" + Session.get().send_and_read_async(message, 0, None, callback) + + @staticmethod + def check_known_errors(data: dict) -> None | ProviderError: + """Checks data for possible response errors and return a found error if any + This should be implemented by subclases""" + return None + + @staticmethod + def process_response( + session: Session, + result: Gio.AsyncResult, + on_continue: Callable[[dict], None], + on_fail: Callable[[ProviderError], None], + check_common: bool = True, + ): + """Helper method for the most common workflow for processing soup responses + + Checks for soup errors, then checks for common errors on data and calls on_fail + if any, otherwise calls on_continue where the provider will finish the process. + """ + + try: + data = SoupProvider.read_response(session, result) + + if check_common: + error = SoupProvider.check_known_errors(data) + if error: + on_fail(error) + return + + on_continue(data) + + except Exception as exc: + logging.warning(exc) + on_fail(ProviderError(ProviderErrorCode.NETWORK, str(exc))) + + @staticmethod + def send_and_read_and_process_response( + message: Soup.Message, + on_continue: Callable[[dict], None], + on_fail: Callable[[ProviderError], None], + check_common: bool = True, + ): + """Helper packaging send_and_read and process_response + + Avoids implementors having to deal with many callbacks.""" + + def on_response(session: Session, result: Gio.AsyncResult): + SoupProvider.process_response(session, result, on_continue, on_fail, check_common) + + SoupProvider.send_and_read(message, on_response) From ae950a072e7ef7c69ff9d3de56ad470a12b64333 Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Wed, 4 Oct 2023 10:10:40 -0500 Subject: [PATCH 04/15] providers: API: Add ProviderFeature.NONE We want Provider.features to always be iterable, so let's add a new flag to represent None --- dialect/providers/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dialect/providers/base.py b/dialect/providers/base.py index 3d13df09..e6dbd63d 100644 --- a/dialect/providers/base.py +++ b/dialect/providers/base.py @@ -23,6 +23,8 @@ class ProviderCapability(Flag): class ProviderFeature(Flag): + NONE = auto() + """ Provider has no features """ INSTANCES = auto() """ If it supports changing the instance url """ API_KEY = auto() @@ -83,7 +85,7 @@ class BaseProvider: """ Module name for UI display """ capabilities: ProviderCapability | None = None """ Provider capabilities, translation, tts, etc """ - features: ProviderFeature | None = None + features: ProviderFeature = ProviderFeature.NONE """ Provider features """ defaults = { From f45cd7a33d4816408ca269e0d03e10b22150ed1c Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Wed, 4 Oct 2023 17:44:04 -0500 Subject: [PATCH 05/15] providers: Allow providers implementations to fail Adds a try block when dynamically importing the providers python modules. In the future we might want to include optional providers that only work when some dependency is met, for example some offline providers could not satisfy its deps in a arm machine. In the future we probably will want to expose failing providers in the UI. --- dialect/providers/__init__.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/dialect/providers/__init__.py b/dialect/providers/__init__.py index 0dce1b35..6a1c7ae8 100644 --- a/dialect/providers/__init__.py +++ b/dialect/providers/__init__.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import importlib +import logging import pkgutil from gi.repository import Gio, GObject @@ -14,13 +15,16 @@ TRANSLATORS = {} TTS = {} for _importer, modname, _ispkg in pkgutil.iter_modules(modules.__path__): - modclass = importlib.import_module('dialect.providers.modules.' + modname).Provider - MODULES[modclass.name] = modclass - if modclass.capabilities: - if ProviderCapability.TRANSLATION in modclass.capabilities: - TRANSLATORS[modclass.name] = modclass - if ProviderCapability.TTS in modclass.capabilities: - TTS[modclass.name] = modclass + try: + modclass = importlib.import_module('dialect.providers.modules.' + modname).Provider + MODULES[modclass.name] = modclass + if modclass.capabilities: + if ProviderCapability.TRANSLATION in modclass.capabilities: + TRANSLATORS[modclass.name] = modclass + if ProviderCapability.TTS in modclass.capabilities: + TTS[modclass.name] = modclass + except Exception as exc: + logging.warning(f'Could not load the {modname} provider: {exc}') def check_translator_availability(provider_name): From 75561dac5c757ca41c07dfc93662125bf223deba Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Thu, 5 Oct 2023 22:29:34 -0500 Subject: [PATCH 06/15] providers: base: Move API methods to the top --- dialect/providers/base.py | 98 +++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/dialect/providers/base.py b/dialect/providers/base.py index e6dbd63d..29924eaf 100644 --- a/dialect/providers/base.py +++ b/dialect/providers/base.py @@ -64,7 +64,6 @@ def __init__(self, code: ProviderErrorCode, message: str = '') -> None: class Translation: - def __init__( self, text: str, @@ -80,7 +79,7 @@ def __init__( class BaseProvider: name = '' - """ Module name for itern use, like settings storing """ + """ Module name for code use, like settings storing """ prettyname = '' """ Module name for UI display """ capabilities: ProviderCapability | None = None @@ -94,6 +93,7 @@ class BaseProvider: 'src_langs': ['en', 'fr', 'es', 'de'], 'dest_langs': ['fr', 'es', 'de', 'en'], } + """ Default provider settings """ def __init__(self): self.languages = [] @@ -114,6 +114,53 @@ def __init__(self): # GSettings self.settings = Gio.Settings(f'{APP_ID}.translator', f'/app/drey/Dialect/translators/{self.name}/') + """ + Providers API methods + """ + + @staticmethod + def validate_instance(url: str, on_done: Callable[[bool], None], on_fail: Callable[[ProviderError], None]): + raise NotImplementedError() + + def validate_api_key(self, key: str, on_done: Callable[[bool], None], on_fail: Callable[[ProviderError], None]): + raise NotImplementedError() + + def init_trans(self, on_done: Callable, on_fail: Callable[[ProviderError], None]): + on_done() + + def init_tts(self, on_done: Callable, on_fail: Callable[[ProviderError], None]): + on_done() + + def translate( + self, + text: str, + src: str, + dest: str, + on_done: Callable[[Translation], None], + on_fail: Callable[[ProviderError], None], + ): + raise NotImplementedError() + + def suggest( + self, + text: str, + src: str, + dest: str, + suggestion: str, + on_done: Callable[[bool], None], + on_fail: Callable[[ProviderError], None], + ): + raise NotImplementedError() + + def speech( + self, + text: str, + language: str, + on_done: Callable[[io.BytesIO], None], + on_fail: Callable[[ProviderError], None], + ): + raise NotImplementedError() + """ Provider settings helpers and properties """ @@ -220,50 +267,3 @@ def get_lang_name(self, code): return self._languages_names.get(code, code) return name - - """ - Providers API methods - """ - - @staticmethod - def validate_instance(url: str, on_done: Callable[[bool], None], on_fail: Callable[[ProviderError], None]): - raise NotImplementedError() - - def validate_api_key(self, key: str, on_done: Callable[[bool], None], on_fail: Callable[[ProviderError], None]): - raise NotImplementedError() - - def init_trans(self, on_done: Callable, on_fail: Callable[[ProviderError], None]): - on_done() - - def init_tts(self, on_done: Callable, on_fail: Callable[[ProviderError], None]): - on_done() - - def translate( - self, - text: str, - src: str, - dest: str, - on_done: Callable[[Translation], None], - on_fail: Callable[[ProviderError], None], - ): - raise NotImplementedError() - - def suggest( - self, - text: str, - src: str, - dest: str, - suggestion: str, - on_done: Callable[[bool], None], - on_fail: Callable[[ProviderError], None], - ): - raise NotImplementedError() - - def speech( - self, - text: str, - language: str, - on_done: Callable[[io.BytesIO], None], - on_fail: Callable[[ProviderError], None], - ): - raise NotImplementedError() From 9df0d2b1f0b43312e01b64c763b37699d2113123 Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Fri, 6 Oct 2023 08:31:33 -0500 Subject: [PATCH 07/15] providers: soup: Allow returning bytes in the high level helper functions Update bing and google provider to make use of it. --- dialect/providers/base.py | 2 +- dialect/providers/modules/bing.py | 65 +++++---- dialect/providers/modules/google.py | 199 ++++++++++++++-------------- dialect/providers/soup.py | 19 ++- 4 files changed, 139 insertions(+), 146 deletions(-) diff --git a/dialect/providers/base.py b/dialect/providers/base.py index 29924eaf..2baebc45 100644 --- a/dialect/providers/base.py +++ b/dialect/providers/base.py @@ -43,7 +43,7 @@ class ProviderFeature(Flag): class ProviderErrorCode(Enum): UNEXPECTED = auto() - NETWORK = auto + NETWORK = auto() EMPTY = auto() API_KEY_REQUIRED = auto() API_KEY_INVALID = auto() diff --git a/dialect/providers/modules/bing.py b/dialect/providers/modules/bing.py index 77417276..ef4f875c 100644 --- a/dialect/providers/modules/bing.py +++ b/dialect/providers/modules/bing.py @@ -60,53 +60,48 @@ def translate_url(self): return self.format_url('www.bing.com', '/ttranslatev3', params) def init_trans(self, on_done, on_fail): - def on_response(session, result): - try: - data = Session.get_response(session, result) - if data: - try: - soup = BeautifulSoup(data, 'html.parser') - - # Get Langs - langs = soup.find('optgroup', {'id': 't_tgtAllLang'}) - for child in langs.findChildren(): - if child.name == 'option': - self.languages.append(child['value']) + def on_response(data): + if data: + try: + soup = BeautifulSoup(data, 'html.parser') - # Get IID - iid = soup.find('div', {'id': 'rich_tta'}) - self._iid = iid['data-iid'] + # Get Langs + langs = soup.find('optgroup', {'id': 't_tgtAllLang'}) + for child in langs.findChildren(): + if child.name == 'option': + self.languages.append(child['value']) - # Decode response bytes - data = data.decode('utf-8') + # Get IID + iid = soup.find('div', {'id': 'rich_tta'}) + self._iid = iid['data-iid'] - # Look for abuse prevention data - params = re.findall("var params_AbusePreventionHelper = \[(.*?)\];", data)[0] # noqa - abuse_params = params.replace('"', '').split(',') - self._key = abuse_params[0] - self._token = abuse_params[1] + # Decode response bytes + data = data.decode('utf-8') - # Look for IG - self._ig = re.findall("IG:\"(.*?)\",", data)[0] + # Look for abuse prevention data + params = re.findall("var params_AbusePreventionHelper = \[(.*?)\];", data)[0] # noqa + abuse_params = params.replace('"', '').split(',') + self._key = abuse_params[0] + self._token = abuse_params[1] - on_done() + # Look for IG + self._ig = re.findall("IG:\"(.*?)\",", data)[0] - except Exception as exc: - error = 'Failed parsing HTML from bing.com' - logging.warning(error, exc) - on_fail(ProviderError(ProviderErrorCode.NETWORK, error)) + on_done() - else: - on_fail(ProviderError(ProviderErrorCode.EMPTY, 'Could not get HTML from bing.com')) + except Exception as exc: + error = 'Failed parsing HTML from bing.com' + logging.warning(error, exc) + on_fail(ProviderError(ProviderErrorCode.NETWORK, error)) - except Exception as exc: - logging.warning(exc) - on_fail(ProviderError(ProviderErrorCode.NETWORK, str(exc))) + else: + on_fail(ProviderError(ProviderErrorCode.EMPTY, 'Could not get HTML from bing.com')) # Message request to get bing's website html message = self.create_message('GET', self.html_url, headers=self._headers) + # Do async request - self.send_and_read(message, on_response) + self.send_and_read_and_process_response(message, on_response, on_fail, json=False) def translate(self, text, src, dest, on_done, on_fail): def on_response(data): diff --git a/dialect/providers/modules/google.py b/dialect/providers/modules/google.py index d5261309..bac41fb5 100644 --- a/dialect/providers/modules/google.py +++ b/dialect/providers/modules/google.py @@ -410,124 +410,117 @@ def translate_url(self): return self.format_url(url, params=params) def translate(self, text, src_lang, dest_lang, on_done, on_fail): - def on_response(session, result): + def on_response(data): try: - data = session.get_response(session, result) - + token_found = False + square_bracket_counts = [0, 0] + resp = '' + data = data.decode('utf-8') + + for line in data.split('\n'): + token_found = token_found or f'"{RPC_ID}"' in line[:30] + if not token_found: + continue + + is_in_string = False + for index, char in enumerate(line): + if char == '\"' and line[max(0, index - 1)] != '\\': + is_in_string = not is_in_string + if not is_in_string: + if char == '[': + square_bracket_counts[0] += 1 + elif char == ']': + square_bracket_counts[1] += 1 + + resp += line + if square_bracket_counts[0] == square_bracket_counts[1]: + break + + data = json.loads(resp) + parsed = json.loads(data[0][2]) + translated_parts = None + translated = None try: - token_found = False - square_bracket_counts = [0, 0] - resp = '' - data = data.decode('utf-8') - - for line in data.split('\n'): - token_found = token_found or f'"{RPC_ID}"' in line[:30] - if not token_found: - continue - - is_in_string = False - for index, char in enumerate(line): - if char == '\"' and line[max(0, index - 1)] != '\\': - is_in_string = not is_in_string - if not is_in_string: - if char == '[': - square_bracket_counts[0] += 1 - elif char == ']': - square_bracket_counts[1] += 1 - - resp += line - if square_bracket_counts[0] == square_bracket_counts[1]: - break - - data = json.loads(resp) - parsed = json.loads(data[0][2]) - translated_parts = None - translated = None - try: - translated_parts = list( - map( - lambda part: TranslatedPart( - part[0] if len(part) > 0 else '', part[1] if len(part) >= 2 else [] - ), - parsed[1][0][0][5], - ) + translated_parts = list( + map( + lambda part: TranslatedPart( + part[0] if len(part) > 0 else '', part[1] if len(part) >= 2 else [] + ), + parsed[1][0][0][5], ) - except TypeError: - translated_parts = [ - TranslatedPart(parsed[1][0][1][0], [parsed[1][0][0][0], parsed[1][0][1][0]]) - ] - - first_iter = True - translated = "" - for part in translated_parts: - if not part.text.isspace() and not first_iter: - translated += " " - if first_iter: - first_iter = False - translated += part.text - - src = None - try: - src = parsed[1][-1][1] - except (IndexError, TypeError): - pass - - if not src == src_lang: - on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, 'source language mismatch')) - return + ) + except TypeError: + translated_parts = [ + TranslatedPart(parsed[1][0][1][0], [parsed[1][0][0][0], parsed[1][0][1][0]]) + ] + + first_iter = True + translated = "" + for part in translated_parts: + if not part.text.isspace() and not first_iter: + translated += " " + if first_iter: + first_iter = False + translated += part.text + + src = None + try: + src = parsed[1][-1][1] + except (IndexError, TypeError): + pass - if src == 'auto': - try: - if parsed[0][2] in self.languages: - src = parsed[0][2] - except (IndexError, TypeError): - pass + if not src == src_lang: + on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, 'source language mismatch')) + return - dest = None + if src == 'auto': try: - dest = parsed[1][-1][2] + if parsed[0][2] in self.languages: + src = parsed[0][2] except (IndexError, TypeError): pass - if not dest == dest_lang: - on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, 'destination language mismatch')) - return - - origin_pronunciation = None - try: - origin_pronunciation = parsed[0][0] - except (IndexError, TypeError): - pass + dest = None + try: + dest = parsed[1][-1][2] + except (IndexError, TypeError): + pass - pronunciation = None - try: - pronunciation = parsed[1][0][0][1] - except (IndexError, TypeError): - pass + if not dest == dest_lang: + on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, 'destination language mismatch')) + return - mistake = None - try: - mistake = parsed[0][1][0][0][1] - # Convert to pango markup - mistake = mistake.replace('', '').replace('', '') - except (IndexError, TypeError): - pass + origin_pronunciation = None + try: + origin_pronunciation = parsed[0][0] + except (IndexError, TypeError): + pass - result = Translation( - translated, - src, - (mistake, self._strip_html_tags(mistake)), - (origin_pronunciation, pronunciation), - ) - on_done(result) + pronunciation = None + try: + pronunciation = parsed[1][0][0][1] + except (IndexError, TypeError): + pass - except Exception as exc: - logging.warning(exc) - on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, str(exc))) + mistake = None + try: + mistake = parsed[0][1][0][0][1] + # Convert to pango markup + mistake = mistake.replace('', '').replace('', '') + except (IndexError, TypeError): + pass + + result = Translation( + translated, + src, + (mistake, self._strip_html_tags(mistake)), + (origin_pronunciation, pronunciation), + ) + on_done(result) except Exception as exc: logging.warning(exc) - on_fail(ProviderError(ProviderErrorCode.NETWORK, str(exc))) + on_fail(ProviderError(ProviderErrorCode.TRANSLATION_FAILED, str(exc))) # Form data data = { @@ -538,7 +531,7 @@ def on_response(session, result): message = self.create_message('POST', self.translate_url, data, self._headers, True) # Do async request - self.send_and_read(message, on_response) + self.send_and_read_and_process_response(message, on_response, on_fail, json=False) def _strip_html_tags(self, text): """Strip html tags""" diff --git a/dialect/providers/soup.py b/dialect/providers/soup.py index 854d70cc..7f947162 100644 --- a/dialect/providers/soup.py +++ b/dialect/providers/soup.py @@ -77,6 +77,7 @@ def process_response( on_continue: Callable[[dict], None], on_fail: Callable[[ProviderError], None], check_common: bool = True, + json: bool = True ): """Helper method for the most common workflow for processing soup responses @@ -85,13 +86,16 @@ def process_response( """ try: - data = SoupProvider.read_response(session, result) + if json: + data = SoupProvider.read_response(session, result) - if check_common: - error = SoupProvider.check_known_errors(data) - if error: - on_fail(error) - return + if check_common: + error = SoupProvider.check_known_errors(data) + if error: + on_fail(error) + return + else: + data = Session.get_response(session, result) on_continue(data) @@ -105,12 +109,13 @@ def send_and_read_and_process_response( on_continue: Callable[[dict], None], on_fail: Callable[[ProviderError], None], check_common: bool = True, + json: bool = True ): """Helper packaging send_and_read and process_response Avoids implementors having to deal with many callbacks.""" def on_response(session: Session, result: Gio.AsyncResult): - SoupProvider.process_response(session, result, on_continue, on_fail, check_common) + SoupProvider.process_response(session, result, on_continue, on_fail, check_common, json) SoupProvider.send_and_read(message, on_response) From 59e347de9b027387749718aea8e6d562267ec4ac Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Mon, 9 Oct 2023 07:11:14 -0500 Subject: [PATCH 08/15] providers: Move lang code normalization func to base provider Also adds the lang_aliases dict prop that providers can overwrite to define their own mapping to Dialect/CLDR's lang codes making it more scalable. We still keep the LANG_ALIASES cosnt in the define module to keep a well know mapping of lang codes provided by Dialect. --- dialect/define.in | 8 ++++++++ dialect/languages.py | 29 ----------------------------- dialect/providers/base.py | 30 +++++++++++++++++++++++++++--- 3 files changed, 35 insertions(+), 32 deletions(-) diff --git a/dialect/define.in b/dialect/define.in index be70731a..7d8d25a9 100644 --- a/dialect/define.in +++ b/dialect/define.in @@ -11,6 +11,14 @@ VERSION = '@VERSION@' TRANS_NUMBER = 10 # number of translations to save in history +LANG_ALIASES = { + 'iw': 'he', # Hebrew + 'jw': 'jv', # Javanese + 'mni-Mtei': 'mni', # Meiteilon (Manipuri) + 'zh-CN': 'zh-Hans', + 'zh-TW': 'zh-Hant', +} + LANGUAGES = { "aa": "Afar", "ab": "Abkhazian", diff --git a/dialect/languages.py b/dialect/languages.py index d83dd126..8f77c13f 100644 --- a/dialect/languages.py +++ b/dialect/languages.py @@ -7,35 +7,6 @@ from dialect.define import LANGUAGES -ALIASES = { - 'iw': 'he', # Hebrew - 'jw': 'jv', # Javanese - 'mni-Mtei': 'mni', # Meiteilon (Manipuri) - 'zh-CN': 'zh-Hans', - 'zh-TW': 'zh-Hant', -} - - -def normalize_lang_code(code): - code = code.replace('_', '-') # Normalize separator - codes = code.split('-') - - if len(codes) == 2: # Code contain a script or country code - - if len(codes[1]) == 4: # ISO 15924 (script) - codes[1] = codes[1].capitalize() - - elif len(codes[1]) == 2: # ISO 3166-1 (country) - codes[1] = codes[1].upper() - - code = '-'.join(codes) - - if code in ALIASES: - code = ALIASES[code] - - return code - - def get_lang_name(code): name = gettext(LANGUAGES.get(code, '')) return name if name else None diff --git a/dialect/providers/base.py b/dialect/providers/base.py index 2baebc45..24b23c93 100644 --- a/dialect/providers/base.py +++ b/dialect/providers/base.py @@ -9,8 +9,8 @@ from gi.repository import Gio -from dialect.define import APP_ID -from dialect.languages import get_lang_name, normalize_lang_code +from dialect.define import APP_ID, LANG_ALIASES +from dialect.languages import get_lang_name class ProviderCapability(Flag): @@ -160,6 +160,10 @@ def speech( on_fail: Callable[[ProviderError], None], ): raise NotImplementedError() + + @property + def lang_aliases(self) -> dict[str, str]: + return {} """ Provider settings helpers and properties @@ -229,11 +233,31 @@ def format_url(url: str, path: str = '', params: dict = {}, http: bool = False): params_str = '?' + params_str return protocol + url + path + params_str + + def normalize_lang_code(self, code): + code = code.replace('_', '-').lower() # Normalize separator + codes = code.split('-') + + if len(codes) == 2: # Code contain a script or country code + + if len(codes[1]) == 4: # ISO 15924 (script) + codes[1] = codes[1].capitalize() + + elif len(codes[1]) == 2: # ISO 3166-1 (country) + codes[1] = codes[1].upper() + + code = '-'.join(codes) + + aliases = {**LANG_ALIASES, **self.lang_aliases} + if code in aliases: + code = aliases[code] + + return code def add_lang(self, original_code, name=None, trans=True, tts=False): """Add lang supported by provider""" - code = normalize_lang_code(original_code) # Get normalized lang code + code = self.normalize_lang_code(original_code) # Get normalized lang code if trans: # Add lang to supported languages list self.languages.append(code) From c5d3d3c1cce215bbfb6c485f77b46adf7a20bd74 Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Tue, 10 Oct 2023 13:00:33 -0500 Subject: [PATCH 09/15] providers: Add docstrings to base classes --- dialect/providers/base.py | 147 ++++++++++++++++++++++++++++++++++--- dialect/providers/local.py | 11 ++- dialect/providers/soup.py | 112 ++++++++++++++++++++++------ 3 files changed, 233 insertions(+), 37 deletions(-) diff --git a/dialect/providers/base.py b/dialect/providers/base.py index 24b23c93..018903ad 100644 --- a/dialect/providers/base.py +++ b/dialect/providers/base.py @@ -120,15 +120,45 @@ def __init__(self): @staticmethod def validate_instance(url: str, on_done: Callable[[bool], None], on_fail: Callable[[ProviderError], None]): + """ + Validate an instance of the provider. + + Args: + url: The instance URL to test, only hostname and tld, e.g. libretranslate.com, localhost + on_done: Called when the validation is done, argument is the result of the validation + on_fail: Called when there's a fail in the validation process + """ raise NotImplementedError() def validate_api_key(self, key: str, on_done: Callable[[bool], None], on_fail: Callable[[ProviderError], None]): + """ + Validate an API key. + + Args: + key: The API key to validate + on_done: Called when the validation is done, argument is the result of the validation + on_fail: Called when there's a fail in the validation process + """ raise NotImplementedError() def init_trans(self, on_done: Callable, on_fail: Callable[[ProviderError], None]): + """ + Initializes the provider translation capabilities. + + Args: + on_done: Called after the provider was successfully initialized + on_fail: Called after any error on initialization + """ on_done() def init_tts(self, on_done: Callable, on_fail: Callable[[ProviderError], None]): + """ + Initializes the provider text-to-speech capabilities. + + Args: + on_done: Called after the provider was successfully initialized + on_fail: Called after any error on initialization + """ on_done() def translate( @@ -139,6 +169,16 @@ def translate( on_done: Callable[[Translation], None], on_fail: Callable[[ProviderError], None], ): + """ + Translates text in the provider. + + Args: + text: The text to translate + src: The lang code of the source text + dest: The lang code to translate the text to + on_done: Called after the text was successfully translated + on_fail: Called after any error on translation + """ raise NotImplementedError() def suggest( @@ -150,6 +190,17 @@ def suggest( on_done: Callable[[bool], None], on_fail: Callable[[ProviderError], None], ): + """ + Sends a translation suggestion to the provider. + + Args: + text: Original text without translation + src: The lang code of the original text + dest: The lang code of the translated text + suggestion: Suggested translation for text + on_done: Called after the suggestion was successfully send, argument means if it was accepted or rejected + on_fail: Called after any error on the suggestion process + """ raise NotImplementedError() def speech( @@ -159,10 +210,32 @@ def speech( on_done: Callable[[io.BytesIO], None], on_fail: Callable[[ProviderError], None], ): + """ + Generate speech audio from text + + Args: + text: Text to generate speech from + language: The lang code of text + on_done: Called after the process successful + on_fail: Called after any error on the speech process + """ raise NotImplementedError() - + @property def lang_aliases(self) -> dict[str, str]: + """ + Mapping of Dialect/CLDR's lang codes to the provider ones. + + Some providers might use different lang codes from the ones used by Dialect to for example get localized + language names. + + This dict is used by `add_lang` so lang codes can later be denormalized with `denormalize_lang`. + + Codes must be formatted with the criteria from normalize_lang_code, because this value would be used by + `add_lang` after normalization. + + Check `dialect.define.LANG_ALIASES` for reference mappings. + """ return {} """ @@ -171,6 +244,7 @@ def lang_aliases(self) -> dict[str, str]: @property def instance_url(self): + """Instance url saved on settings.""" return self.settings.get_string('instance-url') or self.defaults['instance_url'] @instance_url.setter @@ -178,10 +252,12 @@ def instance_url(self, url): self.settings.set_string('instance-url', url) def reset_instance_url(self): + """Resets saved instance url.""" self.instance_url = '' @property def api_key(self): + """API key saved on settings.""" return self.settings.get_string('api-key') or self.defaults['api_key'] @api_key.setter @@ -189,10 +265,12 @@ def api_key(self, api_key): self.settings.set_string('api-key', api_key) def reset_api_key(self): + """Resets saved API key.""" self.api_key = '' @property def src_langs(self): + """Saved recent source langs of the user.""" return self.settings.get_strv('src-langs') or self.defaults['src_langs'] @src_langs.setter @@ -200,10 +278,12 @@ def src_langs(self, src_langs): self.settings.set_strv('src-langs', src_langs) def reset_src_langs(self): + """Reset saved recent user source langs""" self.src_langs = [] @property def dest_langs(self): + """Saved recent destination langs of the user.""" return self.settings.get_strv('dest-langs') or self.defaults['dest_langs'] @dest_langs.setter @@ -211,6 +291,7 @@ def dest_langs(self, dest_langs): self.settings.set_strv('dest-langs', dest_langs) def reset_dest_langs(self): + """Reset saved recent user destination langs""" self.dest_langs = [] """ @@ -219,7 +300,16 @@ def reset_dest_langs(self): @staticmethod def format_url(url: str, path: str = '', params: dict = {}, http: bool = False): - """Formats a given url with path with the https protocol""" + """ + Compose a HTTP url with the given pieces. + + If url is localhost, `http` is ignored and HTTP protocol is forced. + + url: Base url, hostname and tld + path: Path of the url + params: Params to populate a url query + http: If HTTP should be used instead of HTTPS + """ if not path.startswith('/'): path = '/' + path @@ -233,13 +323,27 @@ def format_url(url: str, path: str = '', params: dict = {}, http: bool = False): params_str = '?' + params_str return protocol + url + path + params_str - - def normalize_lang_code(self, code): + + def normalize_lang_code(self, code: str): + """ + Normalice a language code to Dialect's criteria. + + Criteria: + - Codes must be lowercase, e.g. ES => es + - Codes can have a second code delimited by a hyphen, e.g. zh_CN => zh-CN + - If second code is two chars long it's considered a country code and must be uppercase, e.g. zh-cn => zh-CN + - If second code is four chars long it's considered a script code and must be capitalized, + e.g. zh-HANS => zh-Hans + + This method also maps lang codes aliases using `lang_aliases` and `dialect.define.LANG_ALIASES`. + + Args: + code: Language ISO code + """ code = code.replace('_', '-').lower() # Normalize separator codes = code.split('-') if len(codes) == 2: # Code contain a script or country code - if len(codes[1]) == 4: # ISO 15924 (script) codes[1] = codes[1].capitalize() @@ -254,8 +358,18 @@ def normalize_lang_code(self, code): return code - def add_lang(self, original_code, name=None, trans=True, tts=False): - """Add lang supported by provider""" + def add_lang(self, original_code: str, name: str = None, trans: bool = True, tts: bool = False): + """ + Add lang supported by provider after normalization. + + Normalized lang codes are saved for latter denormalization using `denormalize_lang`. + + Args: + original_code: Lang code to add + name: Language name to fallback in case Dialect doesn't provide one + trans: Add language as supported for translation + tts: Add language as supported for text-to-speech + """ code = self.normalize_lang_code(original_code) # Get normalized lang code @@ -272,8 +386,15 @@ def add_lang(self, original_code, name=None, trans=True, tts=False): # Save name provider by the service self._languages_names[code] = name - def denormalize_lang(self, *codes): - """Get denormalized lang code if available""" + def denormalize_lang(self, *codes: str) -> str | tuple[str]: + """ + Get denormalized lang code if available. + + This method will return a tuple with the same length of given codes or a str if only one code was passed. + + Args: + *codes: Lang codes to denormalize + """ if len(codes) == 1: return self._nonstandard_langs.get(codes[0], codes[0]) @@ -283,8 +404,12 @@ def denormalize_lang(self, *codes): result.append(self._nonstandard_langs.get(code, code)) return tuple(result) - def get_lang_name(self, code): - """Get language name""" + def get_lang_name(self, code: str) -> str: + """ + Get a localized language name. + + Fallback to a name provided by the provider if available or ultimately just the code. + """ name = get_lang_name(code) # Try getting translated name from Dialect if name is None: # Get name from provider if available diff --git a/dialect/providers/local.py b/dialect/providers/local.py index a350c32e..50d88044 100644 --- a/dialect/providers/local.py +++ b/dialect/providers/local.py @@ -11,5 +11,12 @@ class LocalProvider(BaseProvider): """Base class for providers needing local threaded helpers""" - def launch_thread(self, callback: Callable, *args): - threading.Thread(target=callback, args=args, daemon=True).start() + def launch_thread(self, worker: Callable, *args): + """ + Launches a thread using Python's threading. + + Args: + worker: Function to execute on the thread + *args: Args for the worker + """ + threading.Thread(target=worker, args=args, daemon=True).start() diff --git a/dialect/providers/soup.py b/dialect/providers/soup.py index 7f947162..d4b81f87 100644 --- a/dialect/providers/soup.py +++ b/dialect/providers/soup.py @@ -11,6 +11,7 @@ from dialect.providers.base import BaseProvider, ProviderError, ProviderErrorCode from dialect.session import Session + class SoupProvider(BaseProvider): """Base class for providers needing libsoup helpers""" @@ -19,7 +20,12 @@ def __init__(self, **kwargs): @staticmethod def encode_data(data) -> GLib.Bytes | None: - """Convert dict to JSON and bytes""" + """ + Convert Python data to JSON and bytes. + + Args: + data: Data to encode, anything json.dumps can handle + """ data_glib_bytes = None try: data_bytes = json.dumps(data).encode('utf-8') @@ -29,19 +35,20 @@ def encode_data(data) -> GLib.Bytes | None: return data_glib_bytes @staticmethod - def read_data(data: bytes) -> dict: - """Get JSON data from bytes""" - return json.loads(data) if data else {} + def create_message(method: str, url: str, data={}, headers: dict = {}, form: bool = False) -> Soup.Message: + """ + Create a libsoup's message. - @staticmethod - def read_response(session: Session, result: Gio.AsyncResult) -> dict: - """Get JSON data from session result""" - response = session.get_response(session, result) - return SoupProvider.read_data(response) + Encodes data and adds it to the message as the request body. + If form is true, data is encoded as application/x-www-form-urlencoded. - @staticmethod - def create_message(method: str, url: str, data={}, headers: dict = {}, form: bool = False) -> Soup.Message: - """Helper for creating libsoup's message""" + Args: + method: HTTP method of the message + url: Url of the message + data: Request body or form data + headers: HTTP headers of the message + form: If the data should be encoded as a form + """ if form and data: form_data = Soup.form_encode_hash(data) @@ -60,29 +67,78 @@ def create_message(method: str, url: str, data={}, headers: dict = {}, form: boo @staticmethod def send_and_read(message: Soup.Message, callback: Callable[[Session, Gio.AsyncResult], None]): - """Helper method for libsoup's send_and_read_async - Useful when priority and cancellable is not needed""" + """ + Helper method for libsoup's send_and_read_async. + + Useful when priority and cancellable is not needed. + + Args: + message: Message to send + callback: Callback called from send_and_read_async to finish request + """ Session.get().send_and_read_async(message, 0, None, callback) + @staticmethod + def read_data(data: bytes) -> dict: + """ + Get JSON data from bytes. + + Args: + data: Bytes to read + """ + return json.loads(data) if data else {} + + @staticmethod + def read_response(session: Session, result: Gio.AsyncResult) -> dict: + """ + Get JSON data from session result. + + Finishes request from send_and_read_async and gets body dict. + + Args: + session: Session where the request wa sent + result: Result of send_and_read_async callback + """ + response = session.get_response(session, result) + return SoupProvider.read_data(response) + @staticmethod def check_known_errors(data: dict) -> None | ProviderError: - """Checks data for possible response errors and return a found error if any - This should be implemented by subclases""" + """ + Checks data for possible response errors and return a found error if any. + + This should be implemented by subclases. + + Args: + data: Response body data + """ return None @staticmethod def process_response( session: Session, result: Gio.AsyncResult, - on_continue: Callable[[dict], None], + on_continue: Callable[[dict | bytes], None], on_fail: Callable[[ProviderError], None], check_common: bool = True, - json: bool = True + json: bool = True, ): - """Helper method for the most common workflow for processing soup responses + """ + Helper method for the most common workflow for processing soup responses. Checks for soup errors, then checks for common errors on data and calls on_fail if any, otherwise calls on_continue where the provider will finish the process. + + If json is false check_common is ignored and the data isn't processed as JSON and bites are passed to + on_continue. + + Args: + session: Session where the request wa sent + result: Result of send_and_read_async callback + on_continue: Called after data was got successfully + on_fail: Called after any error on request or in check_known_errors + check_common: If response data should be checked for errors using check_known_errors + json: If data should be processed as JSON using read_response """ try: @@ -95,7 +151,7 @@ def process_response( on_fail(error) return else: - data = Session.get_response(session, result) + data = Session.get_response(session, result) on_continue(data) @@ -106,14 +162,22 @@ def process_response( @staticmethod def send_and_read_and_process_response( message: Soup.Message, - on_continue: Callable[[dict], None], + on_continue: Callable[[dict | bytes], None], on_fail: Callable[[ProviderError], None], check_common: bool = True, - json: bool = True + json: bool = True, ): - """Helper packaging send_and_read and process_response + """ + Helper packaging send_and_read and process_response. - Avoids implementors having to deal with many callbacks.""" + Avoids providers having to deal with many callbacks. + + message: Message to send + on_continue: Called after data was got successfully + on_fail: Called after any error on request or in check_known_errors + check_common: If response data should be checked for errors using check_known_errors + json: If data should be processed as JSON using read_response + """ def on_response(session: Session, result: Gio.AsyncResult): SoupProvider.process_response(session, result, on_continue, on_fail, check_common, json) From b300fcee5cc0bd349ffb3c544341bb86190e7cd9 Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Fri, 17 Nov 2023 17:46:27 -0500 Subject: [PATCH 10/15] window: Reset UI state after translation failed --- dialect/window.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dialect/window.py b/dialect/window.py index 44bf2f52..7c4e452a 100644 --- a/dialect/window.py +++ b/dialect/window.py @@ -1092,6 +1092,9 @@ def on_translation_success(self, translation: Translation): self.translation_finish() def on_translation_fail(self, error: ProviderError): + if not self.next_trans: + self.translation_finish() + self.trans_warning.props.visible = True self.lookup_action('copy').props.enabled = False self.lookup_action('listen-src').props.enabled = False From 9ef4f4a947a329dac07ca021bfbc45da17f9c3c1 Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Sun, 19 Nov 2023 13:52:07 -0500 Subject: [PATCH 11/15] window: Make history use Translation objects --- dialect/providers/base.py | 22 +++++++-------- dialect/providers/modules/bing.py | 3 ++- dialect/providers/modules/google.py | 1 + dialect/providers/modules/libretrans.py | 2 +- dialect/providers/modules/lingva.py | 6 ++++- dialect/providers/modules/yandex.py | 2 +- dialect/window.py | 36 +++++++++---------------- 7 files changed, 32 insertions(+), 40 deletions(-) diff --git a/dialect/providers/base.py b/dialect/providers/base.py index 018903ad..16a9a07c 100644 --- a/dialect/providers/base.py +++ b/dialect/providers/base.py @@ -4,8 +4,9 @@ import io import urllib.parse +from dataclasses import dataclass from enum import Enum, Flag, auto -from typing import Callable +from typing import Callable, Optional from gi.repository import Gio @@ -63,18 +64,13 @@ def __init__(self, code: ProviderErrorCode, message: str = '') -> None: self.message = message # More detailed error info if needed +@dataclass class Translation: - def __init__( - self, - text: str, - detected: None | str = None, - mistakes: tuple[None | str, None | str] = (None, None), - pronunciation: tuple[None | str, None | str] = (None, None), - ): - self.text = text - self.detected = detected - self.mistakes = mistakes - self.pronunciation = pronunciation + text: str + original: tuple[str, str, str] + detected: Optional[str] = None + mistakes: tuple[Optional[str], Optional[str]] = (None, None) # + pronunciation: tuple[Optional[str], Optional[str]] = (None, None) class BaseProvider: @@ -108,7 +104,7 @@ def __init__(self): self.chars_limit = -1 """ Translation char limit """ - self.history = [] + self.history: list[Translation] = [] """ Here we save the translation history """ # GSettings diff --git a/dialect/providers/modules/bing.py b/dialect/providers/modules/bing.py index ef4f875c..5a58ae94 100644 --- a/dialect/providers/modules/bing.py +++ b/dialect/providers/modules/bing.py @@ -119,7 +119,8 @@ def on_response(data): translation = Translation( data['translations'][0]['text'], - detected, + (text, src, dest), + detected=detected, pronunciation=(None, pronunciation) ) on_done(translation) diff --git a/dialect/providers/modules/google.py b/dialect/providers/modules/google.py index bac41fb5..2930fcc1 100644 --- a/dialect/providers/modules/google.py +++ b/dialect/providers/modules/google.py @@ -512,6 +512,7 @@ def on_response(data): result = Translation( translated, + (text, src, dest), src, (mistake, self._strip_html_tags(mistake)), (origin_pronunciation, pronunciation), diff --git a/dialect/providers/modules/libretrans.py b/dialect/providers/modules/libretrans.py index c74773b3..c639d503 100644 --- a/dialect/providers/modules/libretrans.py +++ b/dialect/providers/modules/libretrans.py @@ -148,7 +148,7 @@ def on_response(data): def translate(self, text, src, dest, on_done, on_fail): def on_response(data): detected = data.get('detectedLanguage', {}).get('language', None) - translation = Translation(data['translatedText'], detected) + translation = Translation(data['translatedText'], (text, src, dest), detected) on_done(translation) # Request body diff --git a/dialect/providers/modules/lingva.py b/dialect/providers/modules/lingva.py index 8608c9cf..8b99dd5c 100644 --- a/dialect/providers/modules/lingva.py +++ b/dialect/providers/modules/lingva.py @@ -95,7 +95,11 @@ def on_response(data): dest_pronunciation = data['info']['pronunciation'].get('translation', None) translation = Translation( - data['translation'], detected, (mistakes, mistakes), (src_pronunciation, dest_pronunciation) + data['translation'], + (text, src, dest), + detected, + (mistakes, mistakes), + (src_pronunciation, dest_pronunciation), ) on_done(translation) diff --git a/dialect/providers/modules/yandex.py b/dialect/providers/modules/yandex.py index 38a28af8..03c4de7a 100644 --- a/dialect/providers/modules/yandex.py +++ b/dialect/providers/modules/yandex.py @@ -157,7 +157,7 @@ def on_response(data): detected = data['lang'].split('-')[0] if 'text' in data: - translation = Translation(data['text'][0], detected) + translation = Translation(data['text'][0], (text, src, dest), detected) on_done(translation) else: diff --git a/dialect/window.py b/dialect/window.py index 7c4e452a..a91f8c19 100644 --- a/dialect/window.py +++ b/dialect/window.py @@ -664,18 +664,14 @@ def ui_forward(self, _action, _param): self.current_history -= 1 self.history_update() - def add_history_entry(self, src_text, src_language, dest_language, dest_text): + def add_history_entry(self, translation: Translation): """Add a history entry to the history list.""" - new_history_trans = { - 'Languages': [src_language, dest_language], - 'Text': [src_text, dest_text] - } if self.current_history > 0: del self.provider['trans'].history[: self.current_history] self.current_history = 0 if len(self.provider['trans'].history) == TRANS_NUMBER: self.provider['trans'].history.pop() - self.provider['trans'].history.insert(0, new_history_trans) + self.provider['trans'].history.insert(0, translation) GLib.idle_add(self.reset_return_forward_btns) def switch_all(self, src_language, dest_language, src_text, dest_text): @@ -683,7 +679,7 @@ def switch_all(self, src_language, dest_language, src_text, dest_text): self.dest_lang_selector.selected = src_language self.src_buffer.props.text = dest_text self.dest_buffer.props.text = src_text - self.add_history_entry(src_language, dest_language, src_text, dest_text) + self.add_history_entry(Translation(src_text, (dest_text, src_language, dest_language))) # Re-enable widgets self.langs_button_box.props.sensitive = True @@ -965,11 +961,11 @@ def reset_return_forward_btns(self): # Retrieve translation history def history_update(self): self.reset_return_forward_btns() - lang_hist = self.provider['trans'].history[self.current_history] - self.src_lang_selector.selected = lang_hist['Languages'][0] - self.dest_lang_selector.selected = lang_hist['Languages'][1] - self.src_buffer.props.text = lang_hist['Text'][0] - self.dest_buffer.props.text = lang_hist['Text'][1] + translation = self.provider['trans'].history[self.current_history] + self.src_lang_selector.selected = translation.original[1] + self.dest_lang_selector.selected = translation.original[2] + self.src_buffer.props.text = translation.original[0] + self.dest_buffer.props.text = translation.text def appeared_before(self): src_language = self.src_lang_selector.selected @@ -981,9 +977,9 @@ def appeared_before(self): ) if ( len(self.provider['trans'].history) >= self.current_history + 1 - and (self.provider['trans'].history[self.current_history]['Languages'][0] == src_language or 'auto') - and self.provider['trans'].history[self.current_history]['Languages'][1] == dest_language - and self.provider['trans'].history[self.current_history]['Text'][0] == src_text + and (self.provider['trans'].history[self.current_history].original[1] == src_language or 'auto') + and self.provider['trans'].history[self.current_history].original[2] == dest_language + and self.provider['trans'].history[self.current_history].original[0] == src_text ): return True return False @@ -1054,14 +1050,8 @@ def on_translation_success(self, translation: Translation): self.trans_src_pron = translation.pronunciation[0] self.trans_dest_pron = translation.pronunciation[1] - # FIXME: Make history work again - # Finally, everything is saved in history - """self.add_history_entry( - original[0], - original[1], - original[2], - dest_text - )""" + # Finally, translation is saved in history + self.add_history_entry(translation) # Mistakes if ProviderFeature.MISTAKES in self.provider['trans'].features and not self.trans_mistakes == (None, None): From 1119bac5f5473ee56df0bcd7c8d3894ed4a6a0e4 Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Sun, 19 Nov 2023 13:57:56 -0500 Subject: [PATCH 12/15] window: Set translator loaded to false on fail --- dialect/window.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dialect/window.py b/dialect/window.py index a91f8c19..cffb01eb 100644 --- a/dialect/window.py +++ b/dialect/window.py @@ -296,6 +296,7 @@ def on_done(): self.check_apikey() def on_fail(error: ProviderError): + self.translator_loading = False self.loading_failed(error) provider = Settings.get().active_translator From b18f2804ad46ea20fa100a7ab18f9152c7e9878b Mon Sep 17 00:00:00 2001 From: Rafael Mardojai CM Date: Mon, 20 Nov 2023 11:18:19 -0500 Subject: [PATCH 13/15] providers: Add langs comparison method This method will probaly come handy in scenarios like DeepL where "en" lang can't be translated to "en-US" nor "en-GB", so extra logic for comparison can be added by the provider. --- dialect/providers/base.py | 15 +++++++++++++++ dialect/window.py | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dialect/providers/base.py b/dialect/providers/base.py index 16a9a07c..87a390dd 100644 --- a/dialect/providers/base.py +++ b/dialect/providers/base.py @@ -354,6 +354,21 @@ def normalize_lang_code(self, code: str): return code + def cmp_langs(self, a: str, b: str) -> bool: + """ + Compare two language codes. + + It assumes that the codes have been normalized by `normalize_lang_code`. + + This method exists so providers can add additional comparison logic. + + Args: + a: First lang to compare + b: Second lang to compare + """ + + return a == b + def add_lang(self, original_code: str, name: str = None, trans: bool = True, tts: bool = False): """ Add lang supported by provider after normalization. diff --git a/dialect/window.py b/dialect/window.py index cffb01eb..f3f66d8e 100644 --- a/dialect/window.py +++ b/dialect/window.py @@ -580,7 +580,7 @@ def _on_src_lang_changed(self, _obj, _param): True ) - if code == dest_code: + if self.provider['trans'].cmp_langs(code, dest_code): if len(self.dest_langs) >= 2: code = self.dest_langs[1] if code == self.src_langs[0] else dest_code if self.src_langs: @@ -626,7 +626,7 @@ def _on_dest_lang_changed(self, _obj, _param): True ) - if code == src_code: + if self.provider['trans'].cmp_langs(code, src_code): self.src_lang_selector.selected = self.dest_langs[0] # Disable or enable listen function. From dec91975ecb1e9a76fb5f277d50eb94f45db0513 Mon Sep 17 00:00:00 2001 From: Mufeed Ali Date: Mon, 11 Dec 2023 21:43:01 +0530 Subject: [PATCH 14/15] libretrans: Fix translation after providers change --- dialect/providers/modules/libretrans.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dialect/providers/modules/libretrans.py b/dialect/providers/modules/libretrans.py index c639d503..7972485f 100644 --- a/dialect/providers/modules/libretrans.py +++ b/dialect/providers/modules/libretrans.py @@ -68,7 +68,7 @@ def suggest_url(self): @property def translate_url(self): - self.format_url(self.instance_url, '/translate') + return self.format_url(self.instance_url, '/translate') def init_trans(self, on_done, on_fail): def check_finished(): From 778954348c3dc3dbeae0d4c2e91730bf47e42a28 Mon Sep 17 00:00:00 2001 From: Mufeed Ali Date: Mon, 11 Dec 2023 21:57:21 +0530 Subject: [PATCH 15/15] lingva: Fix handling of missing output parameters --- dialect/providers/modules/lingva.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dialect/providers/modules/lingva.py b/dialect/providers/modules/lingva.py index 8b99dd5c..9bd98e43 100644 --- a/dialect/providers/modules/lingva.py +++ b/dialect/providers/modules/lingva.py @@ -89,10 +89,10 @@ def init_tts(self, on_done, on_fail): def translate(self, text, src, dest, on_done, on_fail): def on_response(data): try: - detected = data['info'].get('detectedSource', None) - mistakes = data['info'].get('typo', None) - src_pronunciation = data['info']['pronunciation'].get('query', None) - dest_pronunciation = data['info']['pronunciation'].get('translation', None) + detected = data.get('info', {}).get('detectedSource', None) + mistakes = data.get('info', {}).get('typo', None) + src_pronunciation = data.get('info', {}).get('pronunciation', {}).get('query', None) + dest_pronunciation = data.get('info', {}).get('pronunciation', {}).get('translation', None) translation = Translation( data['translation'],