From c74a95f0e957b912af152c8bbb156517df76bca0 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 26 Apr 2024 00:13:56 +0530 Subject: [PATCH 1/8] chore: format --- .../docker_wrapper/DockerCompose.py | 66 +++++-------------- 1 file changed, 16 insertions(+), 50 deletions(-) diff --git a/frappe_manager/docker_wrapper/DockerCompose.py b/frappe_manager/docker_wrapper/DockerCompose.py index 48d77f69..601dd1b3 100644 --- a/frappe_manager/docker_wrapper/DockerCompose.py +++ b/frappe_manager/docker_wrapper/DockerCompose.py @@ -1,6 +1,6 @@ from subprocess import Popen, run, TimeoutExpired, CalledProcessError from pathlib import Path -from typing import Iterable, Union, Literal, Optional, Tuple +from typing import Iterable, List, Union, Literal, Optional, Tuple import shlex @@ -10,6 +10,7 @@ run_command_with_exit_code, ) + # Docker Compose version 2.18.1 class DockerComposeWrapper: """ @@ -17,10 +18,11 @@ class DockerComposeWrapper: Only this args have are different use case. Args: - stream (bool, optional): A boolean flag indicating whether to stream the output of the command as it runs. - If set to True, the output will be displayed in real-time. If set to False, the output will be + stream (bool, optional): A boolean flag indicating whether to stream the output of the command as it runs. + If set to True, the output will be displayed in real-time. If set to False, the output will be displayed after the command completes. Defaults to False. """ + def __init__(self, path: Path, timeout: int = 100): # requires valid path directory # directory where docker-compose resides @@ -46,10 +48,9 @@ def up( pull: Literal["missing", "never", "always"] = "missing", stream: bool = False, ) -> Union[Iterable[Tuple[str, bytes]], SubprocessOutput]: - parameters: dict = locals() - remove_parameters = ["services","stream"] + remove_parameters = ["services", "stream"] up_cmd: list = ["up"] up_cmd += services @@ -59,10 +60,7 @@ def up( # subprocess_env = dict(os.environ) # subprocess_env.update(env) - iterator = run_command_with_exit_code( - self.docker_compose_cmd + up_cmd, - stream=stream - ) + iterator = run_command_with_exit_code(self.docker_compose_cmd + up_cmd, stream=stream) return iterator def down( @@ -74,7 +72,6 @@ def down( dry_run: bool = False, stream: bool = False, ) -> Union[Iterable[Tuple[str, bytes]], SubprocessOutput]: - parameters: dict = locals() parameters["timeout"] = str(timeout) @@ -89,10 +86,7 @@ def down( down_cmd += parameters_to_options(parameters, exclude=remove_parameters) - iterator = run_command_with_exit_code( - self.docker_compose_cmd + down_cmd, - stream=stream - ) + iterator = run_command_with_exit_code(self.docker_compose_cmd + down_cmd, stream=stream) return iterator def start( @@ -101,7 +95,6 @@ def start( dry_run: bool = False, stream: bool = False, ) -> Union[Iterable[Tuple[str, bytes]], SubprocessOutput]: - parameters: dict = locals() start_cmd: list[str] = ["start"] @@ -114,10 +107,7 @@ def start( if type(services) == list: start_cmd += services - iterator = run_command_with_exit_code( - self.docker_compose_cmd + start_cmd, - stream=stream - ) + iterator = run_command_with_exit_code(self.docker_compose_cmd + start_cmd, stream=stream) return iterator def restart( @@ -128,7 +118,6 @@ def restart( no_deps: bool = False, stream: bool = False, ) -> Union[Iterable[Tuple[str, bytes]], SubprocessOutput]: - parameters: dict = locals() parameters["timeout"] = str(timeout) @@ -143,10 +132,7 @@ def restart( if type(services) == list: restart_cmd += services - iterator = run_command_with_exit_code( - self.docker_compose_cmd + restart_cmd, - stream=stream - ) + iterator = run_command_with_exit_code(self.docker_compose_cmd + restart_cmd, stream=stream) return iterator def stop( @@ -155,7 +141,6 @@ def stop( timeout: int = 100, stream: bool = False, ) -> Union[Iterable[Tuple[str, bytes]], SubprocessOutput]: - parameters: dict = locals() parameters["timeout"] = str(timeout) @@ -168,10 +153,7 @@ def stop( if type(services) == list: stop_cmd.extend(services) - iterator = run_command_with_exit_code( - self.docker_compose_cmd + stop_cmd, - stream=stream - ) + iterator = run_command_with_exit_code(self.docker_compose_cmd + stop_cmd, stream=stream) return iterator def exec( @@ -188,7 +170,6 @@ def exec( capture_output: bool = True, use_shlex_split: bool = True, ) -> Union[Iterable[Tuple[str, bytes]], SubprocessOutput]: - parameters: dict = locals() exec_cmd: list[str] = ["exec"] @@ -216,8 +197,7 @@ def exec( exec_cmd += [command] iterator = run_command_with_exit_code( - self.docker_compose_cmd + exec_cmd, - stream=stream, capture_output=capture_output + self.docker_compose_cmd + exec_cmd, stream=stream, capture_output=capture_output ) return iterator @@ -256,7 +236,6 @@ def ps( ] = None, stream: bool = False, ) -> Union[Iterable[Tuple[str, bytes]], SubprocessOutput]: - parameters: dict = locals() ps_cmd: list[str] = ["ps"] @@ -280,9 +259,7 @@ def ps( if service: ps_cmd += service - iterator = run_command_with_exit_code( - self.docker_compose_cmd + ps_cmd, stream=stream - ) + iterator = run_command_with_exit_code(self.docker_compose_cmd + ps_cmd, stream=stream) return iterator def logs( @@ -298,7 +275,6 @@ def logs( timestamps: bool = False, stream: bool = False, ) -> Union[Iterable[Tuple[str, bytes]], SubprocessOutput]: - parameters: dict = locals() logs_cmd: list[str] = ["logs"] @@ -344,7 +320,6 @@ def pull( include_deps: bool = False, stream: bool = False, ) -> Union[Iterable[Tuple[str, bytes]], SubprocessOutput]: - parameters: dict = locals() pull_cmd: list[str] = ["pull"] @@ -370,11 +345,10 @@ def run( use_shlex_split: bool = True, stream: bool = False, ) -> Union[Iterable[Tuple[str, bytes]], SubprocessOutput]: - parameters: dict = locals() run_cmd: list = ["run"] - remove_parameters = ["stream", "command", "service","use_shlex_split"] + remove_parameters = ["stream", "command", "service", "use_shlex_split"] run_cmd += parameters_to_options(parameters, exclude=remove_parameters) @@ -386,11 +360,7 @@ def run( else: run_cmd += [command] - - iterator = run_command_with_exit_code( - self.docker_compose_cmd + run_cmd, - stream=stream - ) + iterator = run_command_with_exit_code(self.docker_compose_cmd + run_cmd, stream=stream) return iterator def cp( @@ -403,7 +373,6 @@ def cp( follow_link: bool = False, stream: bool = False, ) -> Union[Iterable[Tuple[str, bytes]], SubprocessOutput]: - parameters: dict = locals() cp_cmd: list = ["cp"] @@ -427,8 +396,5 @@ def cp( cp_cmd += [f"{source}"] cp_cmd += [f"{destination}"] - iterator = run_command_with_exit_code( - self.docker_compose_cmd + cp_cmd, - stream=stream - ) + iterator = run_command_with_exit_code(self.docker_compose_cmd + cp_cmd, stream=stream) return iterator From e123cb19410cd90bb91613c64fb64625c937dcae Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 26 Apr 2024 00:14:53 +0530 Subject: [PATCH 2/8] Add fm_headers.conf to nginx-proxy --- frappe_manager/services_manager/services.py | 9 +++++++++ frappe_manager/templates/fm_headers.conf.tmpl | 1 + 2 files changed, 10 insertions(+) create mode 100644 frappe_manager/templates/fm_headers.conf.tmpl diff --git a/frappe_manager/services_manager/services.py b/frappe_manager/services_manager/services.py index d1135bd2..c35f9919 100644 --- a/frappe_manager/services_manager/services.py +++ b/frappe_manager/services_manager/services.py @@ -1,6 +1,7 @@ import shutil import platform import os +from jinja2 import Template import typer from datetime import datetime from pathlib import Path @@ -22,6 +23,8 @@ from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.ssl_manager.nginxproxymanager import NginxProxyManager from frappe_manager.utils.helpers import ( + get_current_fm_version, + get_template_path, random_password_generate, check_and_display_port_status, get_unix_groups, @@ -83,6 +86,12 @@ def init(self): self.compose_project = ComposeProject(compose_file_manager=compose_file_manager) self.proxy_manager: NginxProxyManager = NginxProxyManager('global-nginx-proxy', self.compose_project) + self.fm_headers_path: Path = self.proxy_manager.dirs.confd.host / 'fm_headers.conf' + + template_path: Path = get_template_path('fm_headers.conf.tmpl') + template = Template(template_path.read_text()) + output = template.render(current_version=f'v{get_current_fm_version()}') + self.fm_headers_path.write_text(output) def set_typer_context(self, ctx: typer.Context): """ diff --git a/frappe_manager/templates/fm_headers.conf.tmpl b/frappe_manager/templates/fm_headers.conf.tmpl new file mode 100644 index 00000000..776c8e22 --- /dev/null +++ b/frappe_manager/templates/fm_headers.conf.tmpl @@ -0,0 +1 @@ + add_header X-Powered-By "Frappe-Manager {{ current_version }}"; From 779c8a7b0230e6f0c8c330346dd0e6a623b50af0 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 26 Apr 2024 00:15:22 +0530 Subject: [PATCH 3/8] fix: mailhog default mail server error --- frappe_manager/site_manager/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 5d0b61db..a7341e5f 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -144,7 +144,7 @@ def sync_bench_config_configuration(self): if not self.admin_tools.compose_project.compose_file_manager.compose_path.exists(): self.sync_admin_tools_compose() else: - self.admin_tools.enable() + self.admin_tools.enable(force_configure=True) richprint.print("Enabled Admin-tools.") else: From fd4c98cda188ca21d212a451d4a4601e21b0e81b Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 26 Apr 2024 00:51:43 +0530 Subject: [PATCH 4/8] recreate container when running --- frappe_manager/migration_manager/migrations/migrate_0_13_0.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py index 17561c4f..02e75a51 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py @@ -70,6 +70,9 @@ def migrate_services(self): self.services_manager.compose_project.compose_file_manager.set_version(self.version.version_string()) self.services_manager.compose_project.compose_file_manager.write_to_file() + if self.services_manager.compose_project.is_service_running('global-nginx-proxy'): + self.services_manager.compose_project.docker.compose.up(services=['global-nginx-proxy']) + def up(self): richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") self.logger.info("-" * 40) From 1a0ddf4e3e3c88dbaa99695babf37d452b758a64 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 26 Apr 2024 00:52:44 +0530 Subject: [PATCH 5/8] migration for fm header change patch v0.13.1 --- .../migrations/migrate_0_13_1.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 frappe_manager/migration_manager/migrations/migrate_0_13_1.py diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_1.py b/frappe_manager/migration_manager/migrations/migrate_0_13_1.py new file mode 100644 index 00000000..f816aa55 --- /dev/null +++ b/frappe_manager/migration_manager/migrations/migrate_0_13_1.py @@ -0,0 +1,54 @@ +from frappe_manager.migration_manager.migration_base import MigrationBase +from frappe_manager.migration_manager.migration_exections import MigrationExceptionInBench +from frappe_manager.migration_manager.migration_helpers import ( + MigrationServicesManager, +) +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.migration_manager.version import Version +from frappe_manager import CLI_DIR, CLI_SERVICES_DIRECTORY + + +class MigrationV0131(MigrationBase): + version = Version("0.13.1") + + def __init__(self): + super().init() + self.benches_dir = CLI_DIR / "sites" + self.services_path = CLI_SERVICES_DIRECTORY + + def up(self): + richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") + self.logger.info("-" * 40) + + self.services_manager: MigrationServicesManager = MigrationServicesManager() + + if not self.services_manager.compose_project.compose_file_manager.exists(): + raise MigrationExceptionInBench( + f"Services compose at {self.services_manager.compose_project.compose_file_manager} not found." + ) + + richprint.change_head("Adding fm header config to nginx-proxy") + # create file fmheaders.conf + fm_headers_conf_path = self.services_path / 'nginx-proxy' / 'confd' / 'fm_headers.conf' + add_header = f'add_header X-Powered-By "Frappe-Manager {self.version.version_string()}";' + + fm_headers_conf_path.write_text(add_header) + + if self.services_manager.compose_project.is_service_running('global-nginx-proxy'): + self.services_manager.compose_project.docker.compose.up(services=['global-nginx-proxy']) + + richprint.print("Added fm header config to nginx-proxy.") + + richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") + self.logger.info("-" * 40) + + def down(self): + # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") + richprint.print(f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") + self.logger.info("-" * 40) + + for backup in self.backup_manager.backups: + self.backup_manager.restore(backup, force=True) + + richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") + self.logger.info("-" * 40) From 50ca2f5c56ae57b8ffceb742fd9272e49dd09f8e Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 26 Apr 2024 00:53:33 +0530 Subject: [PATCH 6/8] fix: wait till admin tools started --- frappe_manager/site_manager/admin_tools.py | 29 ++++++++++++++++------ 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/frappe_manager/site_manager/admin_tools.py b/frappe_manager/site_manager/admin_tools.py index 1d1cfa9b..10544627 100644 --- a/frappe_manager/site_manager/admin_tools.py +++ b/frappe_manager/site_manager/admin_tools.py @@ -4,6 +4,7 @@ from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.compose_project.compose_project import ComposeProject from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.docker_wrapper.DockerException import DockerException from frappe_manager.site_manager.site_exceptions import AdminToolsFailedToStart, BenchException from frappe_manager.ssl_manager.nginxproxymanager import NginxProxyManager from frappe_manager.utils.helpers import get_container_name_prefix, get_current_fm_version, get_template_path @@ -126,17 +127,29 @@ def remove_mailhog_as_default_server(self): richprint.print("Removed Mailhog as default mail server.") return frappe_server_restart_required - def wait_till_started(self, interval=2, timeout=30): - for i in range(timeout): - if not self.compose_project.running: - time.sleep(interval) - continue - return - raise AdminToolsFailedToStart(self.bench_name) + def wait_till_services_started(self, interval=2, timeout=30): + admin_tools_services = ['mailhog:8025', 'adminer:8080'] + + for tool in admin_tools_services: + running = False + for i in range(timeout): + try: + check_command = f"wait-for-it -t {interval} {get_container_name_prefix(self.bench_name)}-{tool}" + self.nginx_proxy.compose_project.docker.compose.exec( + service='nginx', command=check_command, stream=False + ) + + running = True + break + except DockerException as e: + continue + + if not running: + raise AdminToolsFailedToStart(self.bench_name) def enable(self, force_recreate_container: bool = False, force_configure: bool = False) -> bool: self.compose_project.start_service(force_recreate=force_recreate_container) - self.wait_till_started() + self.wait_till_services_started() self.save_nginx_location_config() self.nginx_proxy.reload() frappe_server_restart_required = self.configure_mailhog_as_default_server(force=force_configure) From 5826ba50906792e943f4f354e7f44c092ac40296 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 26 Apr 2024 00:53:48 +0530 Subject: [PATCH 7/8] bump v0.13.1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2ba251d4..85179180 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "frappe-manager" -version = "0.13.0" +version = "0.13.1" license = "MIT" repository = "https://github.com/rtcamp/frappe-manager" description = "A CLI tool based on Docker Compose to easily manage Frappe based projects. As of now, only suitable for development in local machines running on Mac and Linux based OS." From 8594c2ce729792b35ab7491a42b53e1e6f5d80a4 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 26 Apr 2024 01:14:59 +0530 Subject: [PATCH 8/8] fix: force restart global-nginx-proxy in migration --- frappe_manager/migration_manager/migrations/migrate_0_13_1.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_1.py b/frappe_manager/migration_manager/migrations/migrate_0_13_1.py index f816aa55..6a147172 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_13_1.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_13_1.py @@ -35,7 +35,8 @@ def up(self): fm_headers_conf_path.write_text(add_header) if self.services_manager.compose_project.is_service_running('global-nginx-proxy'): - self.services_manager.compose_project.docker.compose.up(services=['global-nginx-proxy']) + self.services_manager.compose_project.docker.compose.up(services=['global-nginx-proxy'], stream=False) + self.services_manager.compose_project.docker.compose.restart(services=['global-nginx-proxy'], stream=False) richprint.print("Added fm header config to nginx-proxy.")