From 665605d20bf67d27922814251227b25f9ab98e1f Mon Sep 17 00:00:00 2001 From: Mathieu Pellerin Date: Sat, 24 Feb 2024 11:21:59 +0700 Subject: [PATCH 1/5] Bump libqfieldsync to catch optimization --- docker-qgis/requirements_libqfieldsync.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-qgis/requirements_libqfieldsync.txt b/docker-qgis/requirements_libqfieldsync.txt index 67c1d9893..cbf7306ba 100644 --- a/docker-qgis/requirements_libqfieldsync.txt +++ b/docker-qgis/requirements_libqfieldsync.txt @@ -1 +1 @@ -libqfieldsync @ git+https://github.com/opengisch/libqfieldsync@e20ff5be177c3a221ba4641d8b73a26a35016f66 +libqfieldsync @ git+https://github.com/opengisch/libqfieldsync@3ffdb4b97eae1a648051f2477f7db314b85cc332 From 54dafeb5aacb1039a09f04920bec7d97cbe1971c Mon Sep 17 00:00:00 2001 From: Mathieu Pellerin Date: Sat, 24 Feb 2024 11:38:50 +0700 Subject: [PATCH 2/5] Speed up packaging job by avoiding **3** project loads --- docker-qgis/entrypoint.py | 51 ++++++++++++---------- docker-qgis/requirements_libqfieldsync.txt | 2 +- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/docker-qgis/entrypoint.py b/docker-qgis/entrypoint.py index 9bd2ef905..bea99c602 100755 --- a/docker-qgis/entrypoint.py +++ b/docker-qgis/entrypoint.py @@ -29,18 +29,13 @@ logger.setLevel(logging.INFO) -def _call_libqfieldsync_packager( - project_filename: Path, package_dir: Path, offliner_type: OfflinerType -) -> str: +def _call_libqfieldsync_packager(package_dir: Path, offliner_type: OfflinerType) -> str: """Call `libqfieldsync` to package a project for QField""" logger.info("Preparing QGIS project for packaging…") project = QgsProject.instance() - if not project_filename.exists(): - raise FileNotFoundError(project_filename) - - if not project.read(str(project_filename)): - raise Exception(f"Unable to open file with QGIS: {project_filename}") + if not project.fileName(): + raise Exception("Project is empty") layers = project.mapLayers() project_config = ProjectConfiguration(project) @@ -127,7 +122,7 @@ def _call_libqfieldsync_packager( # Disable the basemap generation because it needs the processing # plugin to be installed offline_converter.project_configuration.create_base_map = False - offline_converter.convert() + offline_converter.convert(reload_original_project=False) logger.info("Packaging finished!") @@ -138,15 +133,11 @@ def _call_libqfieldsync_packager( return packaged_project_filename -def _extract_layer_data(project_filename: Union[str, Path]) -> dict: +def _extract_layer_data() -> dict: logger.info("Extracting QGIS project layer data…") - project_filename = str(project_filename) project = QgsProject.instance() - - with set_bad_layer_handler(project): - project.read(project_filename) - layers_by_id: dict = get_layers_data(project) + layers_by_id: dict = get_layers_data(project) logger.info( f"QGIS project layer data\n{layers_data_to_string(layers_by_id)}", @@ -155,6 +146,19 @@ def _extract_layer_data(project_filename: Union[str, Path]) -> dict: return layers_by_id +def _open_project(project_filename: Union[str, Path]): + logger.info("Loading QGIS project…") + + project_filename = str(project_filename) + project = QgsProject.instance() + with set_bad_layer_handler(project): + if not project.read(project_filename): + project.setFileName("") + raise Exception(f"Unable to open project with QGIS: {project_filename}") + + logger.info("Project loaded.") + + def cmd_package_project(args: argparse.Namespace): workflow = Workflow( id="package_project", @@ -179,11 +183,16 @@ def cmd_package_project(args: argparse.Namespace): return_names=["tmp_project_dir"], ), Step( - id="qgis_layers_data", - name="QGIS Layers Data", + id="qgis_open_project", + name="Open Project", arguments={ "project_filename": WorkDirPath("files", args.project_file), }, + method=_open_project, + ), + Step( + id="qgis_layers_data", + name="Original Project Layers Data", method=_extract_layer_data, return_names=["layers_by_id"], outputs=["layers_by_id"], @@ -192,7 +201,6 @@ def cmd_package_project(args: argparse.Namespace): id="package_project", name="Package Project", arguments={ - "project_filename": WorkDirPath("files", args.project_file), "package_dir": WorkDirPath("export", mkdir=True), "offliner_type": args.offliner_type, }, @@ -201,12 +209,7 @@ def cmd_package_project(args: argparse.Namespace): ), Step( id="qfield_layer_data", - name="Packaged Layers Data", - arguments={ - "project_filename": StepOutput( - "package_project", "qfield_project_filename" - ), - }, + name="Packaged Project Layers Data", method=_extract_layer_data, return_names=["layers_by_id"], outputs=["layers_by_id"], diff --git a/docker-qgis/requirements_libqfieldsync.txt b/docker-qgis/requirements_libqfieldsync.txt index cbf7306ba..446254906 100644 --- a/docker-qgis/requirements_libqfieldsync.txt +++ b/docker-qgis/requirements_libqfieldsync.txt @@ -1 +1 @@ -libqfieldsync @ git+https://github.com/opengisch/libqfieldsync@3ffdb4b97eae1a648051f2477f7db314b85cc332 +libqfieldsync @ git+https://github.com/opengisch/libqfieldsync@fefa28fc0b63fa5e7646fcb5065a19f02378206f From 555eaab1ccbd3e39901ea80ab043d5f335d4f603 Mon Sep 17 00:00:00 2001 From: Mathieu Pellerin Date: Wed, 28 Feb 2024 08:55:06 +0700 Subject: [PATCH 3/5] Regain individual step independence while also retaining optimization --- docker-qgis/entrypoint.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/docker-qgis/entrypoint.py b/docker-qgis/entrypoint.py index bea99c602..9e2b22c77 100755 --- a/docker-qgis/entrypoint.py +++ b/docker-qgis/entrypoint.py @@ -29,13 +29,23 @@ logger.setLevel(logging.INFO) -def _call_libqfieldsync_packager(package_dir: Path, offliner_type: OfflinerType) -> str: +def _call_libqfieldsync_packager( + project_filename: Path, package_dir: Path, offliner_type: OfflinerType +) -> str: """Call `libqfieldsync` to package a project for QField""" logger.info("Preparing QGIS project for packaging…") + if not project_filename.exists(): + raise FileNotFoundError(project_filename) + project = QgsProject.instance() - if not project.fileName(): - raise Exception("Project is empty") + + project_filename_string = str(project_filename) + if project.fileName() != project_filename_string: + with set_bad_layer_handler(project): + if not project.read(project_filename_string): + project.setFileName("") + raise Exception(f"Unable to open file with QGIS: {project_filename}") layers = project.mapLayers() project_config = ProjectConfiguration(project) @@ -133,10 +143,17 @@ def _call_libqfieldsync_packager(package_dir: Path, offliner_type: OfflinerType) return packaged_project_filename -def _extract_layer_data() -> dict: +def _extract_layer_data(project_filename: Union[str, Path]) -> dict: logger.info("Extracting QGIS project layer data…") project = QgsProject.instance() + + if project.fileName() != str(project_filename): + with set_bad_layer_handler(project): + if not project.read(str(project_filename)): + project.setFileName("") + raise Exception(f"Unable to open file with QGIS: {project_filename}") + layers_by_id: dict = get_layers_data(project) logger.info( @@ -149,10 +166,10 @@ def _extract_layer_data() -> dict: def _open_project(project_filename: Union[str, Path]): logger.info("Loading QGIS project…") - project_filename = str(project_filename) project = QgsProject.instance() + with set_bad_layer_handler(project): - if not project.read(project_filename): + if not project.read(str(project_filename)): project.setFileName("") raise Exception(f"Unable to open project with QGIS: {project_filename}") @@ -193,6 +210,9 @@ def cmd_package_project(args: argparse.Namespace): Step( id="qgis_layers_data", name="Original Project Layers Data", + arguments={ + "project_filename": WorkDirPath("files", args.project_file), + }, method=_extract_layer_data, return_names=["layers_by_id"], outputs=["layers_by_id"], @@ -201,6 +221,7 @@ def cmd_package_project(args: argparse.Namespace): id="package_project", name="Package Project", arguments={ + "project_filename": WorkDirPath("files", args.project_file), "package_dir": WorkDirPath("export", mkdir=True), "offliner_type": args.offliner_type, }, @@ -210,6 +231,11 @@ def cmd_package_project(args: argparse.Namespace): Step( id="qfield_layer_data", name="Packaged Project Layers Data", + arguments={ + "project_filename": StepOutput( + "package_project", "qfield_project_filename" + ), + }, method=_extract_layer_data, return_names=["layers_by_id"], outputs=["layers_by_id"], From eaab36fcda0b04e13ee76d7c47eb3cd5fdc180f7 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Wed, 13 Mar 2024 10:21:17 +0200 Subject: [PATCH 4/5] Reduce the code duplication about project loading --- docker-qgis/entrypoint.py | 48 +++++---------------------------- docker-qgis/qfc_worker/utils.py | 32 +++++++++++++++++++--- 2 files changed, 35 insertions(+), 45 deletions(-) diff --git a/docker-qgis/entrypoint.py b/docker-qgis/entrypoint.py index 9e2b22c77..f80ecd791 100755 --- a/docker-qgis/entrypoint.py +++ b/docker-qgis/entrypoint.py @@ -11,7 +11,6 @@ from libqfieldsync.offline_converter import ExportType, OfflineConverter from libqfieldsync.offliners import OfflinerType, PythonMiniOffliner, QgisCoreOffliner from libqfieldsync.project import ProjectConfiguration -from libqfieldsync.utils.bad_layer_handler import set_bad_layer_handler from libqfieldsync.utils.file_utils import get_project_in_folder from qfc_worker.utils import ( Step, @@ -20,8 +19,9 @@ Workflow, get_layers_data, layers_data_to_string, + open_qgis_project, ) -from qgis.core import QgsCoordinateTransform, QgsProject, QgsRectangle, QgsVectorLayer +from qgis.core import QgsCoordinateTransform, QgsRectangle, QgsVectorLayer PGSERVICE_FILE_CONTENTS = os.environ.get("PGSERVICE_FILE_CONTENTS") @@ -35,17 +35,7 @@ def _call_libqfieldsync_packager( """Call `libqfieldsync` to package a project for QField""" logger.info("Preparing QGIS project for packaging…") - if not project_filename.exists(): - raise FileNotFoundError(project_filename) - - project = QgsProject.instance() - - project_filename_string = str(project_filename) - if project.fileName() != project_filename_string: - with set_bad_layer_handler(project): - if not project.read(project_filename_string): - project.setFileName("") - raise Exception(f"Unable to open file with QGIS: {project_filename}") + project = open_qgis_project(str(project_filename)) layers = project.mapLayers() project_config = ProjectConfiguration(project) @@ -146,14 +136,7 @@ def _call_libqfieldsync_packager( def _extract_layer_data(project_filename: Union[str, Path]) -> dict: logger.info("Extracting QGIS project layer data…") - project = QgsProject.instance() - - if project.fileName() != str(project_filename): - with set_bad_layer_handler(project): - if not project.read(str(project_filename)): - project.setFileName("") - raise Exception(f"Unable to open file with QGIS: {project_filename}") - + project = open_qgis_project(str(project_filename)) layers_by_id: dict = get_layers_data(project) logger.info( @@ -164,16 +147,7 @@ def _extract_layer_data(project_filename: Union[str, Path]) -> dict: def _open_project(project_filename: Union[str, Path]): - logger.info("Loading QGIS project…") - - project = QgsProject.instance() - - with set_bad_layer_handler(project): - if not project.read(str(project_filename)): - project.setFileName("") - raise Exception(f"Unable to open project with QGIS: {project_filename}") - - logger.info("Project loaded.") + open_qgis_project(str(project_filename), force_reload=True) def cmd_package_project(args: argparse.Namespace): @@ -199,17 +173,9 @@ def cmd_package_project(args: argparse.Namespace): method=qfc_worker.utils.download_project, return_names=["tmp_project_dir"], ), - Step( - id="qgis_open_project", - name="Open Project", - arguments={ - "project_filename": WorkDirPath("files", args.project_file), - }, - method=_open_project, - ), Step( id="qgis_layers_data", - name="Original Project Layers Data", + name="QGIS Layers Data", arguments={ "project_filename": WorkDirPath("files", args.project_file), }, @@ -230,7 +196,7 @@ def cmd_package_project(args: argparse.Namespace): ), Step( id="qfield_layer_data", - name="Packaged Project Layers Data", + name="Packaged Layers Data", arguments={ "project_filename": StepOutput( "package_project", "qfield_project_filename" diff --git a/docker-qgis/qfc_worker/utils.py b/docker-qgis/qfc_worker/utils.py index ff9a949ef..f948c0236 100644 --- a/docker-qgis/qfc_worker/utils.py +++ b/docker-qgis/qfc_worker/utils.py @@ -18,7 +18,7 @@ from typing import IO, Any, Callable, NamedTuple, Optional from libqfieldsync.layer import LayerSource -from libqfieldsync.utils.bad_layer_handler import bad_layer_handler +from libqfieldsync.utils.bad_layer_handler import set_bad_layer_handler from qfieldcloud_sdk import sdk from qgis.core import ( Qgis, @@ -182,9 +182,6 @@ def exitQgis(): logging.info("QGIS app started!") - # we set the `bad_layer_handler` and assume we always have only one single `QgsProject` instance within the job's life - QgsProject.instance().setBadLayerHandler(bad_layer_handler) - return QGISAPP @@ -206,6 +203,33 @@ def stop_app(): del QGISAPP +def open_qgis_project(project_filename: str, force_reload: bool = False) -> QgsProject: + logging.info(f'Loading QGIS project "{project_filename}"…') + + if not Path(project_filename).exists(): + raise FileNotFoundError(project_filename) + + project = QgsProject.instance() + + if project.fileName() == str(project_filename) and not force_reload: + logging.info( + f'Skip loading QGIS project "{project_filename}", it is already loaded' + ) + return project + + with set_bad_layer_handler(project): + if not project.read(str(project_filename)): + logging.error(f'Failed to load QGIS project "{project_filename}"!') + + project.setFileName("") + + raise Exception(f"Unable to open project with QGIS: {project_filename}") + + logging.info("Project loaded.") + + return project + + def download_project( project_id: str, destination: Path = None, skip_attachments: bool = True ) -> Path: From 4d9006efebab99e79d221af9fc41c4bb5e7a0df9 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Fri, 15 Mar 2024 21:30:19 +0200 Subject: [PATCH 5/5] Remove the `load_project_file` as a separate function, use the new `_open_project` --- docker-qgis/entrypoint.py | 4 ++-- docker-qgis/qfc_worker/process_projectfile.py | 12 ------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/docker-qgis/entrypoint.py b/docker-qgis/entrypoint.py index f80ecd791..8dd2e1e80 100755 --- a/docker-qgis/entrypoint.py +++ b/docker-qgis/entrypoint.py @@ -147,7 +147,7 @@ def _extract_layer_data(project_filename: Union[str, Path]) -> dict: def _open_project(project_filename: Union[str, Path]): - open_qgis_project(str(project_filename), force_reload=True) + return open_qgis_project(str(project_filename), force_reload=True) def cmd_package_project(args: argparse.Namespace): @@ -323,7 +323,7 @@ def cmd_process_projectfile(args: argparse.Namespace): arguments={ "project_filename": WorkDirPath("files", args.project_file), }, - method=qfc_worker.process_projectfile.load_project_file, + method=_open_project, return_names=["project"], ), Step( diff --git a/docker-qgis/qfc_worker/process_projectfile.py b/docker-qgis/qfc_worker/process_projectfile.py index 261bd5c3a..d7aa4b511 100644 --- a/docker-qgis/qfc_worker/process_projectfile.py +++ b/docker-qgis/qfc_worker/process_projectfile.py @@ -44,18 +44,6 @@ def check_valid_project_file(project_filename: Path) -> None: logger.info("QGIS project file is valid!") -def load_project_file(project_filename: Path) -> QgsProject: - logger.info("Open QGIS project file…") - - project = QgsProject.instance() - if not project.read(str(project_filename)): - raise InvalidXmlFileException(error=project.error()) - - logger.info("QGIS project file opened!") - - return project - - def extract_project_details(project: QgsProject) -> dict[str, str]: """Extract project details""" logger.info("Extract project details…")