Merge pull request #894 from opengisch/speed_up_packaging

Speed up packaging

docker-qgis/entrypoint.py

@@ -11,7 +11,6 @@
 from libqfieldsync.offline_converter import ExportType, OfflineConverter
 from libqfieldsync.offliners import OfflinerType, PythonMiniOffliner, QgisCoreOffliner
 from libqfieldsync.project import ProjectConfiguration
-from libqfieldsync.utils.bad_layer_handler import set_bad_layer_handler
 from libqfieldsync.utils.file_utils import get_project_in_folder
 from qfc_worker.utils import (
     Step,
@@ -20,8 +19,9 @@
     Workflow,
     get_layers_data,
     layers_data_to_string,
+    open_qgis_project,
 )
-from qgis.core import QgsCoordinateTransform, QgsProject, QgsRectangle, QgsVectorLayer
+from qgis.core import QgsCoordinateTransform, QgsRectangle, QgsVectorLayer
 
 PGSERVICE_FILE_CONTENTS = os.environ.get("PGSERVICE_FILE_CONTENTS")
 
@@ -35,12 +35,7 @@ def _call_libqfieldsync_packager(
     """Call `libqfieldsync` to package a project for QField"""
     logger.info("Preparing QGIS project for packaging…")
 
-    project = QgsProject.instance()
-    if not project_filename.exists():
-        raise FileNotFoundError(project_filename)
-
-    if not project.read(str(project_filename)):
-        raise Exception(f"Unable to open file with QGIS: {project_filename}")
+    project = open_qgis_project(str(project_filename))
 
     layers = project.mapLayers()
     project_config = ProjectConfiguration(project)
@@ -127,7 +122,7 @@ def _call_libqfieldsync_packager(
     # Disable the basemap generation because it needs the processing
     # plugin to be installed
     offline_converter.project_configuration.create_base_map = False
-    offline_converter.convert()
+    offline_converter.convert(reload_original_project=False)
 
     logger.info("Packaging finished!")
 
@@ -141,12 +136,8 @@ def _call_libqfieldsync_packager(
 def _extract_layer_data(project_filename: Union[str, Path]) -> dict:
     logger.info("Extracting QGIS project layer data…")
 
-    project_filename = str(project_filename)
-    project = QgsProject.instance()
-
-    with set_bad_layer_handler(project):
-        project.read(project_filename)
-        layers_by_id: dict = get_layers_data(project)
+    project = open_qgis_project(str(project_filename))
+    layers_by_id: dict = get_layers_data(project)
 
     logger.info(
         f"QGIS project layer data\n{layers_data_to_string(layers_by_id)}",
@@ -155,6 +146,10 @@ def _extract_layer_data(project_filename: Union[str, Path]) -> dict:
     return layers_by_id
 
 
+def _open_project(project_filename: Union[str, Path]):
+    return open_qgis_project(str(project_filename), force_reload=True)
+
+
 def cmd_package_project(args: argparse.Namespace):
     workflow = Workflow(
         id="package_project",
@@ -328,7 +323,7 @@ def cmd_process_projectfile(args: argparse.Namespace):
                 arguments={
                     "project_filename": WorkDirPath("files", args.project_file),
                 },
-                method=qfc_worker.process_projectfile.load_project_file,
+                method=_open_project,
                 return_names=["project"],
             ),
             Step(

docker-qgis/qfc_worker/process_projectfile.py

@@ -44,18 +44,6 @@ def check_valid_project_file(project_filename: Path) -> None:
     logger.info("QGIS project file is valid!")
 
 
-def load_project_file(project_filename: Path) -> QgsProject:
-    logger.info("Open QGIS project file…")
-
-    project = QgsProject.instance()
-    if not project.read(str(project_filename)):
-        raise InvalidXmlFileException(error=project.error())
-
-    logger.info("QGIS project file opened!")
-
-    return project
-
-
 def extract_project_details(project: QgsProject) -> dict[str, str]:
     """Extract project details"""
     logger.info("Extract project details…")

docker-qgis/qfc_worker/utils.py

@@ -18,7 +18,7 @@
 from typing import IO, Any, Callable, NamedTuple, Optional
 
 from libqfieldsync.layer import LayerSource
-from libqfieldsync.utils.bad_layer_handler import bad_layer_handler
+from libqfieldsync.utils.bad_layer_handler import set_bad_layer_handler
 from qfieldcloud_sdk import sdk
 from qgis.core import (
     Qgis,
@@ -182,9 +182,6 @@ def exitQgis():
 
         logging.info("QGIS app started!")
 
-    # we set the `bad_layer_handler` and assume we always have only one single `QgsProject` instance within the job's life
-    QgsProject.instance().setBadLayerHandler(bad_layer_handler)
-
     return QGISAPP
 
 
@@ -206,6 +203,33 @@ def stop_app():
         del QGISAPP
 
 
+def open_qgis_project(project_filename: str, force_reload: bool = False) -> QgsProject:
+    logging.info(f'Loading QGIS project "{project_filename}"…')
+
+    if not Path(project_filename).exists():
+        raise FileNotFoundError(project_filename)
+
+    project = QgsProject.instance()
+
+    if project.fileName() == str(project_filename) and not force_reload:
+        logging.info(
+            f'Skip loading QGIS project "{project_filename}", it is already loaded'
+        )
+        return project
+
+    with set_bad_layer_handler(project):
+        if not project.read(str(project_filename)):
+            logging.error(f'Failed to load QGIS project "{project_filename}"!')
+
+            project.setFileName("")
+
+            raise Exception(f"Unable to open project with QGIS: {project_filename}")
+
+    logging.info("Project loaded.")
+
+    return project
+
+
 def download_project(
     project_id: str, destination: Path = None, skip_attachments: bool = True
 ) -> Path:

docker-qgis/requirements_libqfieldsync.txt

@@ -1 +1 @@
-libqfieldsync @ git+https://github.com/opengisch/libqfieldsync@e20ff5be177c3a221ba4641d8b73a26a35016f66
+libqfieldsync @ git+https://github.com/opengisch/libqfieldsync@fefa28fc0b63fa5e7646fcb5065a19f02378206f