Use limit for DELETE statement in DbCleanup.py as per SQLAlchemy docu…

…mentation Using with_dialect_options to call DELETE to limit mysql and mariadb accordingly
Map-A-Droid · Sep 21, 2023 · 5d29762 · 5d29762
1 parent c26bc50
commit 5d29762
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 23 deletions.
diff --git a/mapadroid/db/DbCleanup.py b/mapadroid/db/DbCleanup.py
@@ -21,7 +21,8 @@ def __init__(self, db_wrapper: DbWrapper):
 
     async def start(self):
         if not self.__cleanup_task:
-            logger.info("Starting DB cleanup routine with interval {} seconds", MadGlobals.application_args.cleanup_interval)
+            logger.info("Starting DB cleanup routine with interval {} seconds",
+                        MadGlobals.application_args.cleanup_interval)
             loop = asyncio.get_running_loop()
             self.__cleanup_task = loop.create_task(self._run_cleanup_routine())
 
@@ -32,21 +33,25 @@ async def stop(self):
 
     async def _run_cleanup_routine(self):
         while True:
-            async with self.__db_wrapper as session, session:
-                if MadGlobals.application_args.delete_mons_n_hours:
-                    logger.info("Cleaning up records of mons disappeared more than {} hours ago.",
-                                MadGlobals.application_args.delete_mons_n_hours)
-                    mon_limit: Optional[int] = None if MadGlobals.application_args.delete_mons_limit <= 0 \
-                        else MadGlobals.application_args.delete_mons_limit
-                    await PokemonHelper.delete_older_than_n_hours(session,
-                                                                  MadGlobals.application_args.delete_mons_n_hours,
-                                                                  mon_limit)
-                    await PokemonHelper.run_optimize(session)
-                if MadGlobals.application_args.delete_incidents_n_hours:
-                    logger.info("Cleaning up records of incidents disappeared more than {} hours ago.",
-                                MadGlobals.application_args.delete_incidents_n_hours)
-                    await PokestopIncidentHelper.delete_older_than_n_hours(
-                        session, MadGlobals.application_args.delete_incidents_n_hours)
-                    await PokestopIncidentHelper.run_optimize(session)
-                await session.commit()
+            try:
+                async with self.__db_wrapper as session, session:
+                    if MadGlobals.application_args.delete_mons_n_hours:
+                        logger.info("Cleaning up records of mons disappeared more than {} hours ago.",
+                                    MadGlobals.application_args.delete_mons_n_hours)
+                        mon_limit: Optional[int] = None if MadGlobals.application_args.delete_mons_limit <= 0 \
+                            else MadGlobals.application_args.delete_mons_limit
+                        await PokemonHelper.delete_older_than_n_hours(session,
+                                                                      MadGlobals.application_args.delete_mons_n_hours,
+                                                                      mon_limit)
+                        await PokemonHelper.run_optimize(session)
+                    if MadGlobals.application_args.delete_incidents_n_hours:
+                        logger.info("Cleaning up records of incidents disappeared more than {} hours ago.",
+                                    MadGlobals.application_args.delete_incidents_n_hours)
+                        await PokestopIncidentHelper.delete_older_than_n_hours(
+                            session, MadGlobals.application_args.delete_incidents_n_hours, mon_limit)
+                        await PokestopIncidentHelper.run_optimize(session)
+                    await session.commit()
+            except Exception as e:
+                logger.error("Failed cleaning up DB.")
+                logger.exception(e)
             await asyncio.sleep(MadGlobals.application_args.cleanup_interval)
diff --git a/mapadroid/db/helper/PokemonHelper.py b/mapadroid/db/helper/PokemonHelper.py
@@ -320,10 +320,7 @@ async def delete_older_than_n_hours(session: AsyncSession, hours: int, limit: Op
         where_condition = Pokemon.disappear_time < DatetimeWrapper.now() - datetime.timedelta(hours=hours)
         stmt = delete(Pokemon).where(where_condition)
         if limit:
-            encounter_ids_query = select(Pokemon.encounter_id).where(where_condition).limit(limit)
-            result_encounter_ids = await session.execute(encounter_ids_query)
-            encounter_ids = result_encounter_ids.scalars().all()
-            stmt = delete(Pokemon).where(Pokemon.encounter_id.in_(encounter_ids))
+            stmt = stmt.with_dialect_options(mysql_limit=limit, mariadb_limit=limit)
         await session.execute(stmt)
 
     @staticmethod

diff --git a/mapadroid/db/helper/PokestopIncidentHelper.py b/mapadroid/db/helper/PokestopIncidentHelper.py
@@ -21,9 +21,11 @@ async def get(session: AsyncSession, pokestop_id: str, incident_id: str) -> Opti
         return result.scalars().first()
 
     @staticmethod
-    async def delete_older_than_n_hours(session: AsyncSession, hours: int) -> None:
+    async def delete_older_than_n_hours(session: AsyncSession, hours: int, limit: Optional[int]) -> None:
         where_condition = PokestopIncident.incident_expiration < DatetimeWrapper.now() - datetime.timedelta(hours=hours)
         stmt = delete(PokestopIncident).where(where_condition)
+        if limit is not None:
+            stmt = stmt.with_dialect_options(mysql_limit=limit, mariadb_limit=limit)
         await session.execute(stmt)
 
     @staticmethod