WIP: Working raw parsing and processing

mapadroid/data_handler/mitm_data/PlayerData.py

@@ -69,12 +69,10 @@ def update_latest(self, key: str, value: Union[List, Dict, bytes],
                       location: Optional[Location] = None) -> None:
         self._latest_data_holder.update(key, value, timestamp_received, timestamp_of_data_retrieval, location)
         if key == str(ProtoIdentifier.GMO.value) and isinstance(value, bytes):
-            gmo: pogoprotos.GetMapObjectsOutProto = pogoprotos.GetMapObjectsOutProto.ParseFromString(
-                value)
+            gmo: pogoprotos.GetMapObjectsOutProto = pogoprotos.GetMapObjectsOutProto()
+            gmo.ParseFromString(value)
             self.__parse_gmo_for_location(gmo, timestamp_received, location)
             self._injected = True
-        else:
-            logger.warning("update_latest not of GMO type")
 
     # Async since we may move it to DB for persistence, same for above methods like level and
     # pokestops visited (today/week/total/whatever)

mapadroid/data_handler/mitm_data/RedisMitmMapper.py

@@ -69,8 +69,8 @@ async def update_latest(self, worker: str, key: str, value: Union[List, Dict, by
             except Exception as e:
                 logger.exception(e)
         if key == str(ProtoIdentifier.GMO.value) and isinstance(value, bytes):
-            gmo: pogoprotos.GetMapObjectsOutProto = pogoprotos.GetMapObjectsOutProto.ParseFromString(
-                value)
+            gmo: pogoprotos.GetMapObjectsOutProto = pogoprotos.GetMapObjectsOutProto()
+            gmo.ParseFromString(value)
             await self.__parse_gmo_for_location(worker, gmo, timestamp_received_raw, location)
             await self.__cache.set(RedisMitmMapper.IS_INJECTED_KEY.format(worker), 1)
 

mapadroid/db/DbPogoProtoSubmitRaw.py

@@ -7,6 +7,7 @@
 import sqlalchemy
 from bitstring import BitArray
 from google.protobuf.internal.containers import RepeatedCompositeFieldContainer, RepeatedScalarFieldContainer
+from google.protobuf.json_format import MessageToJson
 from redis import Redis
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -714,8 +715,7 @@ async def quest(self, session: AsyncSession, quest_proto: pogoprotos.FortSearchO
         target: Optional[int] = goal.target
         condition: RepeatedCompositeFieldContainer[pogoprotos.QuestConditionProto] = goal.condition
 
-        # TODO: Json dumping protos...
-        json_condition: str = json.dumps(condition)
+        json_condition: str = MessageToJson(condition)
         task = await quest_gen.questtask(int(quest_type), json_condition, int(target), quest_template,
                                          quest_title_resource_id)
         quest: Optional[TrsQuest] = await TrsQuestHelper.get(session, fort_id, quest_layer)
@@ -982,7 +982,7 @@ async def _handle_route_cell(self, session: AsyncSession, s2_cell_id: int, route
                     route.route_id = route_id
 
                 # TODO: Make sure dumps works properly...
-                route.waypoints = json.dumps(route_data.waypoints)
+                route.waypoints = MessageToJson(route_data.waypoints)
                 route.type = route_data.type
                 route.path_type = route_data.path_type
                 route.name = route_data.name
@@ -997,13 +997,11 @@ async def _handle_route_cell(self, session: AsyncSession, s2_cell_id: int, route
                 route.route_distance_meters = route_data.route_distance_meters
                 route.route_duration_seconds = route_data.route_duration_seconds
 
-                pins_raw: Optional[Dict] = route_data.pins
-                # TODO: Make sure dumps works properly...
-                route.pins = json.dumps(pins_raw)
+                pins_raw: RepeatedCompositeFieldContainer[pogoprotos.RoutePin] = route_data.pins
+                route.pins = MessageToJson(pins_raw)
 
-                tags_raw: Optional[Dict] = route_data.tags
-                # TODO: Make sure dumps works properly...
-                route.tags = json.dumps(tags_raw)
+                tags_raw: RepeatedScalarFieldContainer[str] = route_data.tags
+                route.tags = MessageToJson(tags_raw)
 
                 image_data: pogoprotos.RouteImageProto = route_data.image
                 route.image = image_data.image_url

mapadroid/mitm_receiver/data_processing/SerializedMitmDataProcessor.py

@@ -19,6 +19,7 @@
 from mapadroid.db.DbWrapper import DbWrapper
 from mapadroid.db.helper.SettingsDeviceHelper import SettingsDeviceHelper
 from mapadroid.db.model import SettingsDevice
+from mapadroid.mitm_receiver.protos.ProtoHelper import ProtoHelper
 from mapadroid.utils.DatetimeWrapper import DatetimeWrapper
 from mapadroid.utils.ProtoIdentifier import ProtoIdentifier
 from mapadroid.utils.gamemechanicutil import determine_current_quest_layer
@@ -101,8 +102,8 @@ async def _process_data_raw(self, received_timestamp: int, data: Dict, origin: s
             logger.debug("Processing proto 101 (FORT_SEARCH)")
             async with self.__db_wrapper as session, session:
                 try:
-                    fort_search: pogoprotos.FortSearchOutProto = pogoprotos.FortSearchOutProto.ParseFromString(
-                        data["payload"])
+                    fort_search: pogoprotos.FortSearchOutProto = pogoprotos.FortSearchOutProto()
+                    fort_search.ParseFromString(data["payload"])
                     # TODO: Check enum works with int comparison
                     if fort_search.result == 1:
                         async with session.begin_nested() as nested_transaction:
@@ -127,8 +128,8 @@ async def _process_data_raw(self, received_timestamp: int, data: Dict, origin: s
             logger.debug("Processing proto 104 (FORT_DETAILS)")
             async with self.__db_wrapper as session, session:
                 try:
-                    fort_details: pogoprotos.FortDetailsOutProto = pogoprotos.FortDetailsOutProto.ParseFromString(
-                        data["payload"])
+                    fort_details: pogoprotos.FortDetailsOutProto = pogoprotos.FortDetailsOutProto()
+                    fort_details.ParseFromString(data["payload"])
                     await self.__db_submit.stop_details(session, fort_details)
                     await session.commit()
                 except Exception as e:
@@ -148,8 +149,8 @@ async def _process_data_raw(self, received_timestamp: int, data: Dict, origin: s
             logger.debug("Done processing proto 1405 in {}ms", end_time)
         elif method_id == ProtoIdentifier.GYM_INFO.value:
             logger.debug("Processing proto 156 (GYM_GET_INFO)")
-            gym_info: pogoprotos.GymGetInfoOutProto = pogoprotos.GymGetInfoOutProto.ParseFromString(
-                data["payload"])
+            gym_info: pogoprotos.GymGetInfoOutProto = pogoprotos.GymGetInfoOutProto()
+            gym_info.ParseFromString(data["payload"])
             async with self.__db_wrapper as session, session:
                 try:
                     await self.__db_submit.gym_info(session, gym_info)
@@ -167,8 +168,8 @@ async def __process_lured_encounter(self, data: Dict, origin: str,
         playerlevel = await self.__mitm_mapper.get_level(origin)
         if MadGlobals.application_args.scan_lured_mons and (playerlevel >= 30):
             logger.debug("Processing lure encounter received at {}", processed_timestamp)
-            encounter_proto: pogoprotos.DiskEncounterOutProto = pogoprotos.DiskEncounterOutProto.ParseFromString(
-                data["payload"])
+            encounter_proto: pogoprotos.DiskEncounterOutProto = ProtoHelper.parse(ProtoIdentifier.DISK_ENCOUNTER,
+                                                                                  data["payload"])
             async with self.__db_wrapper as session, session:
                 lure_encounter: Optional[Tuple[int, datetime]] = await self.__db_submit \
                     .mon_lure_iv(session, received_timestamp, encounter_proto)
@@ -181,8 +182,7 @@ async def __process_lured_encounter(self, data: Dict, origin: str,
 
     async def __process_encounter(self, data: Dict, origin: str, received_date: datetime, received_timestamp: int,
                                   start_time_ms: int):
-        encounter_proto: pogoprotos.EncounterOutProto = pogoprotos.EncounterOutProto.ParseFromString(
-            data["payload"])
+        encounter_proto: pogoprotos.EncounterOutProto = ProtoHelper.parse(ProtoIdentifier.ENCOUNTER, data["payload"])
         # TODO: Cache result in SerializedMitmDataProcessor to not spam the MITMMapper too much in that regard
         playerlevel = await self.__mitm_mapper.get_level(origin)
         if playerlevel >= 30:
@@ -207,8 +207,7 @@ async def __process_gmo_raw(self, data: Dict, origin: str, received_date: dateti
                                 received_timestamp: int, start_time_ms: int):
         logger.debug("Processing GMO. Received at {}", received_date)
         # TODO: Offload conversion?
-        gmo: pogoprotos.GetMapObjectsOutProto = pogoprotos.GetMapObjectsOutProto.ParseFromString(
-            data["payload"])
+        gmo: pogoprotos.GetMapObjectsOutProto = ProtoHelper.parse(ProtoIdentifier.GMO, data["payload"])
         loop = asyncio.get_running_loop()
         weather_task = loop.create_task(self.__process_weather(gmo, received_timestamp))
         stops_task = loop.create_task(self.__process_stops(gmo))
@@ -370,8 +369,7 @@ async def __process_raids(self, gmo: pogoprotos.GetMapObjectsOutProto, timestamp
 
     async def __process_routes(self, data: bytes, received_timestamp: int) -> None:
         routes_time_start = self.get_time_ms()
-        routes: pogoprotos.GetRoutesOutProto = pogoprotos.GetRoutesOutProto.ParseFromString(
-            data)
+        routes: pogoprotos.GetRoutesOutProto = ProtoHelper.parse(ProtoIdentifier.GET_ROUTES, data)
         async with self.__db_wrapper as session, session:
             try:
                 await self.__db_submit.routes(session, routes, received_timestamp)
@@ -422,8 +420,7 @@ def get_time_ms():
         return int(time.time() * 1000)
 
     async def _handle_inventory_data(self, origin: str, data: bytes) -> None:
-        inventory_data: pogoprotos.GetHoloholoInventoryOutProto = pogoprotos.GetHoloholoInventoryOutProto.ParseFromString(
-            data)
+        inventory_data: pogoprotos.GetHoloholoInventoryOutProto = ProtoHelper.parse(ProtoIdentifier.INVENTORY, data)
         if not inventory_data.inventory_delta:
             logger.debug2('gen_player_stats cannot generate new stats')
             return

mapadroid/mitm_receiver/endpoints/ReceiveProtosEndpoint.py

@@ -99,15 +99,14 @@ async def __handle_proto_data_dict(self, origin: str, data: dict) -> None:
                                                     location=location_of_data)
         if proto_type == ProtoIdentifier.GMO.value:
             # TODO: Offload transformation
-            gmo: pogoprotos.GetMapObjectsOutProto = pogoprotos.GetMapObjectsOutProto.ParseFromString(
-                decoded_raw_proto)
+            gmo: pogoprotos.GetMapObjectsOutProto = ProtoHelper.parse(ProtoIdentifier.GMO, decoded_raw_proto)
             if not gmo.map_cell:
                 logger.debug("Ignoring apparently empty GMO")
                 return
         elif proto_type == ProtoIdentifier.FORT_SEARCH.value:
             logger.debug("Checking fort search proto type 101")
-            fort_search: pogoprotos.FortSearchOutProto = pogoprotos.FortSearchOutProto.ParseFromString(
-                decoded_raw_proto)
+            fort_search: pogoprotos.FortSearchOutProto = ProtoHelper.parse(ProtoIdentifier.FORT_SEARCH,
+                                                                           decoded_raw_proto)
             if fort_search.result == 2:
                 location_of_data: Location = Location(data.get("lat", 0.0), data.get("lng", 0.0))
                 # Fort search out of range, abort
@@ -118,14 +117,12 @@ async def __handle_proto_data_dict(self, origin: str, data: dict) -> None:
             await self._handle_fort_search_proto(origin, fort_search, location_of_data, timestamp)
         elif proto_type == ProtoIdentifier.ENCOUNTER.value:
             # TODO: Offload transformation
-            encounter: pogoprotos.EncounterOutProto = pogoprotos.EncounterOutProto.ParseFromString(
-                decoded_raw_proto)
+            encounter: pogoprotos.EncounterOutProto = ProtoHelper.parse(ProtoIdentifier.ENCOUNTER, decoded_raw_proto)
             if encounter.status != 1:
                 logger.warning("Encounter with status {} being ignored", encounter.status)
                 return
         elif proto_type == ProtoIdentifier.GET_ROUTES.value:
-            get_routes: pogoprotos.GetRoutesOutProto = pogoprotos.GetRoutesOutProto.ParseFromString(
-                decoded_raw_proto)
+            get_routes: pogoprotos.GetRoutesOutProto = ProtoHelper.parse(ProtoIdentifier.GET_ROUTES, decoded_raw_proto)
             if not get_routes.route_map_cell:
                 logger.info("No routes in payload to be processed")
                 return

mapadroid/mitm_receiver/protos/ProtoHelper.py

@@ -1,5 +1,11 @@
+import base64
 import pickle
-from typing import Any
+from typing import Any, Optional, Union, Type
+
+from google.protobuf.message import Message
+
+from mapadroid.utils.ProtoIdentifier import ProtoIdentifier
+import mapadroid.mitm_receiver.protos.Rpc_pb2 as pogoprotos
 
 
 class ProtoHelper:
@@ -20,4 +26,32 @@ def decode(encoded_val: str) -> bytes:
         """
         PD encodes the raw protos in HEX for JSON serialization
         """
-        return bytes.fromhex(encoded_val)
+        return base64.b64decode(encoded_val)
+
+    @staticmethod
+    def parse(method: ProtoIdentifier, value: Union[bytes, str]) -> Any:
+        if isinstance(value, str):
+            value = ProtoHelper.decode(value)
+        else:
+            # already in bytes format which we need
+            pass
+        message: Optional[Message] = None
+        if method == ProtoIdentifier.GMO:
+            message: pogoprotos.GetMapObjectsOutProto = pogoprotos.GetMapObjectsOutProto()
+        elif method == ProtoIdentifier.ENCOUNTER:
+            message: pogoprotos.EncounterOutProto = pogoprotos.EncounterOutProto()
+        elif method == ProtoIdentifier.GET_ROUTES:
+            message: pogoprotos.GetRoutesOutProto = pogoprotos.GetRoutesOutProto()
+        elif method == ProtoIdentifier.GYM_INFO:
+            message: pogoprotos.GymGetInfoOutProto = pogoprotos.GymGetInfoOutProto()
+        elif method == ProtoIdentifier.FORT_SEARCH:
+            message: pogoprotos.FortSearchOutProto = pogoprotos.FortSearchOutProto()
+        elif method == ProtoIdentifier.DISK_ENCOUNTER:
+            message: pogoprotos.DiskEncounterOutProto = pogoprotos.DiskEncounterOutProto()
+        elif method == ProtoIdentifier.INVENTORY:
+            message: pogoprotos.GetHoloholoInventoryOutProto = pogoprotos.GetHoloholoInventoryOutProto()
+        else:
+            raise ValueError(f"Method {method} could not be parsed.")
+
+        message.ParseFromString(value)
+        return message

mapadroid/websocket/communicator.py

@@ -272,7 +272,7 @@ async def get_external_ip(self) -> Optional[str]:
         try:
             # Regex ^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$ from
             # https://stackoverflow.com/questions/5284147/validating-ipv4-addresses-with-regexp
-            found = re.search(r'(((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\\b){4})', res)
+            found = re.search(r'(((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4})', res)
             if found:
                 ip_address_found = found.group(1)
         except Exception as e:

mapadroid/worker/strategy/plain/WorkerInitStrategy.py

@@ -4,6 +4,7 @@
 from mapadroid.data_handler.mitm_data.holder.latest_mitm_data.LatestMitmDataEntry import \
     LatestMitmDataEntry
 from mapadroid.db.model import SettingsAreaInitMitm
+from mapadroid.mitm_receiver.protos.ProtoHelper import ProtoHelper
 from mapadroid.utils.DatetimeWrapper import DatetimeWrapper
 from mapadroid.utils.ProtoIdentifier import ProtoIdentifier
 from mapadroid.utils.logging import LoggerEnums, get_logger
@@ -48,8 +49,7 @@ async def _check_for_data_content(self, latest: Optional[LatestMitmDataEntry],
         if not latest_proto_data:
             return ReceivedType.UNDEFINED, data_found
         elif proto_to_wait_for == ProtoIdentifier.GMO:
-            gmo: pogoprotos.GetMapObjectsOutProto = pogoprotos.GetMapObjectsOutProto.ParseFromString(
-                latest_proto_data)
+            gmo: pogoprotos.GetMapObjectsOutProto = ProtoHelper.parse(ProtoIdentifier.GMO, latest_proto_data)
             area_settings: Optional[SettingsAreaInitMitm] = await self._mapping_manager.routemanager_get_settings(
                 self._area_id)
             init_type: InitTypes = InitTypes(area_settings.init_type)
@@ -62,8 +62,7 @@ async def _check_for_data_content(self, latest: Optional[LatestMitmDataEntry],
             else:
                 logger.debug("Data looked for not in GMO")
         elif proto_to_wait_for == ProtoIdentifier.ENCOUNTER:
-            data_found: pogoprotos.EncounterOutProto = pogoprotos.EncounterOutProto.ParseFromString(
-                latest_proto_data)
+            data_found: pogoprotos.EncounterOutProto = ProtoHelper.parse(ProtoIdentifier.ENCOUNTER, latest_proto_data)
             type_of_data_found = ReceivedType.MON
 
         return type_of_data_found, data_found

mapadroid/worker/strategy/plain/WorkerMonIvStrategy.py

@@ -1,6 +1,7 @@
 from typing import Optional, Tuple, List, Union, Any, Dict
 
 from mapadroid.data_handler.mitm_data.holder.latest_mitm_data.LatestMitmDataEntry import LatestMitmDataEntry
+from mapadroid.mitm_receiver.protos.ProtoHelper import ProtoHelper
 from mapadroid.utils.DatetimeWrapper import DatetimeWrapper
 from mapadroid.utils.ProtoIdentifier import ProtoIdentifier
 from mapadroid.utils.logging import LoggerEnums, get_logger
@@ -35,16 +36,14 @@ async def _check_for_data_content(self, latest: Optional[LatestMitmDataEntry],
         if not latest_proto_data:
             return ReceivedType.UNDEFINED, data_found
         elif proto_to_wait_for == ProtoIdentifier.GMO:
-            gmo: pogoprotos.GetMapObjectsOutProto = pogoprotos.GetMapObjectsOutProto.ParseFromString(
-                latest_proto_data)
+            gmo: pogoprotos.GetMapObjectsOutProto = ProtoHelper.parse(ProtoIdentifier.GMO, latest_proto_data)
             if await self._gmo_contains_wild_mons_closeby(gmo):
                 data_found = gmo
                 type_of_data_found = ReceivedType.GMO
             else:
                 logger.debug("Data looked for not in GMO")
         elif proto_to_wait_for == ProtoIdentifier.ENCOUNTER:
-            data_found: pogoprotos.EncounterOutProto = pogoprotos.EncounterOutProto.ParseFromString(
-                latest_proto_data)
+            data_found: pogoprotos.EncounterOutProto = ProtoHelper.parse(ProtoIdentifier.ENCOUNTER, latest_proto_data)
             type_of_data_found = ReceivedType.MON
 
         return type_of_data_found, data_found

mapadroid/worker/strategy/plain/WorkerMonMitmStrategy.py

@@ -1,6 +1,7 @@
 from typing import Optional, Tuple, List, Union, Any, Dict
 
 from mapadroid.data_handler.mitm_data.holder.latest_mitm_data.LatestMitmDataEntry import LatestMitmDataEntry
+from mapadroid.mitm_receiver.protos.ProtoHelper import ProtoHelper
 from mapadroid.utils.DatetimeWrapper import DatetimeWrapper
 from mapadroid.utils.ProtoIdentifier import ProtoIdentifier
 from mapadroid.utils.logging import LoggerEnums, get_logger
@@ -36,17 +37,15 @@ async def _check_for_data_content(self, latest: Optional[LatestMitmDataEntry],
         if not latest_proto_data:
             return ReceivedType.UNDEFINED, data_found
         elif proto_to_wait_for == ProtoIdentifier.GMO:
-            gmo: pogoprotos.GetMapObjectsOutProto = pogoprotos.GetMapObjectsOutProto.ParseFromString(
-                latest_proto_data)
+            gmo: pogoprotos.GetMapObjectsOutProto = ProtoHelper.parse(ProtoIdentifier.GMO, latest_proto_data)
             if await self._gmo_contains_wild_mons_closeby(gmo):
                 data_found = latest_proto_data
                 type_of_data_found = ReceivedType.GMO
             else:
                 # TODO: If there is no spawnpoint with a valid timer, this results in timeouts during ordinary routes...
                 logger.debug("Data looked for not in GMO")
         elif proto_to_wait_for == ProtoIdentifier.ENCOUNTER:
-            data_found: pogoprotos.EncounterOutProto = pogoprotos.EncounterOutProto.ParseFromString(
-                latest_proto_data)
+            data_found: pogoprotos.EncounterOutProto = ProtoHelper.parse(ProtoIdentifier.ENCOUNTER, latest_proto_data)
             type_of_data_found = ReceivedType.MON
 
         return type_of_data_found, data_found