Allow building AP_Periph

metadata_manager/__init__.py

@@ -70,3 +70,10 @@
         waf_build_command="AntennaTracker"
     )
 )
+
+vehicles_manager.add_vehicle(
+    Vehicle(
+        name="AP_Periph",
+        waf_build_command="AP_Periph"
+    )
+)

metadata_manager/ap_src_meta_fetcher.py

@@ -89,14 +89,14 @@ def __build_options_key(self, commit_id: str) -> str:
         return self.__build_options_key_prefix + f"{commit_id}"
 
     def __cache_boards_at_commit(self,
-                                 boards: list,
+                                 boards: tuple,
                                  commit_id: str,
                                  ttl_sec: int = 86400) -> None:
         """
-        Cache the given list of boards for a particular commit.
+        Cache the given tuple of boards for a particular commit.
 
         Parameters:
-            boards (list): The list of boards.
+            boards (tuple): The tuple of boards (both non-periph and periph).
             commit_id (str): The git sha for the commit.
             ttl_sec (int): Time-to-live (TTL) in seconds after which the
             cached list expires.
@@ -179,16 +179,18 @@ def __get_build_options_at_commit_from_cache(self,
         self.logger.debug(f"Got value {value} at key {key}")
         return dill.loads(value) if value else None
 
-    def __get_boards_at_commit_from_cache(self, commit_id: str) -> list:
+    def __get_boards_at_commit_from_cache(self, commit_id: str) -> tuple:
         """
-        Returns the list of boards for a given commit from cache if exists,
-        None otherwise.
+        Returns the tuple of boards (for non-periph and periph targets,
+        in order) for a given commit from cache if exists, None otherwise.
 
         Parameters:
             commit_id (str): The commit id to get boards list for.
 
         Returns:
-            list: A list of boards available at the specified commit.
+            tuple: A tuple of two lists in order:
+                - A list contains boards for NON-'ap_periph' targets.
+                - A list contains boards for the 'ap_periph' target.
 
         Raises:
             RuntimeError: If the method is called when caching is disabled.
@@ -203,19 +205,51 @@ def __get_boards_at_commit_from_cache(self, commit_id: str) -> list:
         )
         value = self.__redis_client.get(key)
         self.logger.debug(f"Got value {value} at key {key}")
-        return dill.loads(value) if value else None
+        boards = dill.loads(value) if value else None
+
+        if not boards:
+            return None
+
+        # Ensure the data retrieved from the cache is correct
+        # We should get a tuple containing two lists
+        try:
+            non_periph_boards, periph_boards = boards
+        except ValueError as e:
+            self.logger.debug(f"Boards from cache: '{boards}'")
+            self.logger.exception(e)
+            return None
+
+        return (
+            non_periph_boards,
+            periph_boards
+        )
+
+    def __exclude_boards_matching_patterns(self, boards: list, patterns: list):
+        ret = []
+        for b in boards:
+            excluded = False
+            for p in patterns:
+                if fnmatch.fnmatch(b.lower(), p.lower()):
+                    excluded = True
+                    break
+            if not excluded:
+                ret.append(b)
+        return ret
 
     def __get_boards_at_commit_from_repo(self, remote: str,
-                                         commit_ref: str) -> list:
+                                         commit_ref: str) -> tuple:
         """
-        Returns the list of boards for a given commit from the git repo.
+        Returns the tuple of boards (for both non-periph and periph targets,
+        in order) for a given commit from the git repo.
 
         Parameters:
             remote (str): The name of the remote repository.
             commit_ref (str): The commit reference to check out.
 
         Returns:
-            list: A list of boards available at the specified commit.
+            tuple: A tuple of two lists in order:
+                - A list contains boards for NON-'ap_periph' targets.
+                - A list contains boards for the 'ap_periph' target.
         """
         with self.repo.get_checkout_lock():
             self.repo.checkout_remote_commit_ref(
@@ -235,19 +269,29 @@ def __get_boards_at_commit_from_repo(self, remote: str,
                 )
             mod = importlib.util.module_from_spec(spec)
             spec.loader.exec_module(mod)
-            all_boards = mod.AUTOBUILD_BOARDS
-        exclude_patterns = ['fmuv*', 'SITL*']
-        boards = []
-        for b in all_boards:
-            excluded = False
-            for p in exclude_patterns:
-                if fnmatch.fnmatch(b.lower(), p.lower()):
-                    excluded = True
-                    break
-            if not excluded:
-                boards.append(b)
-        boards.sort()
-        return boards
+            non_periph_boards = mod.AUTOBUILD_BOARDS
+            periph_boards = mod.AP_PERIPH_BOARDS
+            self.logger.debug(f"non_periph_boards raw: {non_periph_boards}")
+            self.logger.debug(f"periph_boards raw: {periph_boards}")
+
+        non_periph_boards = self.__exclude_boards_matching_patterns(
+            boards=non_periph_boards,
+            patterns=['fmuv*', 'SITL*'],
+        )
+        self.logger.debug(f"non_periph_boards filtered: {non_periph_boards}")
+
+        non_periph_boards_sorted = sorted(non_periph_boards)
+        periph_boards_sorted = sorted(periph_boards)
+
+        self.logger.debug(
+            f"non_periph_boards sorted: {non_periph_boards_sorted}"
+        )
+        self.logger.debug(f"periph_boards sorted: {periph_boards_sorted}")
+
+        return (
+            non_periph_boards_sorted,
+            periph_boards_sorted,
+        )
 
     def __get_build_options_at_commit_from_repo(self, remote: str,
                                                 commit_ref: str) -> tuple:
@@ -284,11 +328,12 @@ def __get_build_options_at_commit_from_repo(self, remote: str,
             build_options = mod.BUILD_OPTIONS
         return build_options
 
-    def get_boards_at_commit(self, remote: str,
-                             commit_ref: str) -> list:
+    def __get_boards_at_commit(self, remote: str,
+                               commit_ref: str) -> tuple:
         """
-        Retrieves a list of boards available for building at a
-        specified commit and returns the list.
+        Retrieves lists of boards available for building at a
+        specified commit for both NON-'ap_periph' and ap_periph targets
+        and returns a tuple containing both lists.
         If caching is enabled, this would first look in the cache for
         the list. In case of a cache miss, it would retrive the list
         by checkout out the git repo and running `board_list.py` and
@@ -299,7 +344,9 @@ def get_boards_at_commit(self, remote: str,
             commit_ref (str): The commit reference to check out.
 
         Returns:
-            list: A list of boards available at the specified commit.
+            tuple: A tuple of two lists in order:
+                - A list contains boards for NON-'ap_periph' targets.
+                - A list contains boards for the 'ap_periph' target.
         """
         tstart = time.time()
         if not self.caching_enabled:
@@ -343,6 +390,30 @@ def get_boards_at_commit(self, remote: str,
         )
         return boards
 
+    def get_boards(self, remote: str, commit_ref: str,
+                   vehicle: str) -> list:
+        """
+        Returns a list of boards available for building at a
+        specified commit for given vehicle.
+
+        Parameters:
+            remote (str): The name of the remote repository.
+            commit_ref (str): The commit reference to check out.
+            vehicle (str): The vehicle to get the boards list for.
+
+        Returns:
+            list: A list of boards.
+        """
+        non_periph_boards, periph_boards = self.__get_boards_at_commit(
+            remote=remote,
+            commit_ref=commit_ref,
+        )
+
+        if vehicle == 'AP_Periph':
+            return periph_boards
+
+        return non_periph_boards
+
     def get_build_options_at_commit(self, remote: str,
                                     commit_ref: str) -> list:
         """

scripts/fetch_releases.py

@@ -4,9 +4,6 @@
 import re
 import requests
 
-IGNORE_VERSIONS_BEFORE = '4.3'
-
-
 def version_number_and_type(git_hash, ap_source_subdir):
     url = (
         "https://raw.githubusercontent.com/ArduPilot/ardupilot/"
@@ -70,7 +67,8 @@ def remove_duplicate_entries(releases):
 
 def construct_vehicle_versions_list(vehicle, ap_source_subdir,
                                     fw_server_vehicle_sdir,
-                                    tag_filter_exps, tags):
+                                    tag_filter_exps, tags,
+                                    ignore_versions_before=''):
     ret = []
     for tag_info in tags:
         tag = tag_info['ref'].replace('refs/tags/', '')
@@ -102,7 +100,7 @@ def construct_vehicle_versions_list(vehicle, ap_source_subdir,
                 print(e)
                 continue
 
-            if v_num < IGNORE_VERSIONS_BEFORE:
+            if v_num < ignore_versions_before:
                 print(f"{v_num} Version too old. Ignoring.")
                 continue
 
@@ -161,7 +159,8 @@ def run(base_dir, remote_name):
             "(ArduCopter-(beta-4.3|beta|stable))",
             "(Copter-(\d+\.\d+\.\d+))"  # noqa
         ],
-        tags
+        tags,
+        '4.3',
     ))
 
     vehicles.append(construct_vehicle_versions_list(
@@ -172,7 +171,8 @@ def run(base_dir, remote_name):
             "(ArduPlane-(beta-4.3|beta|stable))",
             "(Plane-(\d+\.\d+\.\d+))"  # noqa
         ],
-        tags
+        tags,
+        '4.3',
     ))
 
     vehicles.append(construct_vehicle_versions_list(
@@ -183,7 +183,8 @@ def run(base_dir, remote_name):
             "(APMrover2-(beta-4.3|beta|stable))",
             "(Rover-(\d+\.\d+\.\d+))"  # noqa
         ],
-        tags
+        tags,
+        '4.3',
     ))
 
     vehicles.append(construct_vehicle_versions_list(
@@ -194,7 +195,8 @@ def run(base_dir, remote_name):
             "(ArduSub-(beta-4.3|beta|stable))",
             "(Sub-(\d+\.\d+\.\d+))"  # noqa
         ],
-        tags
+        tags,
+        '4.3',
     ))
 
     vehicles.append(construct_vehicle_versions_list(
@@ -205,7 +207,8 @@ def run(base_dir, remote_name):
             "(AntennaTracker-(beta-4.3|beta|stable))",
             "(Tracker-(\d+\.\d+\.\d+))"  # noqa
         ],
-        tags
+        tags,
+        '4.3',
     ))
 
     vehicles.append(construct_vehicle_versions_list(
@@ -215,7 +218,8 @@ def run(base_dir, remote_name):
         [
             "(Blimp-(beta-4.3|beta|stable|\d+\.\d+\.\d+))"  # noqa
         ],
-        tags
+        tags,
+        '4.3',
     ))
 
     vehicles.append(construct_vehicle_versions_list(
@@ -225,7 +229,19 @@ def run(base_dir, remote_name):
         [
             "(ArduCopter-(beta-4.3|beta|stable)-heli)"
         ],
-        tags
+        tags,
+        '4.3',
+    ))
+
+    vehicles.append(construct_vehicle_versions_list(
+        "AP_Periph",
+        "Tools/AP_Periph",
+        "AP_Periph",
+        [
+            "(AP_Periph-(beta|stable))"
+        ],
+        tags,
+        '1.7.0',
     ))
 
     remotes_json = {

scripts/fetch_whitelisted_tags.py

@@ -48,7 +48,8 @@
     'Sub',
     'AntennaTracker',
     'Blimp',
-    'Heli'
+    'Heli',
+    'AP_Periph',
 ]
 
 

web/app.py

@@ -151,9 +151,10 @@ def generate():
             raise Exception("Commit reference invalid or not listed to be built for given vehicle for remote")
 
         board = request.form['board']
-        boards_at_commit = ap_src_metadata_fetcher.get_boards_at_commit(
+        boards_at_commit = ap_src_metadata_fetcher.get_boards(
             remote=remote_name,
-            commit_ref=commit_ref
+            commit_ref=commit_ref,
+            vehicle=vehicle,
         )
         if board not in boards_at_commit:
             raise Exception("bad board")
@@ -235,9 +236,10 @@ def boards_and_features(vehicle_name, remote_name, commit_reference):
     app.logger.info('Board list and build options requested for %s %s %s' % (vehicle_name, remote_name, commit_reference))
     # getting board list for the branch
     with repo.get_checkout_lock():
-        boards = ap_src_metadata_fetcher.get_boards_at_commit(
+        boards = ap_src_metadata_fetcher.get_boards(
             remote=remote_name,
-            commit_ref=commit_reference
+            commit_ref=commit_reference,
+            vehicle=vehicle_name,
         )
 
         options = ap_src_metadata_fetcher.get_build_options_at_commit(