Merge branch 'edge' into EXEC-655-store-commands-error-list-in-db

Makefile

@@ -152,6 +152,10 @@ push:
 	sleep 1
 	$(MAKE) -C $(UPDATE_SERVER_DIR) push
 
+.PHONY: push-folder
+PUSH_HELPER := abr-testing/abr_testing/tools/make_push.py
+push-folder:
+	$(OT_PYTHON) $(PUSH_HELPER)
 
 .PHONY: push-ot3
 push-ot3:

abr-testing/abr_testing/automation/jira_tool.py

@@ -107,6 +107,12 @@ def open_issue(self, issue_key: str) -> str:
         webbrowser.open(url)
         return url
 
+    def get_labels(self) -> List[str]:
+        """Get list of available labels."""
+        url = f"{self.url}/rest/api/3/label"
+        response = requests.request("GET", url, headers=self.headers, auth=self.auth)
+        return response.json()
+
     def create_ticket(
         self,
         summary: str,
@@ -118,18 +124,21 @@ def create_ticket(
         priority: str,
         components: list,
         affects_versions: str,
-        robot: str,
+        labels: list,
+        parent_name: str,
     ) -> Tuple[str, str]:
         """Create ticket."""
         # Check if software version is a field on JIRA, if not replaces with existing version
+        # TODO: automate parent linking
         data = {
             "fields": {
                 "project": {"id": "10273", "key": project_key},
                 "issuetype": {"name": issue_type},
                 "summary": summary,
                 "reporter": {"id": reporter_id},
                 "assignee": {"id": assignee_id},
-                "parent": {"key": robot},
+                # "parent": {"key": parent_name},
+                "labels": labels,
                 "priority": {"name": priority},
                 "components": [{"name": component} for component in components],
                 "description": {
@@ -194,6 +203,7 @@ def post_attachment_to_ticket(self, issue_id: str, attachment_path: str) -> None
 
     def get_project_issues(self, project_key: str) -> Dict[str, Any]:
         """Retrieve all issues for the given project key."""
+        # TODO: add field for ticket type.
         headers = {"Accept": "application/json"}
         query = {"jql": f"project={project_key}"}
         response = requests.request(
@@ -203,7 +213,6 @@ def get_project_issues(self, project_key: str) -> Dict[str, Any]:
             params=query,
             auth=self.auth,
         )
-        response.raise_for_status()
         return response.json()
 
     def get_project_versions(self, project_key: str) -> List[str]:

abr-testing/abr_testing/data_collection/abr_calibration_logs.py

@@ -286,6 +286,7 @@ def run(
     ip_json_file = os.path.join(storage_directory, "IPs.json")
     try:
         ip_file = json.load(open(ip_json_file))
+        robot_dict = ip_file.get("ip_address_list")
     except FileNotFoundError:
         print(f"Add .json file with robot IPs to: {storage_directory}.")
         sys.exit()
@@ -294,7 +295,7 @@ def run(
         ip_or_all = input("IP Address or ALL: ")
         calibration_data = []
         if ip_or_all.upper() == "ALL":
-            ip_address_list = ip_file["ip_address_list"]
+            ip_address_list = list(robot_dict.keys())
             for ip in ip_address_list:
                 saved_file_path, calibration = read_robot_logs.get_calibration_offsets(
                     ip, storage_directory

abr-testing/abr_testing/data_collection/abr_google_drive.py

@@ -44,6 +44,7 @@ def create_data_dictionary(
     headers: List[str] = []
     headers_lpc: List[str] = []
     list_of_heights: List[List[Any]] = [[], [], [], [], [], [], [], []]
+    hellma_plate_orientation = False  # default hellma plate is not rotated.
     for filename in os.listdir(storage_directory):
         file_path = os.path.join(storage_directory, filename)
         if file_path.endswith(".json"):
@@ -67,6 +68,10 @@ def create_data_dictionary(
         if run_id in runs_to_save:
             print(f"started reading run {run_id}.")
             robot = file_results.get("robot_name")
+            parameters = file_results.get("runTimeParameters", "")
+            for parameter in parameters:
+                if parameter["displayName"] == "Hellma Plate Orientation":
+                    hellma_plate_orientation = bool(parameter["value"])
             protocol_name = file_results["protocol"]["metadata"].get("protocolName", "")
             software_version = file_results.get("API_Version", "")
             left_pipette = file_results.get("left", "")
@@ -123,7 +128,7 @@ def create_data_dictionary(
                     file_results, labware_name="opentrons_tough_pcr_auto_sealing_lid"
                 )
                 plate_reader_dict = read_robot_logs.plate_reader_commands(
-                    file_results, hellma_plate_standards
+                    file_results, hellma_plate_standards, hellma_plate_orientation
                 )
                 list_of_heights = read_robot_logs.liquid_height_commands(
                     file_results, list_of_heights

abr-testing/abr_testing/data_collection/abr_robot_error.py

@@ -125,15 +125,20 @@ def compare_lpc_to_historical_data(
         & (df_lpc_data["Robot"] == robot)
         & (df_lpc_data["Module"] == labware_dict["Module"])
         & (df_lpc_data["Adapter"] == labware_dict["Adapter"])
-        & (df_lpc_data["Run Ending Error"] < 1)
+        & (df_lpc_data["Run Ending Error"])
+        < 1
     ]
     # Converts coordinates to floats and finds averages.
-    x_float = [float(value) for value in relevant_lpc["X"]]
-    y_float = [float(value) for value in relevant_lpc["Y"]]
-    z_float = [float(value) for value in relevant_lpc["Z"]]
-    current_x = round(labware_dict["X"], 2)
-    current_y = round(labware_dict["Y"], 2)
-    current_z = round(labware_dict["Z"], 2)
+    try:
+        x_float = [float(value) for value in relevant_lpc["X"]]
+        y_float = [float(value) for value in relevant_lpc["Y"]]
+        z_float = [float(value) for value in relevant_lpc["Z"]]
+        current_x = round(labware_dict["X"], 2)
+        current_y = round(labware_dict["Y"], 2)
+        current_z = round(labware_dict["Z"], 2)
+    except (ValueError):
+        x_float, y_float, z_float = [0.0], [0.0], [0.0]
+        current_x, current_y, current_z = 0.0, 0.0, 0.0
     try:
         avg_x = round(mean(x_float), 2)
         avg_y = round(mean(y_float), 2)
@@ -247,7 +252,7 @@ def get_error_runs_from_robot(ip: str) -> List[str]:
         f"http://{ip}:31950/runs", headers={"opentrons-version": "3"}
     )
     run_data = response.json()
-    run_list = run_data["data"]
+    run_list = run_data.get("data", [])
     for run in run_list:
         run_id = run["id"]
         num_of_errors = len(run["errors"])
@@ -258,7 +263,7 @@ def get_error_runs_from_robot(ip: str) -> List[str]:
 
 def get_robot_state(
     ip: str, reported_string: str
-) -> Tuple[Any, Any, Any, List[str], str]:
+) -> Tuple[Any, Any, Any, List[str], List[str], str]:
     """Get robot status in case of non run error."""
     description = dict()
     # Get instruments attached to robot
@@ -274,10 +279,11 @@ def get_robot_state(
         f"http://{ip}:31950/health", headers={"opentrons-version": "3"}
     )
     health_data = response.json()
-    parent = health_data.get("name", "")
+    print(f"health data {health_data}")
+    robot = health_data.get("name", "")
     # Create summary name
-    description["robot_name"] = parent
-    summary = parent + "_" + reported_string
+    description["robot_name"] = robot
+    summary = robot + "_" + reported_string
     affects_version = health_data.get("api_version", "")
     description["affects_version"] = affects_version
     # Instruments Attached
@@ -297,6 +303,12 @@ def get_robot_state(
         description[module["moduleType"]] = module
     components = ["Flex-RABR"]
     components = match_error_to_component("RABR", reported_string, components)
+    if "alpha" in affects_version:
+        components.append("flex internal releases")
+    labels = [robot]
+    if "8.2" in affects_version:
+        labels.append("8_2_0")
+    parent = affects_version + " Bugs"
     print(components)
     end_time = datetime.now()
     print(end_time)
@@ -317,13 +329,14 @@ def get_robot_state(
         parent,
         affects_version,
         components,
+        labels,
         whole_description_str,
     )
 
 
 def get_run_error_info_from_robot(
     ip: str, one_run: str, storage_directory: str
-) -> Tuple[str, str, str, List[str], str, str]:
+) -> Tuple[str, str, str, List[str], List[str], str, str]:
     """Get error information from robot to fill out ticket."""
     description = dict()
     # get run information
@@ -339,16 +352,19 @@ def get_run_error_info_from_robot(
     error_code = error_dict["Error_Code"]
     error_instrument = error_dict["Error_Instrument"]
     # JIRA Ticket Fields
-
+    robot = results.get("robot_name", "")
     failure_level = "Level " + str(error_level) + " Failure"
 
     components = [failure_level, "Flex-RABR"]
     components = match_error_to_component("RABR", str(error_type), components)
-    print(components)
     affects_version = results["API_Version"]
-    parent = results.get("robot_name", "")
-    print(parent)
-    summary = parent + "_" + str(one_run) + "_" + str(error_code) + "_" + error_type
+    if "alpha" in affects_version:
+        components.append("flex internal releases")
+    labels = [robot]
+    if "8.2" in affects_version:
+        labels.append("8_2_0")
+    parent = affects_version + " Bugs"
+    summary = robot + "_" + str(one_run) + "_" + str(error_code) + "_" + error_type
     # Description of error
     description["protocol_name"] = results["protocol"]["metadata"].get(
         "protocolName", ""
@@ -430,6 +446,7 @@ def get_run_error_info_from_robot(
         parent,
         affects_version,
         components,
+        labels,
         whole_description_str,
         saved_file_path,
     )
@@ -503,18 +520,20 @@ def get_run_error_info_from_robot(
         one_run = error_runs[-1]  # Most recent run with error.
         (
             summary,
-            robot,
+            parent,
             affects_version,
             components,
+            labels,
             whole_description_str,
             run_log_file_path,
         ) = get_run_error_info_from_robot(ip, one_run, storage_directory)
     else:
         (
             summary,
-            robot,
+            parent,
             affects_version,
             components,
+            labels,
             whole_description_str,
         ) = get_robot_state(ip, run_or_other)
     # Get Calibration Data
@@ -525,16 +544,8 @@ def get_run_error_info_from_robot(
     print(f"Making ticket for {summary}.")
     # TODO: make argument or see if I can get rid of with using board_id.
     project_key = "RABR"
-    print(robot)
-    try:
-        parent_key = project_key + "-" + robot.split("ABR")[1]
-    except IndexError:
-        parent_key = ""
-
-    # Grab all previous issues
-    all_issues = ticket.issues_on_board(project_key)
-
     # TODO: read board to see if ticket for run id already exists.
+    all_issues = ticket.issues_on_board(project_key)
     # CREATE TICKET
     issue_key, raw_issue_url = ticket.create_ticket(
         summary,
@@ -546,7 +557,8 @@ def get_run_error_info_from_robot(
         "Medium",
         components,
         affects_version,
-        parent_key,
+        labels,
+        parent,
     )
     # Link Tickets
     to_link = ticket.match_issues(all_issues, summary)

abr-testing/abr_testing/data_collection/get_run_logs.py

@@ -104,10 +104,11 @@ def get_all_run_logs(
     ip_json_file = os.path.join(storage_directory, "IPs.json")
     try:
         ip_file = json.load(open(ip_json_file))
+        robot_dict = ip_file.get("ip_address_list")
     except FileNotFoundError:
         print(f"Add .json file with robot IPs to: {storage_directory}.")
         sys.exit()
-    ip_address_list = ip_file["ip_address_list"]
+    ip_address_list = list(robot_dict.keys())
     runs_from_storage = read_robot_logs.get_run_ids_from_google_drive(google_drive)
     for ip in ip_address_list:
         runs = get_run_ids_from_robot(ip)

abr-testing/abr_testing/data_collection/read_robot_logs.py

@@ -250,7 +250,9 @@ def liquid_height_commands(
 
 
 def plate_reader_commands(
-    file_results: Dict[str, Any], hellma_plate_standards: List[Dict[str, Any]]
+    file_results: Dict[str, Any],
+    hellma_plate_standards: List[Dict[str, Any]],
+    orientation: bool,
 ) -> Dict[str, object]:
     """Plate Reader Command Counts."""
     commandData = file_results.get("commands", "")
@@ -279,38 +281,46 @@ def plate_reader_commands(
             read = "yes"
         elif read == "yes" and commandType == "comment":
             result = command["params"].get("message", "")
-            formatted_result = result.split("result: ")[1]
-            result_dict = eval(formatted_result)
-            result_dict_keys = list(result_dict.keys())
-            if len(result_dict_keys) > 1:
-                read_type = "multi"
-            else:
-                read_type = "single"
-            for wavelength in result_dict_keys:
-                one_wavelength_dict = result_dict.get(wavelength)
-                result_ndarray = plate_reader.convert_read_dictionary_to_array(
-                    one_wavelength_dict
-                )
-                for item in hellma_plate_standards:
-                    wavelength_of_interest = item["wavelength"]
-                    if str(wavelength) == str(wavelength_of_interest):
-                        error_cells = plate_reader.check_byonoy_data_accuracy(
-                            result_ndarray, item, False
+            if "result:" in result:
+                plate_name = result.split("result:")[0]
+                formatted_result = result.split("result: ")[1]
+                print(formatted_result)
+                result_dict = eval(formatted_result)
+                result_dict_keys = list(result_dict.keys())
+                if len(result_dict_keys) > 1:
+                    read_type = "multi"
+                else:
+                    read_type = "single"
+                if "hellma_plate" in plate_name:
+                    for wavelength in result_dict_keys:
+                        one_wavelength_dict = result_dict.get(wavelength)
+                        result_ndarray = plate_reader.convert_read_dictionary_to_array(
+                            one_wavelength_dict
                         )
-                        if len(error_cells[0]) > 0:
-                            percent = (96 - len(error_cells)) / 96 * 100
-                            for cell in error_cells:
-                                print(
-                                    "FAIL: Cell " + str(cell) + " out of accuracy spec."
+                        for item in hellma_plate_standards:
+                            wavelength_of_interest = item["wavelength"]
+                            if str(wavelength) == str(wavelength_of_interest):
+                                error_cells = plate_reader.check_byonoy_data_accuracy(
+                                    result_ndarray, item, orientation
                                 )
-                        else:
-                            percent = 100
-                            print(
-                                f"PASS: {wavelength_of_interest} meet accuracy specification"
-                            )
-                        final_result[read_type, wavelength, read_num] = percent
-                        read_num += 1
-            read = "no"
+                                if len(error_cells[0]) > 0:
+                                    percent = (96 - len(error_cells)) / 96 * 100
+                                    for cell in error_cells:
+                                        print(
+                                            "FAIL: Cell "
+                                            + str(cell)
+                                            + " out of accuracy spec."
+                                        )
+                                else:
+                                    percent = 100
+                                    print(
+                                        f"PASS: {wavelength_of_interest} meet accuracy spec."
+                                    )
+                                final_result[read_type, wavelength, read_num] = percent
+                                read_num += 1
+                else:
+                    final_result = result_dict
+                read = "no"
     plate_dict = {
         "Plate Reader # of Reads": read_count,
         "Plate Reader Avg Read Time (sec)": avg_read_time,