Update pandas requirement from ~=1.3.4 to ~=1.5.2

.gitignore

@@ -1,5 +1,7 @@
 .idea/
 *.gcode
+*.log
+*.key
 secrets.json
 
 # Byte-compiled / optimized / DLL files
@@ -131,6 +133,3 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
-
-# Secrets
-secrets.key

google_spreadsheet.py

@@ -1,6 +1,11 @@
+import copy
+import logging
+import random
+import threading
 import time
 
 import gspread
+import pandas
 import pandas as pd
 from oauth2client.service_account import ServiceAccountCredentials
 from cryptography.fernet import Fernet
@@ -16,7 +21,11 @@ def __init__(self, google_secrets):
         self.gspread_creds = gspread.authorize(self.creds)
         self.queue_sheet = self.gspread_creds.open_by_key(self.vars["tokens"]["sheet"]).worksheet("Queue")
         self.dataframe = None
-        self.update_data()
+        self.df_lock = threading.Lock()
+
+        # Start update daemon
+        update_daemon = threading.Thread(target=self._update_data, daemon=True)
+        update_daemon.start()
 
     def __enter__(self):
         return self
@@ -25,7 +34,13 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         # disconnect?
         return
 
-    def update_data(self):
+    def get_data(self):
+        with self.df_lock:
+            df = copy.deepcopy(self.dataframe)
+        return df
+
+    def _update_data(self):
+        s = 5.0
         while True:
             try:
                 # TODO: fixme
@@ -34,67 +49,66 @@ def update_data(self):
                     requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='sheets.googleapis.com', port=443): Read timed out. (read timeout=120)
                 if left active for too long
                 """
-                self.dataframe = pd.DataFrame(self.queue_sheet.get_all_records(value_render_option="FORMULA", head=3))
-                break
+                with self.df_lock:
+                    self.dataframe = pd.DataFrame(self.queue_sheet.get_all_records(value_render_option="FORMULA", head=3))
+                s = 5.0  # reset s when successful
+                time.sleep(5)
             except gspread.exceptions.APIError as e:
                 # temporary error, keep trying
-                print(f"APIError: {e}\nRetrying every 5 seconds")
-                time.sleep(5)
+                logging.warning(f"APIError: {e}\nRetrying in {s:.2f} seconds")
+                s += random.random() * 5.0  # Increase backoff
+                time.sleep(s)
 
-        # example usage
-        # print(self.dataframe)
-        # print(self.dataframe.loc[:, "prus"] == "Alistair Mitchell")
-        # print(self.dataframe.loc[self.dataframe.loc[:, "prus"] == "Alistair Mitchell"])
+    def force_update_data(self):
+        while True:
+            try:
+                with self.df_lock:
+                    self.dataframe = pd.DataFrame(self.queue_sheet.get_all_records(value_render_option="FORMULA", head=3))
+                return
+            except gspread.exceptions.APIError as e:
+                # temporary error, keep trying
+                logging.error(f"APIError: {e}\nForce update so not retrying")
+                with self.df_lock:
+                    self.dataframe = pd.DataFrame()
 
-    def get_running(self):
-        # return dict of two dataframes, one for each printer type, for rows where "Status" column is "Running"
-        prusaDf = self.dataframe.loc[
-            (self.dataframe.loc[:, "Status"] == "Running") & (self.dataframe.loc[:, "Printer Type"] == "Prusa")]
-        ultiDf = self.dataframe.loc[
-            (self.dataframe.loc[:, "Status"] == "Running") & (self.dataframe.loc[:, "Printer Type"] == "Ultimaker")]
-        return {"Prusa": prusaDf, "Ultimaker": ultiDf}
+    def get_printers(self, status):
+        result = {}
+        df = self.get_data()
+        # return dict of two dataframes, one for each printer type, for rows where "Status" column is "Queued"
+        printer_types = list(set(df.loc[:, "Printer Type"]))
+        for printer_type in printer_types:
+            try:
+                result[printer_type] = df.loc[(df.loc[:, "Status"] == status) & (df.loc[:, "Printer Type"] == printer_type)]
+            except KeyError:
+                result[printer_type] = pandas.DataFrame()
 
-    #####
+        return result
+
+    def get_running(self):
+        return self.get_printers("Running")
 
     def get_queued(self):
-        # return dict of two dataframes, one for each printer type, for rows where "Status" column is "Queued"
-        prusaDf = self.dataframe.loc[
-            (self.dataframe.loc[:, "Status"] == "Queued") & (self.dataframe.loc[:, "Printer Type"] == "Prusa")]
-        ultiDf = self.dataframe.loc[
-            (self.dataframe.loc[:, "Status"] == "Queued") & (self.dataframe.loc[:, "Printer Type"] == "Ultimaker")]
-        return {"Prusa": prusaDf, "Ultimaker": ultiDf}
-
-        # out_rows = []
-        # for cell in self.queue_sheet.findall(search_str):
-        #     if cell.col == 9:  # only search "Status" column
-        #         if printer_type == "" or printer_type == self.get_cell_value(cell.row, 10):
-        #             out_rows.append(self.queue_sheet.row_values(cell.row))
-        # return out_rows
+        return self.get_printers("Queued")
 
     def get_cell_value(self, row, col):
         return self.queue_sheet.cell(row, col).value
 
     def set_row(self, data):
-        self.update_data()  # ensure data is up to date
-        row = self.dataframe.index[self.dataframe.loc[:, "Unique ID"] == data.loc[:, "Unique ID"].values[0]].tolist()
+        df = self.get_data()
+        # self.update_data()  # ensure data is up to date
+        row = df.index[df.loc[:, "Unique ID"] == data.loc[:, "Unique ID"].values[0]].tolist()
         if len(row) != 1:
             raise TypeError(f"Multiple rows match: {data.loc[:, 'Unique ID']}")
 
         row = row[0] + 4  # +4 for header & zero-indexing,
 
-        # values = []
-        # for val in data.values.tolist():  # elementwise convert numpy numbers to standard numbers
-        #     try:
-        #         values.append(val.item())
-        #     except AttributeError:  # for inconvertible data-types
-        #         values.append(val)
+        # [values] must be a 2x-nested list to write correctly
+        self.queue_sheet.update(f"{row}:{row}", data.values.tolist(), raw=False)
 
-        self.queue_sheet.update(f"{row}:{row}", data.values.tolist(), raw=False)  # [values] must be a 2x-nested list to write correctly
-        """
-        [[1, 2], [3, 4]]        - writes 2x2
-        [[1, 2, 3, 4]]          - writes 1x4
-        [[1], [2], [3], [4]]    - writes 4x1
-        """
+        # Examples:
+        # [[1, 2], [3, 4]]        - writes 2x2
+        # [[1, 2, 3, 4]]          - writes 1x4
+        # [[1], [2], [3], [4]]    - writes 4x1
 
 
 if __name__ == "__main__":
@@ -119,11 +133,10 @@ def set_row(self, data):
     secret_vars = json.loads(decrypted)
 
     queue = Spreadsheet(secret_vars["google_secrets"])
-    queue.update_data()
+    # queue.update_data()
     print(queue.get_running()["Prusa"]["Printer"].tolist())
 
     pd.reset_option('display.max_rows')
     pd.reset_option('display.max_columns')
     pd.reset_option('display.width')
     pd.reset_option('display.max_colwidth')
-

main.py

@@ -186,7 +186,7 @@ def cancel_print(backend):
     print(f"Go and check {printing_printers[n]} is clear and ready to print again.")
     time.sleep(10)
 
-def detach_printer(backend):
+def disconnect_printer(backend):
     online_printers = []
     for i_printer in backend.printers.keys():
         if "available" in backend.printers[i_printer]["details"]["state"]:
@@ -206,7 +206,7 @@ def detach_printer(backend):
     backend.disconnect_printer(online_printers[n])
     print(f'{online_printers[n]} is now {backend.printers[online_printers[n]]["details"]["state"].lower()}')
 
-def attach_printer(backend):
+def connect_printer(backend):
     offline_printers = []
     for i_printer in backend.printers.keys():
         if "offline" in backend.printers[i_printer]["details"]["state"]:
@@ -235,14 +235,14 @@ def attach_printer(backend):
     # args = parser.parse_args()
     # secrets_key = args.secrets_key
 
-    printer_list = input("Enter printer type: ('Prusa' or 'Ultimaker')\n").lower()
+    group = input("Select area: ('Mainspace' or 'Heartspace')\n").lower()
     # support shortcuts for common selections
-    if printer_list == "p":
-        printer_list = "prusa"
-    elif printer_list == "u":
-        printer_list = "ultimaker"
+    if group == "m":
+        group = "mainspace"
+    elif group == "h":
+        group = "heartspace"
 
-    backend = Backend(printer_list=str(printer_list).capitalize())
+    backend = Backend(printer_group=str(group).capitalize())
 
     # start with some information
     backend.update()
@@ -257,8 +257,8 @@ def attach_printer(backend):
                        # "'a'\t-\tAdmin Mode"
 
     admin_option_list = "\nAdmin Options:\n" \
-                        "'a'\t-\tAttach a printer (to a Pi)\n" \
-                        "'d'\t-\tDetach a printer (from a Pi)"
+                        "'c'\t-\tConnect a printer (to a Pi)\n" \
+                        "'d'\t-\tDisconnect a printer (from a Pi)"
 
     loop = True
     while loop:
@@ -293,9 +293,9 @@ def attach_printer(backend):
                 choice = input(">> ").upper()
 
                 if choice == "C":
-                    attach_printer(backend)
+                    connect_printer(backend)
                 elif choice == "D":
-                    detach_printer(backend)
+                    disconnect_printer(backend)
             else:
                 print("Access denied")
 

main_backend.py

@@ -4,15 +4,24 @@
 import time
 from cryptography.fernet import Fernet
 import os
+import logging
 
+logging.basicConfig()
+logging.getLogger().setLevel(logging.ERROR)
 
 class Backend:
-    def __init__(self, printer_list):
+    def __init__(self, printer_group):
+        logging.info(f"Starting backend, {printer_group}")
         self.secrets = {}
         self.load_secrets()
 
-        self.printers = self.secrets["printers"][printer_list]
-        self.printer_type = list(set([val["type"] for i, val in self.printers.items()]))[0].capitalize()
+        valid_groups = list(self.secrets["printers"].keys())
+        if not printer_group in valid_groups:
+            logging.critical("Invalid printer group provided")
+            exit(-1)
+
+        self.printers = self.secrets["printers"][printer_group]
+        self.printer_type = list(set([val["type"] for i, val in self.printers.items()]))[0]
         self.queue = print_queue.PrintQueue(google_secrets=self.secrets["google_secrets"], printer_type=self.printer_type)
         print("Performing initial printer connection, this may take some time")
         self.fleet = print_fleet.PrintFleet(self.printers)
@@ -32,22 +41,21 @@ def load_secrets(self):
         decrypted = fernet.decrypt(data)
 
         self.secrets = json.loads(decrypted)
-
     def connect(self):
         self.fleet.connect("all")
 
     def connect_printer(self, printer_name):
-        self.fleet.attach_printer(printer_name)
+        self.fleet.connect_printer(printer_name)
         self.update()
 
     def disconnect_printer(self, printer_name):
-        self.fleet.detach_printer(printer_name)
+        self.fleet.disconnect_printer(printer_name)
         self.update()
 
     def update(self):
         self.queue.update()
         running_printers = self.queue.get_running_printers()
-        self.fleet.update("all", queue_running=running_printers)
+        self.fleet.update_running(running_printers)
 
     def do_print(self, printer_name):
         filename = self.queue.download_selected()

print_fleet.py

@@ -23,7 +23,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 
     def connect_clients(self):
         for printer, accessDict in self.printer_access.items():
-            self.printers[printer] = {"name": printer, "client": None, "print_job": None, 'status': 'offline',
+            self.printers[printer] = {"printer_name": printer, "client": None, "print_job": None, 'status': 'offline',
                                       'printing': False, 'details': {}}
             try:
                 print(f"Connecting to {printer.capitalize()}... ", end='')
@@ -64,7 +64,7 @@ def update_status(self, queue_running):
                         status = printer['client'].printer()
                         job_info = printer['client'].job_info()
 
-                        # print(f"Octoprint Status for {printer['name']}:\n{status}\nend")  # TODO make debug
+                        # print(f"Octoprint Status for {printer['printer_name']}:\n{status}\nend")  # TODO make debug
 
                         printer['details'] = {'status': status,
                                               'job_info': job_info
@@ -96,13 +96,13 @@ def update_status(self, queue_running):
                 continue
 
         for printer in self.printers.values():
-            if printer['name'] not in queue_running and printer['printing']:
+            if printer['printer_name'] not in queue_running and printer['printing']:
                 printer['status'] = "invalid"
 
     def get_status(self):
         status_dict = {"available": [], "printing": [], "finished": [], "invalid": [], "offline": []}
         for printer in self.printers.values():
-            status_dict[printer["status"]].append(printer["name"])
+            status_dict[printer["status"]].append(printer["printer_name"])
         return status_dict
 
     def add_print(self, filename, path=""):
@@ -122,7 +122,7 @@ def cancel_print(self):
         t0 = time.time()
         while self.selected_printer["client"].printer()['state']['flags']['cancelling']:
             time.sleep(0.5)
-        print(f"Cancel of {self.selected_printer['name']} complete, time taken: {time.time() - t0:.1f}s")
+        print(f"Cancel of {self.selected_printer['printer_name']} complete, time taken: {time.time() - t0:.1f}s")
         time.sleep(0.5)
 
     def clear_files(self):
@@ -146,5 +146,5 @@ def select_printer(self, name):
     #     """
     #     message = "\nCurrent GCODE Files:\n"
     #     for i, k in enumerate(self.client.files()['files']):
-    #         message += f"{i}.\t{k['name']}\n"
+    #         message += f"{i}.\t{k['printer_name']}\n"
     #     print(message)