BUG: Fix RuntimeError: cannot schedule new futures after shutdown (#589)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

python/pyproject.toml

@@ -13,7 +13,7 @@ requires = [
     "requests>=2.4.0",
     "cloudpickle>=2.2.1; python_version>='3.11'",
     "cloudpickle==1.5.0; python_version<'3.11'",
-    "xoscar>=0.0.3",
+    "xoscar>=0.0.8",
 ]
 build-backend = "setuptools.build_meta"
 

python/setup.cfg

@@ -24,7 +24,7 @@ zip_safe = False
 include_package_data = True
 packages = find:
 install_requires =
-    xoscar>=0.0.3
+    xoscar>=0.0.8
     numpy>=1.14.0
     pandas>=1.0.0
     scipy>=1.0.0; sys_platform!="win32" or python_version>="3.10"

python/xorbits/_mars/services/cluster/uploader.py

@@ -90,11 +90,6 @@ async def _periodical_upload_node_info(self):
                     self._uploaded_future.set_result(None)
             except asyncio.CancelledError:  # pragma: no cover
                 break
-            except RuntimeError as ex:  # pragma: no cover
-                if "cannot schedule new futures" not in str(ex):
-                    # when atexit is triggered, the default pool might be shutdown
-                    # and to_thread will fail
-                    break
             except (
                 Exception
             ) as ex:  # pragma: no cover  # noqa: E722  # nosec  # pylint: disable=bare-except
@@ -166,6 +161,13 @@ async def upload_node_info(self, status: NodeStatus = None):
                     self._env_uploaded = True
                 except ValueError:
                     pass
+        except RuntimeError as ex:  # pragma: no cover
+            if "cannot schedule new futures" not in str(ex):
+                # when atexit is triggered, the default pool might be shutdown
+                # and to_thread will fail
+                return
+            logger.exception(f"Failed to upload node info")
+            raise
         except:  # noqa: E722  # nosec  # pylint: disable=bare-except  # pragma: no cover
             logger.exception(f"Failed to upload node info")
             raise