Skip to content

Commit

Permalink
fix lint
Browse files Browse the repository at this point in the history
  • Loading branch information
lu-wang-dl committed May 24, 2024
1 parent ccde2d8 commit 25deb39
Showing 1 changed file with 4 additions and 1 deletion.
5 changes: 4 additions & 1 deletion joblibspark/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"""
The utils functions for joblib spark backend.
"""

from packaging.version import Version

# pylint: disable=import-outside-toplevel
def get_spark_session():
Expand All @@ -42,6 +42,9 @@ def create_resource_profile(num_cpus_worker_node, num_gpus_worker_node):
:param num_gpus_worker_node: Number of gpus for the Spark worker node.
:return: Spark ResourceProfile
"""

if Version(pyspark.__version__).release < (3, 1, 0):
return None
from pyspark.resource.profile import ResourceProfileBuilder
from pyspark.resource.requests import TaskResourceRequests

Expand Down

0 comments on commit 25deb39

Please sign in to comment.