From 1786702c51ac6154cbd814296eff2e29a5e8a183 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Tue, 22 Jun 2021 10:51:48 -0400 Subject: [PATCH 01/12] Added lsf jobgroup - Updated submit, abort, suspend, and resume lsf functions - Added suspend and resume to general job_submitter - Updated tests --- batch_systems/lsf_client/lsf_client.py | 53 +++++++++++-------- orchestrator/tasks.py | 26 +++++++-- submitter/jobsubmitter.py | 22 ++++++-- .../nextflow_jobsubmitter.py | 4 +- submitter/toil_submitter/toil_jobsubmitter.py | 5 +- tests/test_lsf_client.py | 15 ++++-- 6 files changed, 84 insertions(+), 41 deletions(-) diff --git a/batch_systems/lsf_client/lsf_client.py b/batch_systems/lsf_client/lsf_client.py index d5446a0e..c8142be9 100644 --- a/batch_systems/lsf_client/lsf_client.py +++ b/batch_systems/lsf_client/lsf_client.py @@ -1,5 +1,5 @@ """ -Submit and monitor LSF jobs +Submit, monitor, and control LSF jobs """ import os import re @@ -25,7 +25,7 @@ def __init__(self): """ self.logger = logging.getLogger("LSF_client") - def submit(self, command, job_args, stdout, env={}): + def submit(self, command, job_args, stdout, job_id, env={}): """ Submit command to LSF and store log in stdout @@ -38,7 +38,7 @@ def submit(self, command, job_args, stdout, env={}): Returns: int: lsf job id """ - bsub_command = ["bsub", "-sla", settings.LSF_SLA, "-oo", stdout] + job_args + bsub_command = ["bsub", "-sla", settings.LSF_SLA, "-g", job_id, "-oo", stdout] + job_args bsub_command.extend(command) current_env = os.environ.copy() @@ -57,20 +57,19 @@ def submit(self, command, job_args, stdout, env={}): ) return self._parse_procid(process.stdout) - def abort(self, external_job_id): + def abort(self, job_id): """ Kill LSF job Args: - external_job_id (str): external_job_id + job_id (str): job_id Returns: bool: successful """ - bkill_command = ["bkill", external_job_id] - process = subprocess.run( - bkill_command, check=True, stdout=subprocess.PIPE, universal_newlines=True - ) + self.logger.debug("Aborting LSF jobs for job %s", job_id) + bkill_command = ["bkill", "-g", job_id, "0"] + process = subprocess.run(bkill_command, check=True, stdout=subprocess.PIPE, universal_newlines=True) if process.returncode == 0: return True return False @@ -217,26 +216,36 @@ def status(self, external_job_id): str(external_job_id), ] self.logger.debug("Checking lsf status for job: %s", external_job_id) - process = subprocess.run( - bsub_command, check=True, stdout=subprocess.PIPE, universal_newlines=True - ) + process = subprocess.run(bsub_command, check=True, stdout=subprocess.PIPE, universal_newlines=True) status = self._parse_status(process.stdout, external_job_id) return status - def suspend(self, external_job_id): - bsub_command = ["bstop", str(external_job_id)] - process = subprocess.run( - bsub_command, stdout=subprocess.PIPE, universal_newlines=True - ) + def suspend(self, job_id): + """ + Suspend LSF job + Args: + extrnsl_job_id (str): id of job + Returns: + bool: successful + """ + self.logger.debug("Suspending LSF jobs for job %s", job_id) + bsub_command = ["bstop", "-g", job_id, "0"] + process = subprocess.run(bsub_command, stdout=subprocess.PIPE, universal_newlines=True) if process.returncode == 0: return True return False - def resume(self, external_job_id): - bsub_command = ["bresume", str(external_job_id)] - process = subprocess.run( - bsub_command, stdout=subprocess.PIPE, universal_newlines=True - ) + def resume(self, job_id): + """ + Resume LSF job + Args: + job_id (str): id of job + Returns: + bool: successful + """ + self.logger.debug("Unsuspending LSF jobs for job %s", job_id) + bsub_command = ["bresume", "-g", job_id, "0"] + process = subprocess.run(bsub_command, stdout=subprocess.PIPE, universal_newlines=True) if process.returncode == 0: return True return False diff --git a/orchestrator/tasks.py b/orchestrator/tasks.py index 0e2e82a0..14ff1db7 100644 --- a/orchestrator/tasks.py +++ b/orchestrator/tasks.py @@ -52,8 +52,16 @@ def on_failure_to_submit(self, exc, task_id, args, kwargs, einfo): def suspend_job(job): if Status(job.status).transition(Status.SUSPENDED): - client = LSFClient() - if not client.suspend(job.external_id): + submitter = JobSubmitterFactory.factory( + job.type, + str(job.id), + job.app, + job.inputs, + job.root_dir, + job.resume_job_store_location, + ) + job_suspended = submitter.suspend() + if not job_suspended: raise RetryException("Failed to suspend job: %s" % str(job.id)) job.update_status(Status.SUSPENDED) return @@ -61,8 +69,16 @@ def suspend_job(job): def resume_job(job): if Status(job.status) == Status.SUSPENDED: - client = LSFClient() - if not client.resume(job.external_id): + submitter = JobSubmitterFactory.factory( + job.type, + str(job.id), + job.app, + job.inputs, + job.root_dir, + job.resume_job_store_location, + ) + job_resumed = submitter.resume() + if not job_resumed: raise RetryException("Failed to resume job: %s" % str(job.id)) job.update_status(Status.RUNNING) return @@ -299,7 +315,7 @@ def abort_job(job): job.root_dir, job.resume_job_store_location, ) - job_killed = submitter.abort(job.external_id) + job_killed = submitter.abort() if not job_killed: raise RetryException("Failed to abort job %s" % str(job.id)) job.abort() diff --git a/submitter/jobsubmitter.py b/submitter/jobsubmitter.py index 730be315..c5884485 100644 --- a/submitter/jobsubmitter.py +++ b/submitter/jobsubmitter.py @@ -3,8 +3,9 @@ class JobSubmitter(object): - def __init__(self, app, inputs, walltime, memlimit): + def __init__(self, job_id, app, inputs, walltime, memlimit): self.app = App.factory(app) + self.job_id = job_id self.inputs = inputs self.lsf_client = LSFClient() self.walltime = walltime @@ -20,8 +21,23 @@ def submit(self): def status(self, external_id): return self.lsf_client.status(external_id) - def abort(self, external_id): - return self.lsf_client.abort(external_id) + def abort(self): + """ + Aborts the job + """ + return self.lsf_client.abort(self.job_id) + + def resume(self): + """ + Resumes the job + """ + return self.lsf_client.resume(self.job_id) + + def suspend(self): + """ + Suspends the job + """ + return self.lsf_client.suspend(self.job_id) def get_commandline_status(self, cache): """ diff --git a/submitter/nextflow_submitter/nextflow_jobsubmitter.py b/submitter/nextflow_submitter/nextflow_jobsubmitter.py index 6f2f8273..9b157eb8 100644 --- a/submitter/nextflow_submitter/nextflow_jobsubmitter.py +++ b/submitter/nextflow_submitter/nextflow_jobsubmitter.py @@ -6,9 +6,7 @@ class NextflowJobSubmitter(JobSubmitter): - def __init__( - self, job_id, app, inputs, root_dir, resume_jobstore, walltime, memlimit - ): + def __init__(self, job_id, app, inputs, root_dir, resume_jobstore, walltime, memlimit): """ :param job_id: :param app: github.url diff --git a/submitter/toil_submitter/toil_jobsubmitter.py b/submitter/toil_submitter/toil_jobsubmitter.py index cc735b93..4a94b85b 100644 --- a/submitter/toil_submitter/toil_jobsubmitter.py +++ b/submitter/toil_submitter/toil_jobsubmitter.py @@ -27,8 +27,7 @@ class ToilJobSubmitter(JobSubmitter): def __init__( self, job_id, app, inputs, root_dir, resume_jobstore, walltime, memlimit ): - JobSubmitter.__init__(self, app, inputs, walltime, memlimit) - self.job_id = job_id + JobSubmitter.__init__(self, job_id, app, inputs, walltime, memlimit) self.resume_jobstore = resume_jobstore if resume_jobstore: self.job_store_dir = resume_jobstore @@ -55,7 +54,7 @@ def submit(self): env[e] = None external_id = self.lsf_client.submit( - command_line, self._job_args(), log_path, env + command_line, self._job_args(), log_path, self.job_id, env ) return external_id, self.job_store_dir, self.job_work_dir, self.job_outputs_dir diff --git a/tests/test_lsf_client.py b/tests/test_lsf_client.py index 4c3162d7..00b095af 100644 --- a/tests/test_lsf_client.py +++ b/tests/test_lsf_client.py @@ -16,6 +16,7 @@ def setUp(self): Cannot connect to LSF. Please wait ... """ self.example_id = 12345678 + self.example_job_id = 12345 self.submit_response = "Job <{}> is submitted".format(self.example_id) self.submit_response_please_wait = please_wait_str + self.submit_response self.lsf_client = LSFClient() @@ -65,8 +66,12 @@ def test_submit(self, submit_process): submit_process_obj = Mock() submit_process_obj.stdout = self.submit_response submit_process.return_value = submit_process_obj - lsf_id = self.lsf_client.submit(command, args, stdout_file, {}) - expected_command = ["bsub", "-sla", settings.LSF_SLA, "-oo", stdout_file] + args + command + lsf_id = self.lsf_client.submit(command, args, stdout_file, self.example_job_id, {}) + expected_command = ( + ["bsub", "-sla", settings.LSF_SLA, "-g", self.example_job_id, "-oo", stdout_file] + + args + + command + ) self.assertEqual(lsf_id, self.example_id) self.assertEqual(submit_process.call_args[0][0], expected_command) @@ -81,7 +86,7 @@ def test_submit_slow_lsf(self, submit_process): submit_process_obj = Mock() submit_process_obj.stdout = self.submit_response_please_wait submit_process.return_value = submit_process_obj - lsf_id = self.lsf_client.submit(command, args, stdout_file, {}) + lsf_id = self.lsf_client.submit(command, args, stdout_file, self.example_job_id, {}) self.assertEqual(lsf_id, self.example_id) @patch("subprocess.run") @@ -92,8 +97,8 @@ def test_abort(self, abort_process): abort_process_obj = Mock() abort_process_obj.returncode = 0 abort_process.return_value = abort_process_obj - expected_command = ["bkill", self.example_id] - aborted = self.lsf_client.abort(self.example_id) + expected_command = ["bkill", "-g", self.example_job_id, "0"] + aborted = self.lsf_client.abort(self.example_job_id) self.assertEqual(abort_process.call_args[0][0], expected_command) self.assertEqual(aborted, True) From 81cbd8112b0c877e04763111259362775eccffe8 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Wed, 23 Jun 2021 15:04:11 -0400 Subject: [PATCH 02/12] Removed unused import --- orchestrator/tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/orchestrator/tasks.py b/orchestrator/tasks.py index 14ff1db7..55045568 100644 --- a/orchestrator/tasks.py +++ b/orchestrator/tasks.py @@ -3,7 +3,6 @@ import logging from datetime import timedelta from celery import shared_task -from batch_systems.lsf_client import LSFClient from django.conf import settings from django.db import transaction from django.utils.timezone import now From cab488f510671a1f45384a793be6f396ca84fce7 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Tue, 6 Jul 2021 12:57:05 -0400 Subject: [PATCH 03/12] Fixed black formatting --- tests/test_lsf_client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_lsf_client.py b/tests/test_lsf_client.py index 00b095af..ee6b0ddc 100644 --- a/tests/test_lsf_client.py +++ b/tests/test_lsf_client.py @@ -68,9 +68,7 @@ def test_submit(self, submit_process): submit_process.return_value = submit_process_obj lsf_id = self.lsf_client.submit(command, args, stdout_file, self.example_job_id, {}) expected_command = ( - ["bsub", "-sla", settings.LSF_SLA, "-g", self.example_job_id, "-oo", stdout_file] - + args - + command + ["bsub", "-sla", settings.LSF_SLA, "-g", self.example_job_id, "-oo", stdout_file] + args + command ) self.assertEqual(lsf_id, self.example_id) self.assertEqual(submit_process.call_args[0][0], expected_command) From 8e27331fa6e0697f8e767499e9cb5ad264e1a175 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Mon, 12 Jul 2021 12:07:29 -0400 Subject: [PATCH 04/12] Update lsf_client.py --- batch_systems/lsf_client/lsf_client.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/batch_systems/lsf_client/lsf_client.py b/batch_systems/lsf_client/lsf_client.py index 8598f8ba..e60eb579 100644 --- a/batch_systems/lsf_client/lsf_client.py +++ b/batch_systems/lsf_client/lsf_client.py @@ -10,6 +10,8 @@ from django.conf import settings from orchestrator.models import Status +def format_lsf_job_id(job_id): + return "/"+job_id class LSFClient(object): """ @@ -38,7 +40,7 @@ def submit(self, command, job_args, stdout, job_id, env={}): Returns: int: lsf job id """ - bsub_command = ["bsub", "-sla", settings.LSF_SLA, "-g", job_id, "-oo", stdout] + job_args + bsub_command = ["bsub", "-sla", settings.LSF_SLA, "-g", format_lsf_job_id(job_id), "-oo", stdout] + job_args bsub_command.extend(command) current_env = os.environ.copy() @@ -68,7 +70,7 @@ def abort(self, job_id): bool: successful """ self.logger.debug("Aborting LSF jobs for job %s", job_id) - bkill_command = ["bkill", "-g", job_id, "0"] + bkill_command = ["bkill", "-g", format_lsf_job_id(job_id), "0"] process = subprocess.run(bkill_command, check=True, stdout=subprocess.PIPE, universal_newlines=True) if process.returncode == 0: return True @@ -221,7 +223,7 @@ def suspend(self, job_id): bool: successful """ self.logger.debug("Suspending LSF jobs for job %s", job_id) - bsub_command = ["bstop", "-g", job_id, "0"] + bsub_command = ["bstop", "-g", format_lsf_job_id(job_id), "0"] process = subprocess.run(bsub_command, stdout=subprocess.PIPE, universal_newlines=True) if process.returncode == 0: return True @@ -236,7 +238,7 @@ def resume(self, job_id): bool: successful """ self.logger.debug("Unsuspending LSF jobs for job %s", job_id) - bsub_command = ["bresume", "-g", job_id, "0"] + bsub_command = ["bresume", "-g", format_lsf_job_id(job_id), "0"] process = subprocess.run(bsub_command, stdout=subprocess.PIPE, universal_newlines=True) if process.returncode == 0: return True From 76b08e38b65c25cc36be5ecd2ada88b26285eeb4 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Mon, 12 Jul 2021 16:09:53 -0400 Subject: [PATCH 05/12] Fixed tests --- tests/test_lsf_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_lsf_client.py b/tests/test_lsf_client.py index ee6b0ddc..ce4dcc92 100644 --- a/tests/test_lsf_client.py +++ b/tests/test_lsf_client.py @@ -17,6 +17,7 @@ def setUp(self): """ self.example_id = 12345678 self.example_job_id = 12345 + self.example_lsf_id = '/12345' self.submit_response = "Job <{}> is submitted".format(self.example_id) self.submit_response_please_wait = please_wait_str + self.submit_response self.lsf_client = LSFClient() @@ -68,7 +69,7 @@ def test_submit(self, submit_process): submit_process.return_value = submit_process_obj lsf_id = self.lsf_client.submit(command, args, stdout_file, self.example_job_id, {}) expected_command = ( - ["bsub", "-sla", settings.LSF_SLA, "-g", self.example_job_id, "-oo", stdout_file] + args + command + ["bsub", "-sla", settings.LSF_SLA, "-g", self.example_lsf_id, "-oo", stdout_file] + args + command ) self.assertEqual(lsf_id, self.example_id) self.assertEqual(submit_process.call_args[0][0], expected_command) @@ -95,7 +96,7 @@ def test_abort(self, abort_process): abort_process_obj = Mock() abort_process_obj.returncode = 0 abort_process.return_value = abort_process_obj - expected_command = ["bkill", "-g", self.example_job_id, "0"] + expected_command = ["bkill", "-g", self.example_lsf_id, "0"] aborted = self.lsf_client.abort(self.example_job_id) self.assertEqual(abort_process.call_args[0][0], expected_command) self.assertEqual(aborted, True) From 127aceb75bcf27ca99e942b4bc3f9c4d8961a757 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Mon, 12 Jul 2021 16:16:57 -0400 Subject: [PATCH 06/12] Fixed lsf job id format --- batch_systems/lsf_client/lsf_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batch_systems/lsf_client/lsf_client.py b/batch_systems/lsf_client/lsf_client.py index e60eb579..54f5b616 100644 --- a/batch_systems/lsf_client/lsf_client.py +++ b/batch_systems/lsf_client/lsf_client.py @@ -11,7 +11,7 @@ from orchestrator.models import Status def format_lsf_job_id(job_id): - return "/"+job_id + return "/{}".format(job_id) class LSFClient(object): """ From e4a39f2f949be39568697fd5717e45ca78bf2f73 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Mon, 12 Jul 2021 16:33:44 -0400 Subject: [PATCH 07/12] Added black formatted code --- batch_systems/lsf_client/lsf_client.py | 2 ++ tests/test_lsf_client.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/batch_systems/lsf_client/lsf_client.py b/batch_systems/lsf_client/lsf_client.py index 54f5b616..3c9851fc 100644 --- a/batch_systems/lsf_client/lsf_client.py +++ b/batch_systems/lsf_client/lsf_client.py @@ -10,9 +10,11 @@ from django.conf import settings from orchestrator.models import Status + def format_lsf_job_id(job_id): return "/{}".format(job_id) + class LSFClient(object): """ Client for LSF diff --git a/tests/test_lsf_client.py b/tests/test_lsf_client.py index ce4dcc92..6ededa8d 100644 --- a/tests/test_lsf_client.py +++ b/tests/test_lsf_client.py @@ -17,7 +17,7 @@ def setUp(self): """ self.example_id = 12345678 self.example_job_id = 12345 - self.example_lsf_id = '/12345' + self.example_lsf_id = "/12345" self.submit_response = "Job <{}> is submitted".format(self.example_id) self.submit_response_please_wait = please_wait_str + self.submit_response self.lsf_client = LSFClient() From 0d4e592f93632fd131d4f1fe55b6fcde09764971 Mon Sep 17 00:00:00 2001 From: Ian Date: Tue, 13 Jul 2021 10:19:50 -0400 Subject: [PATCH 08/12] Update toil_jobsubmitter.py (#234) * Update toil_jobsubmitter.py * need to use github url --- submitter/toil_submitter/toil_jobsubmitter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/submitter/toil_submitter/toil_jobsubmitter.py b/submitter/toil_submitter/toil_jobsubmitter.py index d6e50df8..f51b07d9 100644 --- a/submitter/toil_submitter/toil_jobsubmitter.py +++ b/submitter/toil_submitter/toil_jobsubmitter.py @@ -155,7 +155,12 @@ def _memlimit(self): return ["-M", self.memlimit] if self.memlimit else [] def _command_line(self): - if "access" in self.app.github.lower() and "nucleo" not in self.app.github.lower(): + bypass_access_workflows = [ + "nucleo", + "access_qc_generation" + ] + should_bypass_access_env = any([w in self.app.github.lower() for w in bypass_access_workflows]) + if "access" in self.app.github.lower() and not should_bypass_access_env: """ Start ACCESS-specific code """ From d5ad919f445719b8750dc4f3a9484af628dab817 Mon Sep 17 00:00:00 2001 From: Sinisa Ivkovic Date: Wed, 14 Jul 2021 12:46:33 -0400 Subject: [PATCH 09/12] Fix nextflow submitter --- submitter/nextflow_submitter/nextflow_jobsubmitter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/submitter/nextflow_submitter/nextflow_jobsubmitter.py b/submitter/nextflow_submitter/nextflow_jobsubmitter.py index 3cd1697f..5b20b728 100644 --- a/submitter/nextflow_submitter/nextflow_jobsubmitter.py +++ b/submitter/nextflow_submitter/nextflow_jobsubmitter.py @@ -32,8 +32,7 @@ def __init__(self, job_id, app, inputs, root_dir, resume_jobstore, walltime, mem :param root_dir: :param resume_jobstore: """ - JobSubmitter.__init__(self, app, inputs, walltime, memlimit) - self.job_id = job_id + JobSubmitter.__init__(self, job_id, app, inputs, walltime, memlimit) self.resume_jobstore = resume_jobstore if resume_jobstore: self.job_store_dir = resume_jobstore From 070378c5e9621acbb34d45e3abe5643b83067c5c Mon Sep 17 00:00:00 2001 From: Sinisa Ivkovic Date: Wed, 14 Jul 2021 12:55:05 -0400 Subject: [PATCH 10/12] Fix formating --- submitter/toil_submitter/toil_jobsubmitter.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/submitter/toil_submitter/toil_jobsubmitter.py b/submitter/toil_submitter/toil_jobsubmitter.py index f51b07d9..4fd21d9e 100644 --- a/submitter/toil_submitter/toil_jobsubmitter.py +++ b/submitter/toil_submitter/toil_jobsubmitter.py @@ -155,10 +155,7 @@ def _memlimit(self): return ["-M", self.memlimit] if self.memlimit else [] def _command_line(self): - bypass_access_workflows = [ - "nucleo", - "access_qc_generation" - ] + bypass_access_workflows = ["nucleo", "access_qc_generation"] should_bypass_access_env = any([w in self.app.github.lower() for w in bypass_access_workflows]) if "access" in self.app.github.lower() and not should_bypass_access_env: """ From b886cd4b9f4926f85ff9709a8a4db5a94e0018cf Mon Sep 17 00:00:00 2001 From: Sinisa Ivkovic Date: Wed, 14 Jul 2021 13:12:03 -0400 Subject: [PATCH 11/12] Fix lsf submit --- submitter/nextflow_submitter/nextflow_jobsubmitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submitter/nextflow_submitter/nextflow_jobsubmitter.py b/submitter/nextflow_submitter/nextflow_jobsubmitter.py index 5b20b728..dda28e99 100644 --- a/submitter/nextflow_submitter/nextflow_jobsubmitter.py +++ b/submitter/nextflow_submitter/nextflow_jobsubmitter.py @@ -51,7 +51,7 @@ def submit(self): env["JAVA_HOME"] = "/opt/common/CentOS_7/java/jdk1.8.0_202/" env["PATH"] = env["JAVA_HOME"] + "bin:" + os.environ["PATH"] env["TMPDIR"] = self.job_tmp_dir - external_id = self.lsf_client.submit(command_line, self._job_args(), log_path, env) + external_id = self.lsf_client.submit(command_line, self._job_args(), log_path, self.job_id, env) return external_id, self.job_store_dir, self.job_work_dir, self.job_outputs_dir def _job_args(self): From b41514409ab742fe8021ff03926dc668ddf888b7 Mon Sep 17 00:00:00 2001 From: Sinisa Ivkovic Date: Thu, 15 Jul 2021 04:34:30 -0400 Subject: [PATCH 12/12] Version bump 1.17.0 --- ridgeback/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ridgeback/__init__.py b/ridgeback/__init__.py index 638c1217..30244104 100644 --- a/ridgeback/__init__.py +++ b/ridgeback/__init__.py @@ -1 +1 @@ -__version__ = "1.16.0" +__version__ = "1.17.0"