Update to Python 3.12 (#4901)

* Add Python 3.12 to CI

* Update sphinx-autoapi and astroid to deal with crash

pylint-dev/pylint#8782

* Remove dead comment

* Add rules to 3.11 build

* update htcondor

* Update use of HTcondor in appliance build

* Ensure tests are instanced and don't jumble relative paths + debug logging

* oops, update utilsTest too

* is this a pytest issue?

* Add some more log messages

* Fix time.sleep

* Remove the debug statement in docker

.gitlab-ci.yml

@@ -5,7 +5,7 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.11"
+  MAIN_PYTHON_PKG: "python3.12"
   # Used to tell pytest which tests to be run by specifying markers,
   # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
   # Currently specifies special tests that are not run by quick_test_offline.
@@ -67,8 +67,8 @@ lint:
   script:
     - pwd
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all]
-    - python3.11 -m pip freeze
-    - python3.11 --version
+    - ${MAIN_PYTHON_PKG} -m pip freeze
+    - ${MAIN_PYTHON_PKG} --version
     - make mypy
     - make docs
     # - make diff_pydocstyle_report
@@ -191,6 +191,12 @@ py310_main:
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 
 py311_appliance_build:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
+    - if: $CI_COMMIT_BRANCH =~ /.*3\.11.*/
   stage: basic_tests
   script:
     - pwd
@@ -201,14 +207,37 @@ py311_appliance_build:
 
 py311_main:
   rules:
-    - if: $CI_PIPELINE_SOURCE != "schedule"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
+    - if: $CI_COMMIT_BRANCH =~ /.*3\.11.*/
   stage: basic_tests
   script:
     - pwd
     - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.3'
     - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 
+py312_appliance_build:
+  stage: basic_tests
+  script:
+    - pwd
+    - python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 &&  pip install -U pip wheel && pip install -U build && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==23.6.1'
+    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
+    - python setup_gitlab_docker.py
+    - make push_docker
+
+py312_main:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
+  stage: basic_tests
+  script:
+    - pwd
+    - python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==23.6.1'
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
+    - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+
 slurm_test:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"

Makefile

@@ -135,11 +135,11 @@ clean_sdist:
 # Setting SET_OWNER_TAG will tag cloud resources so that UCSC's cloud murder bot won't kill them.
 test: check_venv check_build_reqs
 	TOIL_OWNER_TAG="shared" \
-	    python -m pytest --durations=0 --strict-markers --log-level DEBUG --log-cli-level INFO -r s $(cov) -n $(threads) --dist loadscope $(tests) -m "$(marker)" --color=yes
+	    python -m pytest --log-format="%(asctime)s %(levelname)s %(message)s" --durations=0 --strict-markers --log-level DEBUG --log-cli-level INFO -r s $(cov) -n $(threads) --dist loadscope $(tests) -m "$(marker)" --color=yes
 
 test_debug: check_venv check_build_reqs
 	TOIL_OWNER_TAG="$(whoami)" \
-	    python -m pytest --durations=0 --strict-markers --log-level DEBUG -s -o log_cli=true --log-cli-level DEBUG -r s $(tests) -m "$(marker)" --tb=native --maxfail=1
+	    python -m pytest --log-format="%(asctime)s %(levelname)s %(message)s" --durations=0 --strict-markers --log-level DEBUG -s -o log_cli=true --log-cli-level DEBUG -r s $(tests) -m "$(marker)" --tb=native --maxfail=1 --color=yes
 
 
 # This target will skip building docker and all docker based tests
@@ -148,12 +148,12 @@ test_offline: check_venv check_build_reqs
 	@printf "$(cyan)All docker related tests will be skipped.$(normal)\n"
 	TOIL_SKIP_DOCKER=True \
 	TOIL_SKIP_ONLINE=True \
-	    python -m pytest -vv --timeout=600 --strict-markers --log-level DEBUG --log-cli-level INFO $(cov) -n $(threads) --dist loadscope $(tests) -m "$(marker)"
+	    python -m pytest --log-format="%(asctime)s %(levelname)s %(message)s" -vv --timeout=600 --strict-markers --log-level DEBUG --log-cli-level INFO $(cov) -n $(threads) --dist loadscope $(tests) -m "$(marker)" --color=yes
 
 # This target will run about 1 minute of tests, and stop at the first failure
 test_1min: check_venv check_build_reqs
 	TOIL_SKIP_DOCKER=True \
-	    python -m pytest -vv --timeout=10 --strict-markers --log-level DEBUG --log-cli-level INFO --maxfail=1 src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::CWLWorkflowTest::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior -m "$(marker)"
+	    python -m pytest --log-format="%(asctime)s %(levelname)s %(message)s" -vv --timeout=10 --strict-markers --log-level DEBUG --log-cli-level INFO --maxfail=1 src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::CWLWorkflowTest::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior -m "$(marker)" --color=yes
 
 ifdef TOIL_DOCKER_REGISTRY
 

requirements-dev.txt

@@ -1,12 +1,11 @@
-pytest>=6.2.1,<9
+pytest>=6.2.1,<9,!=8.2.0
 pytest-cov>=2.12.1,<6
 pytest-timeout>=1.4.2,<3
 stubserver>=1.1,<2
 setuptools>=65.5.1,<70
 sphinx>=7,<8
-sphinx-autoapi>=2.1.1,<3
-# astroid 3 won't work until some sphinx-autoapi release after 2.1.1
-astroid>=2.15,<3
+sphinx-autoapi>=3,<4
+astroid>=3,<4
 sphinx-autodoc-typehints>=1.24.0,<3
 sphinxcontrib-autoprogram==0.1.9
 cwltest>=2.2.20211116163652

src/toil/test/utils/utilsTest.py

@@ -56,6 +56,7 @@ def setUp(self):
         super().setUp()
         self.tempDir = self._createTempDir()
         self.tempFile = get_temp_file(rootDir=self.tempDir)
+        self.outputFile = get_temp_file(rootDir=self.tempDir)
         self.outputFile = 'someSortedStuff.txt'
         self.toilDir = os.path.join(self.tempDir, "jobstore")
         self.assertFalse(os.path.exists(self.toilDir))
@@ -73,9 +74,9 @@ def setUp(self):
             '-m',
             'toil.test.sort.sort',
             f'file:{self.toilDir}',
+            f'--fileToSort={self.tempFile}',
+            f'--outputFile={self.outputFile}',
             '--clean=never',
-            '--numLines=1',
-            '--lineLength=1'
         ]
 
         self.restart_sort_workflow_cmd = [
@@ -91,7 +92,7 @@ def tearDown(self):
         if os.path.exists(self.toilDir):
             shutil.rmtree(self.toilDir)
 
-        for f in ['fileToSort.txt', 'sortedFile.txt', 'output.txt']:
+        for f in [self.tempFile, self.outputFile, os.path.join(self.tempDir, "output.txt")]:
             if os.path.exists(f):
                 os.remove(f)
 
@@ -313,8 +314,20 @@ def check_status(self, status, status_fn, seconds=20):
 
     def testGetPIDStatus(self):
         """Test that ToilStatus.getPIDStatus() behaves as expected."""
+        print(f"I am {self.toilDir}")
+        logger.info(f"I am {self.toilDir}")
         wf = subprocess.Popen(self.sort_workflow_cmd)
-        self.check_status('RUNNING', status_fn=ToilStatus.getPIDStatus, seconds=20)
+
+        for t in range(0, 6):
+            if os.path.exists(self.toilDir):
+                logger.error(f"I am {self.toilDir} and my jobstore exists")
+            else:
+                logger.error(f"I am {self.toilDir} and my jobstore DOES NOT exists")
+            time.sleep(5)
+            logger.critical(f"PID status: {ToilStatus.getPIDStatus(self.toilDir)}")
+            logger.info(f"Slept for {t*5}")
+
+        self.check_status('RUNNING', status_fn=ToilStatus.getPIDStatus, seconds=60)
         wf.wait()
         self.check_status('COMPLETED', status_fn=ToilStatus.getPIDStatus)
 
@@ -330,7 +343,19 @@ def testGetStatusFailedToilWF(self):
         opportunity to test the 'RUNNING' functionality of getStatus().
         """
         # --badWorker is set to force failure.
+        print(f"I am {self.toilDir}")
+        logger.info(f"I am {self.toilDir}")
         wf = subprocess.Popen(self.sort_workflow_cmd + ['--badWorker=1'])
+
+        for t in range(0, 6):
+            if os.path.exists(self.toilDir):
+                logger.error(f"I am {self.toilDir} and my jobstore exists")
+            else:
+                logger.error(f"I am {self.toilDir} and my jobstore DOES NOT exists")
+            time.sleep(5)
+            logger.critical(f"PID status: {ToilStatus.getStatus(self.toilDir)}")
+            logger.info(f"Slept for {t*5}")
+
         self.check_status('RUNNING', status_fn=ToilStatus.getStatus)
         wf.wait()
         self.check_status('ERROR', status_fn=ToilStatus.getStatus)
@@ -340,9 +365,19 @@ def testGetStatusFailedToilWF(self):
     def testGetStatusFailedCWLWF(self):
         """Test that ToilStatus.getStatus() behaves as expected with a failing CWL workflow."""
         # --badWorker is set to force failure.
+        print(f"I am {self.toilDir}")
+        logger.info(f"I am {self.toilDir}")
         cmd = ['toil-cwl-runner', '--jobStore', self.toilDir, '--clean=never', '--badWorker=1',
-               'src/toil/test/cwl/sorttool.cwl', '--reverse', '--input', 'src/toil/test/cwl/whale.txt']
+               'src/toil/test/cwl/sorttool.cwl', '--reverse', '--input', 'src/toil/test/cwl/whale.txt', f'--outdir={self.tempDir}']
         wf = subprocess.Popen(cmd)
+        for t in range(0, 6):
+            if os.path.exists(self.toilDir):
+                logger.error(f"I am {self.toilDir} and my jobstore exists")
+            else:
+                logger.error(f"I am {self.toilDir} and my jobstore DOES NOT exists")
+            time.sleep(5)
+            logger.critical(f"PID status: {ToilStatus.getStatus(self.toilDir)}")
+            logger.info(f"Slept for {t*5}")
         self.check_status('RUNNING', status_fn=ToilStatus.getStatus)
         wf.wait()
         self.check_status('ERROR', status_fn=ToilStatus.getStatus)
@@ -351,9 +386,19 @@ def testGetStatusFailedCWLWF(self):
     @needs_docker
     def testGetStatusSuccessfulCWLWF(self):
         """Test that ToilStatus.getStatus() behaves as expected with a successful CWL workflow."""
+        print(f"I am {self.toilDir}")
+        logger.info(f"I am {self.toilDir}")
         cmd = ['toil-cwl-runner', '--jobStore', self.toilDir, '--clean=never',
-               'src/toil/test/cwl/sorttool.cwl', '--reverse', '--input', 'src/toil/test/cwl/whale.txt']
+               'src/toil/test/cwl/sorttool.cwl', '--reverse', '--input', 'src/toil/test/cwl/whale.txt', f'--outdir={self.tempDir}']
         wf = subprocess.Popen(cmd)
+        for t in range(0, 6):
+            if os.path.exists(self.toilDir):
+                logger.error(f"I am {self.toilDir} and my jobstore exists")
+            else:
+                logger.error(f"I am {self.toilDir} and my jobstore DOES NOT exists")
+            time.sleep(5)
+            logger.critical(f"PID status: {ToilStatus.getStatus(self.toilDir)}")
+            logger.info(f"Slept for {t*5}")
         self.check_status('RUNNING', status_fn=ToilStatus.getStatus, seconds=20)
         wf.wait()
         self.check_status('COMPLETED', status_fn=ToilStatus.getStatus)