From a355016c001fd174a154b1e1c126bf1b881f0103 Mon Sep 17 00:00:00 2001 From: dbouget Date: Thu, 30 Jan 2025 15:25:06 +0100 Subject: [PATCH 1/4] Added flag for skipping already converted files by default, and flag for customizing which file type to look for. --- .gitignore | 5 ++++- vsi2tif/src/process.py | 40 ++++++++++++++++++++++++++++++++-------- vsi2tif/vsi2tif.py | 16 ++++++++++++++-- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 9bca4d1..decda1f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,7 @@ *.vsi *.zip *bftools/ -*.ets \ No newline at end of file +*.ets +*.xml +*.iml +*.idea/ \ No newline at end of file diff --git a/vsi2tif/src/process.py b/vsi2tif/src/process.py index 139b529..9af2856 100644 --- a/vsi2tif/src/process.py +++ b/vsi2tif/src/process.py @@ -19,10 +19,15 @@ def cellsens2tif_single( plane: int = 0, quality: int = 85, max_mem: int = 32, + skip_converted: bool = True, verbose: int = 1, ) -> None: if int(plane) == -1: + if os.path.exists(output_path) and skip_converted: + logging.info("Skipping already converted slide: {}".format(output_path)) + return + image_folder = os.path.join(os.path.dirname(output_path), os.path.basename(output_path).replace(".tif", "")) for s in range(50): try: @@ -49,7 +54,10 @@ def cellsens2tif_single( logging.error("Issue cleaning up after all planes conversion.") logging.error(traceback.format_exc()) else: - cellsens2tif(input_path, output_path, bfconvert, compression, tz, plane, quality, max_mem, verbose) + if os.path.exists(output_path) and not skip_converted: + cellsens2tif(input_path, output_path, bfconvert, compression, tz, plane, quality, max_mem, verbose) + elif skip_converted: + logging.info("Skipping already converted slide: {}".format(output_path)) @benchmark @@ -62,19 +70,22 @@ def cellsens2tif_batch( plane: int = 0, quality: int = 85, max_mem: int = 32, - verbose: int = 1, remove_name_spaces: bool = False, + skip_converted: bool = True, + extension_type: str = ".vsi", + verbose: int = 1, ) -> None: # create directory if it does not exist os.makedirs(output_path, exist_ok=True) - # find path to all cellSens VSI images to convert + # find path to all cellSens images to convert paths = [] for root, _, files in os.walk(input_path): for file in files: - if file.lower().endswith("overview.vsi"): + if "overview" in file.lower(): + logging.info("Skipping overview file: {}".format(file)) continue - if file.endswith(".vsi"): + if file.endswith(extension_type): paths.append((root, file)) # perform conversion in separate processes @@ -82,14 +93,27 @@ def cellsens2tif_batch( curr_input_path = os.path.join(root, file) if remove_name_spaces: curr_output_path = ( - (output_path + "/" + curr_input_path.split(input_path)[-1]).replace(" ", "_").replace(".vsi", ".tif") + (output_path + "/" + curr_input_path.split(input_path)[-1]) + .replace(" ", "_") + .replace(extension_type, ".tif") ) else: - curr_output_path = (output_path + "/" + curr_input_path.split(input_path)[-1]).replace(".vsi", ".tif") + curr_output_path = (output_path + "/" + curr_input_path.split(input_path)[-1]).replace( + extension_type, ".tif" + ) try: cellsens2tif_single( - curr_input_path, curr_output_path, bfconvert, compression, tz, plane, quality, max_mem, verbose + curr_input_path, + curr_output_path, + bfconvert, + compression, + tz, + plane, + quality, + max_mem, + skip_converted, + verbose, ) except Exception: continue diff --git a/vsi2tif/vsi2tif.py b/vsi2tif/vsi2tif.py index e96b84a..1718309 100644 --- a/vsi2tif/vsi2tif.py +++ b/vsi2tif/vsi2tif.py @@ -27,7 +27,9 @@ def main(): parser.add_argument("-m", "--max-mem", help="set maximum memory in the java vm - default 32", default=32) parser.add_argument("-v", "--verbose", help="set verbosity level - default 1", default=1, type=int) parser.add_argument( - "--remove-name-spaces", help="replace spaces in filename with underscores in batch mode", action="store_true" + "--remove-name-spaces", + help="replace spaces in filename with underscores in batch mode", + action="store_true", ) parser.add_argument( "-p", @@ -37,7 +39,14 @@ def main(): default=0, type=int, ) + parser.add_argument( + "--noskip-converted", + help="To specifically request existing files to be converted again", + action="store_true", + ) + parser.add_argument("-f", "--extension", help="extension type to consider (e.g., .vsi)", default=".vsi", type=str) argv = parser.parse_args() + skip_converted = not argv.noskip_converted if argv.verbose not in list(range(6)): raise ValueError("Verbosity level must be an integer between 0 and 5") @@ -60,8 +69,10 @@ def main(): argv.plane, argv.quality, argv.max_mem, - argv.verbose, argv.remove_name_spaces, + skip_converted, + argv.extension, + argv.verbose, ) else: logging.info("Performing single conversion...") @@ -74,6 +85,7 @@ def main(): argv.plane, argv.quality, argv.max_mem, + skip_converted, argv.verbose, ) From bd470f03f42cfbac29bbbcd8139b8e911ad331f8 Mon Sep 17 00:00:00 2001 From: dbouget Date: Thu, 30 Jan 2025 16:12:22 +0100 Subject: [PATCH 2/4] CIs version update --- .github/workflows/build.yml | 4 ++-- .github/workflows/integration_test.yml | 4 ++-- .github/workflows/release.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 773e747..d4f5a69 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,7 @@ jobs: run: python setup.py bdist_wheel - name: Upload Python wheel - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "Python wheel" path: ./dist/* @@ -52,7 +52,7 @@ jobs: run: pip install wheel setuptools - name: Download artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: "Python wheel" diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml index 7477501..a46dae9 100644 --- a/.github/workflows/integration_test.yml +++ b/.github/workflows/integration_test.yml @@ -26,7 +26,7 @@ jobs: run: python setup.py bdist_wheel - name: Upload Python wheel - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "Python wheel" path: ./dist/* @@ -52,7 +52,7 @@ jobs: run: pip install wheel setuptools - name: Download artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: "Python wheel" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40970e4..95a56de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: run: python setup.py bdist_wheel - name: Upload Python wheel - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "Python wheel" path: ./dist/* From 159cd6061329a3ec356fb8c00796cc84b3f0d259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Pedersen?= Date: Thu, 30 Jan 2025 18:25:09 +0100 Subject: [PATCH 3/4] Update vsi2tif/src/process.py --- vsi2tif/src/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vsi2tif/src/process.py b/vsi2tif/src/process.py index 9af2856..5d78e63 100644 --- a/vsi2tif/src/process.py +++ b/vsi2tif/src/process.py @@ -25,7 +25,7 @@ def cellsens2tif_single( if int(plane) == -1: if os.path.exists(output_path) and skip_converted: - logging.info("Skipping already converted slide: {}".format(output_path)) + logging.info(f"Skipping already converted slide: {output_path}") return image_folder = os.path.join(os.path.dirname(output_path), os.path.basename(output_path).replace(".tif", "")) From 124744c190a4bdb1b43940a7f94d4460ece69166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Pedersen?= Date: Thu, 30 Jan 2025 18:25:15 +0100 Subject: [PATCH 4/4] Update vsi2tif/src/process.py --- vsi2tif/src/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vsi2tif/src/process.py b/vsi2tif/src/process.py index 5d78e63..e7793d4 100644 --- a/vsi2tif/src/process.py +++ b/vsi2tif/src/process.py @@ -57,7 +57,7 @@ def cellsens2tif_single( if os.path.exists(output_path) and not skip_converted: cellsens2tif(input_path, output_path, bfconvert, compression, tz, plane, quality, max_mem, verbose) elif skip_converted: - logging.info("Skipping already converted slide: {}".format(output_path)) + logging.info(f"Skipping already converted slide: {output_path}") @benchmark