From 169b9b94c6d9b59f3d0bcd2048afe5b6985c377e Mon Sep 17 00:00:00 2001 From: jhalbauer Date: Wed, 6 Mar 2024 10:21:47 +0100 Subject: [PATCH 1/3] added function for extracting files using zipfile_deflate64 --- .flake8 | 2 +- pyproject.toml | 1 + .../open_geodata_germany/download_data.py | 31 +++++++++++++++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.flake8 b/.flake8 index 48303a8..fca63d5 100644 --- a/.flake8 +++ b/.flake8 @@ -14,7 +14,7 @@ per-file-ignores = ./src/grass_gis_helpers/general.py: F821 ./src/grass_gis_helpers/location.py: F821 ./src/grass_gis_helpers/mapset.py: F821 - ./src/grass_gis_helpers/open_geodata_germany/download_data.py: F821 + ./src/grass_gis_helpers/open_geodata_germany/download_data.py: F821,E501 ./src/grass_gis_helpers/open_geodata_germany/federal_state.py: F821 ./src/grass_gis_helpers/parallel.py: F821 ./src/grass_gis_helpers/tiling.py: F821 diff --git a/pyproject.toml b/pyproject.toml index ac75fa5..56ee344 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ ] dependencies = [ "requests", + "zipfile_deflate64", ] [project.urls] diff --git a/src/grass_gis_helpers/open_geodata_germany/download_data.py b/src/grass_gis_helpers/open_geodata_germany/download_data.py index ad33983..898c06b 100644 --- a/src/grass_gis_helpers/open_geodata_germany/download_data.py +++ b/src/grass_gis_helpers/open_geodata_germany/download_data.py @@ -25,6 +25,7 @@ from multiprocessing.pool import ThreadPool import os import requests +import zipfile_deflate64 from zipfile import ZipFile import grass.script as grass @@ -114,6 +115,28 @@ def extract_compressed_files(file_names, download_dir): return extracted_files +def extract_compressed_files_deflate64(file_names, download_dir): + """Extract compressed files to download directory using the zipfile_deflate64 library. + + Args: + file_names (list): List with compressed e.g. zip file names which + are stored in the download_dir + download_dir (str): Path to directory where the data should be + downloaded + Returns: + extracted_files (list): List with extracted files + """ + extracted_files = [] + for file_name in file_names: + file = os.path.join(download_dir, file_name) + with zipfile_deflate64.ZipFile(file, "r") as zipObj: + zip_content = zipObj.namelist() + # Extract all the contents of zip file in current directory + zipObj.extractall(download_dir) + extracted_files.extend(zip_content) + return extracted_files + + def fix_corrupted_data(file): """Fix corrupted XYZ/TXT data file e.g. for Berlin DOMs Args: @@ -121,15 +144,11 @@ def fix_corrupted_data(file): """ # remove corrupted data from TXT DOM files if not os.path.exists(f"{file}.bak"): - with fileinput.FileInput( - file, inplace=True, backup=".bak" - ) as file_object: + with fileinput.FileInput(file, inplace=True, backup=".bak") as file_object: for line in file_object: # two times replace of white spaces, since some lines contain # 3 spaces print( - line.replace(" ", " ") - .replace(" ", " ") - .replace("\t", " "), + line.replace(" ", " ").replace(" ", " ").replace("\t", " "), end="", ) From f0da8801f0c6cf6d901bf0bc57166af40a8cae8f Mon Sep 17 00:00:00 2001 From: jhalbauer Date: Wed, 6 Mar 2024 10:31:16 +0100 Subject: [PATCH 2/3] fixed black linting issue --- .../open_geodata_germany/download_data.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/grass_gis_helpers/open_geodata_germany/download_data.py b/src/grass_gis_helpers/open_geodata_germany/download_data.py index 898c06b..2d81af4 100644 --- a/src/grass_gis_helpers/open_geodata_germany/download_data.py +++ b/src/grass_gis_helpers/open_geodata_germany/download_data.py @@ -144,11 +144,15 @@ def fix_corrupted_data(file): """ # remove corrupted data from TXT DOM files if not os.path.exists(f"{file}.bak"): - with fileinput.FileInput(file, inplace=True, backup=".bak") as file_object: + with fileinput.FileInput( + file, inplace=True, backup=".bak" + ) as file_object: for line in file_object: # two times replace of white spaces, since some lines contain # 3 spaces print( - line.replace(" ", " ").replace(" ", " ").replace("\t", " "), + line.replace(" ", " ") + .replace(" ", " ") + .replace("\t", " "), end="", ) From 09a897d58b3362867aee0288570d7ac32bd488a7 Mon Sep 17 00:00:00 2001 From: jhalbauer Date: Wed, 6 Mar 2024 14:00:50 +0100 Subject: [PATCH 3/3] changed linting and typo --- .flake8 | 2 +- src/grass_gis_helpers/open_geodata_germany/download_data.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.flake8 b/.flake8 index fca63d5..48303a8 100644 --- a/.flake8 +++ b/.flake8 @@ -14,7 +14,7 @@ per-file-ignores = ./src/grass_gis_helpers/general.py: F821 ./src/grass_gis_helpers/location.py: F821 ./src/grass_gis_helpers/mapset.py: F821 - ./src/grass_gis_helpers/open_geodata_germany/download_data.py: F821,E501 + ./src/grass_gis_helpers/open_geodata_germany/download_data.py: F821 ./src/grass_gis_helpers/open_geodata_germany/federal_state.py: F821 ./src/grass_gis_helpers/parallel.py: F821 ./src/grass_gis_helpers/tiling.py: F821 diff --git a/src/grass_gis_helpers/open_geodata_germany/download_data.py b/src/grass_gis_helpers/open_geodata_germany/download_data.py index 2d81af4..e6f8c2c 100644 --- a/src/grass_gis_helpers/open_geodata_germany/download_data.py +++ b/src/grass_gis_helpers/open_geodata_germany/download_data.py @@ -58,7 +58,7 @@ def check_download_dir(download_dir): def url_response(url): """URL response function which is used by download_data_using_threadpool - Arsg: + Args: url (str): data download url Return: url (str): Return the url for printing @@ -116,7 +116,8 @@ def extract_compressed_files(file_names, download_dir): def extract_compressed_files_deflate64(file_names, download_dir): - """Extract compressed files to download directory using the zipfile_deflate64 library. + """Extract compressed files to download directory using the + zipfile_deflate64 library. Args: file_names (list): List with compressed e.g. zip file names which @@ -139,6 +140,7 @@ def extract_compressed_files_deflate64(file_names, download_dir): def fix_corrupted_data(file): """Fix corrupted XYZ/TXT data file e.g. for Berlin DOMs + Args: file (str): XYZ or TXT data file with corrupted data """