fixed overflow errors when processing of large fld files (more than 2…

….5GB)

GPR_func/read_fld.py

@@ -6,8 +6,9 @@
 from rasterio.transform import from_origin
 
 from cpp_fld_to_npy import fld_to_npy
+from GUI.error_handling import confirm_python_processing
 
-def define_fld_parameters_cpp(file_path, overwrite=False):
+def define_fld_parameters_cpp(file_path, root=None, w1=None, w2=None, overwrite=False):
     base_name = os.path.splitext(file_path)[0]  # Get the path without the extension
     output_npy = base_name + '.npy'
 
@@ -16,18 +17,55 @@ def define_fld_parameters_cpp(file_path, overwrite=False):
 
     xpixels, ypixels, zpixels, pixelsize, x_coor, y_coor, pixelsize_z = read_fld_header(file_content)
     data_start, data_type, pixelsize_z, depth_table, time_table, _ = read_fld_data_specs(file_content, zpixels)
+
+    expected_filesize = get_expected_npy_size(xpixels, ypixels, zpixels)
+
     if not os.path.exists(output_npy):
-        fld_to_npy.process_fld_with_cpp(file_path, output_npy)
+        if expected_filesize > 7000000000:
+            response = confirm_python_processing(root, expected_filesize)
+            root.lift()
+            w1.lift()
+            w2.lift()
+            if response:
+                start_bits, stop_bits, number_of_values = get_start_stop_bits(file_content, zpixels, data_start)
+                read_fld_with_threads(file_content, xpixels, ypixels, zpixels, start_bits, stop_bits,
+                                  number_of_values, output_npy)
+            else:
+                print('User chose not to continue, aborting processing')
+                return None, None, None, None, None, None, None, None, None, None, None
+        else:
+            fld_to_npy.process_fld_with_cpp(file_path, output_npy)
     else:
         if overwrite:
-            fld_to_npy.process_fld_with_cpp(file_path, output_npy)
+            if expected_filesize > 7000000000:
+                response = confirm_python_processing(root, expected_filesize)
+                w1.lift()
+                w2.lift()
+                if response:
+                    start_bits, stop_bits, number_of_values = get_start_stop_bits(file_content, zpixels, data_start)
+                    read_fld_with_threads(file_content, xpixels, ypixels, zpixels, start_bits, stop_bits,
+                                          number_of_values, output_npy)
+                else:
+                    print('User chose not to continue, aborting processing')
+            else:
+                fld_to_npy.process_fld_with_cpp(file_path, output_npy)
+            print('Fld preprocesed')
         else:
             print(f"File {output_npy} already exists. Skipping...")
 
     fld_data = np.load(output_npy, mmap_mode='r')
 
     return fld_data, xpixels, ypixels, zpixels, pixelsize, x_coor, y_coor, pixelsize_z, data_type, depth_table, time_table
 
+def get_expected_npy_size(x, y, z):
+    x = np.int64(x)
+    y = np.int64(y)
+    z = np.int64(z)
+
+    exp_size = ((x * y * z) * 4) + 128
+
+    return exp_size
+
 def define_fld_parameters(file_path, overwrite=False):
     base_name = os.path.splitext(file_path)[0]
     output_npy = base_name + '.npy'
@@ -125,7 +163,7 @@ def get_start_stop_bits(file_content, number_of_layers, data_start):
         number_of_values.append(number_of_values_layer)
 
         # Determine the end position of the layer data
-        data_stop = data_start + 16 + number_of_values_layer * 2
+        data_stop = np.int64(data_start + 16 + number_of_values_layer * 2)
 
         start_bits.append(data_start)
         stop_bits.append(data_stop)
@@ -175,7 +213,8 @@ def read_fld_with_threads(file_content, x_size, y_size, number_of_layers, start_
     return data
 
 def create_depthslice_images(vmin, vmax, npy_file_path, zpixels, pixelsize_z, pixelsize_x, x_coor, y_coor, depth_table, time_table, data_type, z_vals):
-    pixelsize_z = int(pixelsize_z * 100)
+    pixelsize_z = round(pixelsize_z * 100)
+
     radar_data = np.load(npy_file_path)
     npy_base_name = os.path.splitext(os.path.basename(npy_file_path))[0]
     working_directory = os.path.dirname(npy_file_path)

GUI/EditProjectFile.py

@@ -27,6 +27,7 @@ def __init__(self, master, frame_left):
         self.section_data = {}
         self.project_open = False
 
+        self.master = master
         self.bold_font = font.Font(weight="bold", size=10)
         self.create_window(master)
 
@@ -66,11 +67,11 @@ def create_window(self, root):
         open_project_button = Button(file_command_frame, text='Open project', command=self.open_file)
         open_project_button.pack(side=LEFT, padx=5)
 
-        save_project_button = Button(file_command_frame, text='Save project', command=self.save_file)
-        save_project_button.pack(side=LEFT, padx=5)
+        self.save_project_button = Button(file_command_frame, text='Save project', state=DISABLED, command=self.save_file)
+        self.save_project_button.pack(side=LEFT, padx=5)
 
-        button_add = Button(file_command_frame, text='Add dataset to project', command=self.add_dataset_to_project)
-        button_add.pack(side=LEFT, padx=5)
+        self.button_add = Button(file_command_frame, text='Add dataset to project', state=DISABLED, command=self.add_dataset_to_project)
+        self.button_add.pack(side=LEFT, padx=5)
 
         exit_button = Button(file_command_frame, text='Exit', command=self.exit)
         exit_button.pack(side=LEFT, padx=5)
@@ -330,6 +331,11 @@ def open_file(self, file_path=None):
             self.editor_canvas.config(scrollregion=self.editor_canvas.bbox("all"))
 
             self.project_open = True
+            self.activate_buttons()
+
+    def activate_buttons(self):
+        self.button_add.config(state=NORMAL)
+        self.save_project_button.config(state=NORMAL)
 
     def add_dataset_to_project(self):
         # Open a file dialog to let the user select an .fld or .ap_prj file
@@ -565,7 +571,10 @@ def process_files():
 
                 # Use the compiled executable function if not containing "DTM" and the compiled exe is to be used
                 if self.use_compiled_exe:
-                    _, _, _, self.zpixels, _, _, _, self.pixelsize_z, _, _, _ = read_fld.define_fld_parameters_cpp(entry, overwrite=True)
+                    _, _, _, self.zpixels, _, _, _, self.pixelsize_z, _, _, _ = read_fld.define_fld_parameters_cpp(entry, self.master, self.results_window, progress_window, overwrite=True)
+                    if self.zpixels == None:
+                        progress_window.destroy()
+                        return
                 else:
                     _, _, _, self.zpixels, _, _, _, self.pixelsize_z, _, _, _ = read_fld.define_fld_parameters(entry, overwrite=True)
             # Close the progress bar window after processing
@@ -624,7 +633,7 @@ def check_depthslice_images(self):
                         depthslice_folder_path = 'None'
 
                     if depthslice_folder_path != 'None':
-                        missing_images = self.check_ds_folder(fld_file_path, zpixels, int(pixelsize_z * 100),
+                        missing_images = self.check_ds_folder(fld_file_path, zpixels, round(pixelsize_z * 100),
                                                           depthslice_folder_path, data_type)
                         filesize = self.check_ds_images_size(depthslice_folder_path, fld_file_path, xpixels, ypixels)
                         if not filesize: missing_images.append('filesize')
@@ -644,7 +653,6 @@ def check_depthslice_images(self):
                         "data_type": data_type,
                         "z_vals": z_vals
                     }
-
         # Call the function to display the results
         self.display_depthslice_check_results(check_results)
 

GUI/error_handling.py

@@ -1,5 +1,5 @@
 import tkinter as tk
-from tkinter import PhotoImage
+from tkinter import PhotoImage, messagebox
 import platform
 
 def show_error_dialog(message):
@@ -19,3 +19,15 @@ def show_error_dialog(message):
     # Add an OK button
     ok_button = tk.Button(error_dialog, text="OK", command=error_dialog.destroy)
     ok_button.pack(pady=(0, 10))
+
+def confirm_python_processing(root, expected_size):
+    root = root
+
+    expected_size = str(round((expected_size / 1024**2), 2))
+
+    message = f"The expected npy-file size is {expected_size}MB, the program can only use Python code to process the file, which is significantly slower (the processing might take several minutes). Do you want to continue?"
+
+    response = messagebox.askyesno("Processing Confirmation", message)
+
+    # The response will be True if the user clicks 'Yes' and False if the user clicks 'No'.
+    return response

cpp_fld_to_npy/fld_to_npy.cpp

@@ -12,26 +12,28 @@
 void saveDataAsNpy(const std::vector<std::vector<std::vector<float>>>& data,
     const char* output_file_path) {
 
-    int z_size = data.size();
+    unsigned long long z_size = data.size();
     if (z_size == 0) {
         std::cerr << "Error: Empty data." << std::endl;
         return;
     }
 
-    int y_size = data[0].size();
+    unsigned long long y_size = data[0].size();
     if (y_size == 0) {
         std::cerr << "Error: Empty data." << std::endl;
         return;
     }
 
-    int x_size = data[0][0].size();
+    unsigned long long x_size = data[0][0].size();
     if (x_size == 0) {
         std::cerr << "Error: Empty data." << std::endl;
         return;
     }
 
+    static std::vector<uint64_t> const shape{ z_size, y_size, x_size };
+
     // Flatten the data for cnpy
-    std::vector<float> flattened_data;
+    std::vector<float_t> flattened_data;
     flattened_data.reserve(x_size * y_size * z_size);
 
     for (int layer_index = 0; layer_index < z_size; ++layer_index) {
@@ -42,13 +44,8 @@ void saveDataAsNpy(const std::vector<std::vector<std::vector<float>>>& data,
         }
     }
 
-    // Convert dimensions to unsigned long long for cnpy
-    unsigned long long ull_x_size = static_cast<unsigned long long>(x_size);
-    unsigned long long ull_y_size = static_cast<unsigned long long>(y_size);
-    unsigned long long ull_z_size = static_cast<unsigned long long>(z_size);
-
     // Save the flattened data as npy
-    cnpy::npy_save(output_file_path, &flattened_data[0], { ull_z_size, ull_y_size, ull_x_size }, "w");
+    cnpy::npy_save(output_file_path, &flattened_data[0], shape, "w");
 }
 
 
@@ -65,7 +62,7 @@ double readBigEndianDouble(std::ifstream& file) {
 
 
 // Function to read and interpret a 4-byte integer as big-endian
-int32_t readBigEndianInt32(std::ifstream & file) {
+int32_t readBigEndianInt32(std::ifstream& file) {
     int32_t value;
     char rawValue[4];
     file.read(rawValue, 4);
@@ -151,7 +148,7 @@ int readFldDataSpecs(std::ifstream& file, int z_size) {
 
     int start_velo;
     start_velo = stop_b2 + num_dt * 4 * 2;
-    
+
     file.seekg(start_velo); // Skip the time_depth_table
 
     int num_velo;
@@ -167,9 +164,10 @@ int readFldDataSpecs(std::ifstream& file, int z_size) {
 
 // Function to read the FLD data
 void readFldData(std::ifstream& file, int x_size, int y_size, int number_of_layers, std::vector<std::vector<std::vector<float>>>& data) {
-    int data_start = readFldDataSpecs(file, number_of_layers);
+    int64_t data_start = readFldDataSpecs(file, number_of_layers);
 
     for (int layer_index = 0; layer_index < number_of_layers; ++layer_index) {
+        std::cout << "Layer index loaded: " << layer_index << std::endl;
         int64_t number_of_values_layer;
         file.seekg(data_start);
         file.read(reinterpret_cast<char*>(&number_of_values_layer), sizeof(int64_t));
@@ -178,7 +176,7 @@ void readFldData(std::ifstream& file, int x_size, int y_size, int number_of_laye
         min1 = readBigEndianFloat(file);
         max1 = readBigEndianFloat(file);
 
-        int data_stop = data_start + 16 + number_of_values_layer * 2;
+        int64_t data_stop = data_start + 16 + number_of_values_layer * 2;
 
         std::vector<int16_t> data_values(number_of_values_layer);
         file.seekg(data_start + 16);
@@ -241,7 +239,7 @@ int main(int argc, char* argv[]) {
     readFldHeader(file, version, xpixels, ypixels, zpixels, pixelsize, x_coor, y_coor, pixelsize_z, x_start, x_end, z_start, z_end);
 
     // Declare and initialize the 'data' vector
-    std::vector<std::vector<std::vector<float>>> data(zpixels, std::vector<std::vector<float>>(ypixels, std::vector<float>(xpixels, 0.0f)));
+    std::vector<std::vector<std::vector<float_t>>> data(zpixels, std::vector<std::vector<float>>(ypixels, std::vector<float>(xpixels, 0.0f)));
 
     auto start_time = std::chrono::high_resolution_clock::now(); // Get the start time
     readFldData(file, xpixels, ypixels, zpixels, data); // Pass 'data' as an argument
@@ -260,8 +258,4 @@ int main(int argc, char* argv[]) {
     saveDataAsNpy(data, output_file_path);
 
     return 0;
-}
-
-
-
-
+}