File picker: allow parallel traversal w/o sorting

Motivated by the discussion in helix-editor#10995. Sequential traversal is usually
very fast, unless it's Windows, or the files are on a (slow) remotely
mounted FS.

The existing logic of *not* moving injection of file names to a
background thread is preserved: if a directory can be traversed
in 30ms, all the work happens on the same thread sequentially.

To use `WalkParallel`, we need to give up on file name sorting, so
add a file picker option for that (defaults to sorting enabled to
not change the current behaviour). We _could_ make sorting work
for parallel traversal as well, but that requires eagerly traversing
the file tree and fully realizing the collection of file paths
in memory (so that we can merge partial results from independent
workers). It looks like lazy evaluation is preferred and allows
us to stop the work early (e.g. if the file picker window is closed).

Closes helix-editor#11021.

helix-term/src/ui/mod.rs

@@ -23,7 +23,7 @@ use helix_stdx::rope;
 use helix_view::theme::Style;
 pub use markdown::Markdown;
 pub use menu::Menu;
-pub use picker::{Column as PickerColumn, FileLocation, Picker};
+pub use picker::{Column as PickerColumn, FileLocation, Injector, Picker};
 pub use popup::Popup;
 pub use prompt::{Prompt, PromptEvent};
 pub use spinner::{ProgressSpinners, Spinner};
@@ -189,9 +189,6 @@ type FilePicker = Picker<PathBuf, PathBuf>;
 
 pub fn file_picker(root: PathBuf, config: &helix_view::editor::Config) -> FilePicker {
     use ignore::{types::TypesBuilder, WalkBuilder};
-    use std::time::Instant;
-
-    let now = Instant::now();
 
     let dedup_symlinks = config.file_picker.deduplicate_links;
     let absolute_root = root.canonicalize().unwrap_or_else(|_| root.clone());
@@ -205,12 +202,14 @@ pub fn file_picker(root: PathBuf, config: &helix_view::editor::Config) -> FilePi
         .git_ignore(config.file_picker.git_ignore)
         .git_global(config.file_picker.git_global)
         .git_exclude(config.file_picker.git_exclude)
-        .sort_by_file_name(|name1, name2| name1.cmp(name2))
         .max_depth(config.file_picker.max_depth)
         .filter_entry(move |entry| filter_picker_entry(entry, &absolute_root, dedup_symlinks));
 
     walk_builder.add_custom_ignore_filename(helix_loader::config_dir().join("ignore"));
     walk_builder.add_custom_ignore_filename(".helix/ignore");
+    if config.file_picker.sorted {
+        walk_builder.sort_by_file_name(|name1, name2| name1.cmp(name2));
+    }
 
     // We want to exclude files that the editor can't handle yet
     let mut type_builder = TypesBuilder::new();
@@ -225,14 +224,6 @@ pub fn file_picker(root: PathBuf, config: &helix_view::editor::Config) -> FilePi
         .build()
         .expect("failed to build excluded_types");
     walk_builder.types(excluded_types);
-    let mut files = walk_builder.build().filter_map(|entry| {
-        let entry = entry.ok()?;
-        if !entry.file_type()?.is_file() {
-            return None;
-        }
-        Some(entry.into_path())
-    });
-    log::debug!("file_picker init {:?}", Instant::now().duration_since(now));
 
     let columns = [PickerColumn::new(
         "path",
@@ -254,11 +245,37 @@ pub fn file_picker(root: PathBuf, config: &helix_view::editor::Config) -> FilePi
         }
     })
     .with_preview(|_editor, path| Some((path.as_path().into(), None)));
-    let injector = picker.injector();
-    let timeout = std::time::Instant::now() + std::time::Duration::from_millis(30);
+    inject_files(picker.injector(), walk_builder, config.file_picker.sorted);
+
+    picker
+}
+
+fn inject_files(
+    injector: Injector<PathBuf, PathBuf>,
+    mut walk_builder: ignore::WalkBuilder,
+    sorted: bool,
+) {
+    use ignore::WalkState;
+    use std::collections::HashSet;
+    use std::time::{Duration, Instant};
+
+    // How long we are willing to wait for results before displaying the file picker.
+    let now = Instant::now();
+    let timeout = now + Duration::from_millis(30);
+
+    // Keep the track of injected files to prevent duplicates.
+    let mut injected_files = HashSet::new();
+    let mut files_iter = walk_builder.build().filter_map(|entry| {
+        let entry = entry.ok()?;
+        if !entry.file_type()?.is_file() {
+            return None;
+        }
+        Some(entry.into_path())
+    });
 
     let mut hit_timeout = false;
-    for file in &mut files {
+    for file in &mut files_iter {
+        injected_files.insert(file.clone());
         if injector.push(file).is_err() {
             break;
         }
@@ -268,15 +285,49 @@ pub fn file_picker(root: PathBuf, config: &helix_view::editor::Config) -> FilePi
         }
     }
     if hit_timeout {
+        // Finish injecting the files in a background thread to not block displaying the file picker.
         std::thread::spawn(move || {
-            for file in files {
-                if injector.push(file).is_err() {
-                    break;
+            // We want to lazily traverse the file tree as the file picker window can be closed
+            // before a full traversal is complete. If we are not required to present the file list
+            // in the sorted order, we can build an instance of WalkParallel to speed things up.
+            if sorted {
+                for file in files_iter {
+                    if injector.push(file).is_err() {
+                        break;
+                    }
                 }
+            } else {
+                let injector = &injector;
+                let returned_files = &injected_files;
+                let num_threads = std::thread::available_parallelism().map_or(0, |v| v.get());
+
+                let walk_parallel = walk_builder.threads(num_threads).build_parallel();
+                walk_parallel.run(|| {
+                    Box::new(move |path| {
+                        if let Ok(path) = path {
+                            if path
+                                .file_type()
+                                .is_some_and(|file_type| file_type.is_file())
+                            {
+                                let path = path.into_path();
+                                if !returned_files.contains(&path) && injector.push(path).is_err() {
+                                    // Injector is shut down. Tell WalkParallel to stop.
+                                    return WalkState::Quit;
+                                }
+                            }
+                        }
+
+                        // If path is a directory entry, continue descending recursively.
+                        WalkState::Continue
+                    })
+                });
             }
+
+            log::debug!("file_picker init {:?}", Instant::now().duration_since(now));
         });
+    } else {
+        log::debug!("file_picker init {:?}", Instant::now().duration_since(now));
     }
-    picker
 }
 
 type FileExplorer = Picker<(PathBuf, bool), (PathBuf, Style)>;

helix-view/src/editor.rs

@@ -200,6 +200,8 @@ pub struct FilePickerConfig {
     /// WalkBuilder options
     /// Maximum Depth to recurse directories in file picker and global search. Defaults to `None`.
     pub max_depth: Option<usize>,
+    /// Whether to present file entries in the sorted order. Defaults to true.
+    pub sorted: bool,
 }
 
 impl Default for FilePickerConfig {
@@ -214,6 +216,7 @@ impl Default for FilePickerConfig {
             git_global: true,
             git_exclude: true,
             max_depth: None,
+            sorted: true,
         }
     }
 }