cleanup

crates/polars-io/src/csv/read/parser.rs

@@ -341,7 +341,7 @@ pub(super) struct SplitLines<'a> {
     #[cfg(feature = "simd")]
     simd_quote_char: SimdVec,
     #[cfg(feature = "simd")]
-    previous_valid_eol: u64,
+    previous_valid_eols: u64,
     total_index: usize,
     quoting: bool,
 }
@@ -376,7 +376,7 @@ impl<'a> SplitLines<'a> {
             #[cfg(feature = "simd")]
             simd_quote_char,
             #[cfg(feature = "simd")]
-            previous_valid_eol: 0,
+            previous_valid_eols: 0,
             total_index: 0,
             quoting,
         }
@@ -439,6 +439,21 @@ impl<'a> Iterator for SplitLines<'a> {
             self.total_index = 0;
             let mut not_in_field_previous_iter = true;
 
+            if self.previous_valid_eols != 0 {
+                let pos = self.previous_valid_eols.trailing_zeros() as usize;
+                self.previous_valid_eols >>= (pos + 1) as u64;
+
+                unsafe {
+                    debug_assert!((pos) <= self.v.len());
+
+                    // return line up to this position
+                    let ret = Some(self.v.get_unchecked(..pos));
+                    // skip the '\n' token and update slice.
+                    self.v = self.v.get_unchecked_release(pos + 1..);
+                    return ret;
+                }
+            }
+
             loop {
                 let bytes = unsafe { self.v.get_unchecked_release(self.total_index..) };
                 if bytes.len() > SIMD_SIZE {
@@ -468,9 +483,9 @@ impl<'a> Iterator for SplitLines<'a> {
                     if valid_eols != 0 {
                         let pos = valid_eols.trailing_zeros() as usize;
                         if pos == SIMD_SIZE - 1 {
-                            self.previous_valid_eol = 0;
+                            self.previous_valid_eols = 0;
                         } else {
-                            self.previous_valid_eol = valid_eols >> (pos + 1) as u64;
+                            self.previous_valid_eols = valid_eols >> (pos + 1) as u64;
                         }
 
                         unsafe {

crates/polars-io/src/csv/read/read_impl.rs

@@ -409,7 +409,7 @@ impl<'a> CoreReader<'a> {
         let mut total_line_count = 0;
 
         // let t = std::time::Instant::now();
-        // let mut iter = SplitLines::new(bytes, self.quote_char.unwrap_or(b'"'), self.eol_char);
+        // let mut iter = SplitLines::new(bytes, self.quote_char, self.eol_char);
         // let c = iter.count();
         // dbg!(c);
         // dbg!(t.elapsed());

py-polars/polars/io/csv/functions.py

@@ -527,7 +527,6 @@ def read_csv(
                 skip_rows_after_header=skip_rows_after_header,
                 row_index_name=row_index_name,
                 row_index_offset=row_index_offset,
-                sample_size=sample_size,
                 eol_char=eol_char,
                 raise_if_empty=raise_if_empty,
                 truncate_ragged_lines=truncate_ragged_lines,
@@ -674,7 +673,6 @@ def _read_csv_impl(
         try_parse_dates,
         skip_rows_after_header,
         parse_row_index_args(row_index_name, row_index_offset),
-        sample_size=sample_size,
         eol_char=eol_char,
         raise_if_empty=raise_if_empty,
         truncate_ragged_lines=truncate_ragged_lines,
@@ -814,6 +812,9 @@ def read_csv_batched(
     sample_size
         Set the sample size. This is used to sample statistics to estimate the
         allocation needed.
+
+        .. deprecated:: 1.10.0
+            Is a no-op.
     eol_char
         Single byte end of line character (default: `\n`). When encountering a file
         with windows line endings (`\r\n`), one can go with the default `\n`. The extra