feat: String/Binary View Support (#596)

closes #583

Author: WillAyd

python/bootstrap.py

@@ -172,7 +172,10 @@ def _pxd_header(self):
 
 class NanoarrowPxdGenerator(PxdGenerator):
     def _preprocess_content(self, content):
-        return re.sub(r"NANOARROW_MAX_FIXED_BUFFERS", "3", content)
+        content = re.sub(r"NANOARROW_MAX_FIXED_BUFFERS", "3", content)
+        content = re.sub(r"NANOARROW_BINARY_VIEW_INLINE_SIZE", "12", content)
+        content = re.sub(r"NANOARROW_BINARY_VIEW_PREFIX_SIZE", "4", content)
+        return content
 
     def _pxd_header(self):
         return (

python/src/nanoarrow/__init__.py

@@ -48,12 +48,14 @@
     float64,
     string,
     large_string,
+    string_view,
     list_,
     large_list,
     fixed_size_list,
     dictionary,
     binary,
     large_binary,
+    binary_view,
     fixed_size_binary,
     date32,
     date64,
@@ -82,6 +84,7 @@
     "TimeUnit",
     "Type",
     "binary",
+    "binary_view",
     "bool_",
     "c_array",
     "c_array_from_buffers",
@@ -117,6 +120,7 @@
     "nulls_forbid",
     "nulls_separate",
     "string",
+    "string_view",
     "struct",
     "schema",
     "time32",

python/src/nanoarrow/_types.pxd

@@ -67,6 +67,10 @@ cpdef enum CArrowType:
     LARGE_BINARY = NANOARROW_TYPE_LARGE_BINARY
     LARGE_LIST = NANOARROW_TYPE_LARGE_LIST
     INTERVAL_MONTH_DAY_NANO = NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO
+    RUN_END_ENCODED = NANOARROW_TYPE_RUN_END_ENCODED
+    BINARY_VIEW = NANOARROW_TYPE_BINARY_VIEW
+    STRING_VIEW = NANOARROW_TYPE_STRING_VIEW
+
 
 cdef equal(int type_id1, int type_id2)
 

python/src/nanoarrow/schema.py

@@ -76,6 +76,9 @@ class Type(enum.Enum):
     LARGE_BINARY = int(_types.LARGE_BINARY)
     LARGE_LIST = int(_types.LARGE_LIST)
     INTERVAL_MONTH_DAY_NANO = int(_types.INTERVAL_MONTH_DAY_NANO)
+    RUN_END_ENCODED = int(_types.RUN_END_ENCODED)
+    BINARY_VIEW = int(_types.BINARY_VIEW)
+    STRING_VIEW = int(_types.STRING_VIEW)
 
     def __arrow_c_schema__(self):
         # This will only work for parameter-free types
@@ -784,6 +787,24 @@ def large_string(nullable: bool = True) -> Schema:
     return Schema(Type.LARGE_STRING, nullable=nullable)
 
 
+def string_view(nullable: bool = True) -> Schema:
+    """Create an instance of a string view type.
+
+    Parameters
+    ----------
+    nullable : bool, optional
+        Use ``False`` to mark this field as non-nullable.
+
+    Examples
+    --------
+
+    >>> import nanoarrow as na
+    >>> na.string_view()
+    <Schema> string_view
+    """
+    return Schema(Type.STRING_VIEW, nullable=nullable)
+
+
 def binary(nullable: bool = True) -> Schema:
     """Create an instance of a variable or fixed-width binary type.
 
@@ -820,6 +841,24 @@ def large_binary(nullable: bool = True) -> Schema:
     return Schema(Type.LARGE_BINARY, nullable=nullable)
 
 
+def binary_view(nullable: bool = True) -> Schema:
+    """Create an instance of a binary view type.
+
+    Parameters
+    ----------
+    nullable : bool, optional
+        Use ``False`` to mark this field as non-nullable.
+
+    Examples
+    --------
+
+    >>> import nanoarrow as na
+    >>> na.binary_view()
+    <Schema> binary_view
+    """
+    return Schema(Type.BINARY_VIEW, nullable=nullable)
+
+
 def fixed_size_binary(byte_width: int, nullable: bool = True) -> Schema:
     """Create an instance of a variable or fixed-width binary type.
 

python/tests/test_schema.py

@@ -107,6 +107,8 @@ def test_schema_simple():
     assert na.interval_months().type == na.Type.INTERVAL_MONTHS
     assert na.interval_day_time().type == na.Type.INTERVAL_DAY_TIME
     assert na.interval_month_day_nano().type == na.Type.INTERVAL_MONTH_DAY_NANO
+    assert na.binary_view().type == na.Type.BINARY_VIEW
+    assert na.string_view().type == na.Type.STRING_VIEW
 
 
 def test_schema_fixed_size_binary():

src/nanoarrow/common/array.c

@@ -18,6 +18,7 @@
 #include <errno.h>
 #include <inttypes.h>
 #include <stdarg.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -32,6 +33,12 @@ static void ArrowArrayReleaseInternal(struct ArrowArray* array) {
     ArrowBitmapReset(&private_data->bitmap);
     ArrowBufferReset(&private_data->buffers[0]);
     ArrowBufferReset(&private_data->buffers[1]);
+    ArrowFree(private_data->buffer_data);
+    for (int32_t i = 0; i < private_data->n_variadic_buffers; ++i) {
+      ArrowBufferReset(&private_data->variadic_buffers[i]);
+    }
+    ArrowFree(private_data->variadic_buffers);
+    ArrowFree(private_data->variadic_buffer_sizes);
     ArrowFree(private_data);
   }
 
@@ -106,6 +113,10 @@ static ArrowErrorCode ArrowArraySetStorageType(struct ArrowArray* array,
     case NANOARROW_TYPE_DENSE_UNION:
       array->n_buffers = 2;
       break;
+    case NANOARROW_TYPE_BINARY_VIEW:
+    case NANOARROW_TYPE_STRING_VIEW:
+      array->n_buffers = NANOARROW_BINARY_VIEW_FIXED_BUFFERS;
+      break;
     case NANOARROW_TYPE_STRING:
     case NANOARROW_TYPE_LARGE_STRING:
     case NANOARROW_TYPE_BINARY:
@@ -148,12 +159,17 @@ ArrowErrorCode ArrowArrayInitFromType(struct ArrowArray* array,
   ArrowBitmapInit(&private_data->bitmap);
   ArrowBufferInit(&private_data->buffers[0]);
   ArrowBufferInit(&private_data->buffers[1]);
-  private_data->buffer_data[0] = NULL;
-  private_data->buffer_data[1] = NULL;
-  private_data->buffer_data[2] = NULL;
+  private_data->buffer_data =
+      (const void**)ArrowMalloc(sizeof(void*) * NANOARROW_MAX_FIXED_BUFFERS);
+  for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; ++i) {
+    private_data->buffer_data[i] = NULL;
+  }
+  private_data->n_variadic_buffers = 0;
+  private_data->variadic_buffers = NULL;
+  private_data->variadic_buffer_sizes = NULL;
 
   array->private_data = private_data;
-  array->buffers = (const void**)(&private_data->buffer_data);
+  array->buffers = (const void**)(private_data->buffer_data);
 
   // These are not technically "storage" in the sense that they do not appear
   // in the ArrowSchemaView's storage_type member; however, allowing them here
@@ -456,10 +472,26 @@ static void ArrowArrayFlushInternalPointers(struct ArrowArray* array) {
   struct ArrowArrayPrivateData* private_data =
       (struct ArrowArrayPrivateData*)array->private_data;
 
-  for (int64_t i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
+  const bool is_binary_view = private_data->storage_type == NANOARROW_TYPE_STRING_VIEW ||
+                              private_data->storage_type == NANOARROW_TYPE_BINARY_VIEW;
+  const int32_t nfixed_buf = is_binary_view ? 2 : NANOARROW_MAX_FIXED_BUFFERS;
+
+  for (int32_t i = 0; i < nfixed_buf; i++) {
     private_data->buffer_data[i] = ArrowArrayBuffer(array, i)->data;
   }
 
+  if (is_binary_view) {
+    const int32_t nvirt_buf = private_data->n_variadic_buffers;
+    private_data->buffer_data = (const void**)ArrowRealloc(
+        private_data->buffer_data, sizeof(void*) * (nfixed_buf + nvirt_buf + 1));
+    for (int32_t i = 0; i < nvirt_buf; i++) {
+      private_data->buffer_data[nfixed_buf + i] = private_data->variadic_buffers[i].data;
+    }
+    private_data->buffer_data[nfixed_buf + nvirt_buf] =
+        private_data->variadic_buffer_sizes;
+    array->buffers = (const void**)(private_data->buffer_data);
+  }
+
   for (int64_t i = 0; i < array->n_children; i++) {
     ArrowArrayFlushInternalPointers(array->children[i]);
   }
@@ -664,6 +696,7 @@ void ArrowArrayViewSetLength(struct ArrowArrayView* array_view, int64_t length)
             _ArrowRoundUpToMultipleOf8(array_view->layout.element_size_bits[i] * length) /
             8;
         continue;
+      case NANOARROW_BUFFER_TYPE_DATA_VIEW:
       case NANOARROW_BUFFER_TYPE_TYPE_ID:
       case NANOARROW_BUFFER_TYPE_UNION_OFFSET:
         array_view->buffer_views[i].size_bytes = element_size_bytes * length;
@@ -700,9 +733,15 @@ static int ArrowArrayViewSetArrayInternal(struct ArrowArrayView* array_view,
   array_view->offset = array->offset;
   array_view->length = array->length;
   array_view->null_count = array->null_count;
+  array_view->variadic_buffer_sizes = NULL;
+  array_view->n_variadic_buffers = 0;
 
   int64_t buffers_required = 0;
-  for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) {
+  const int nfixed_buf = array_view->storage_type == NANOARROW_TYPE_STRING_VIEW ||
+                                 array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW
+                             ? NANOARROW_BINARY_VIEW_FIXED_BUFFERS
+                             : NANOARROW_MAX_FIXED_BUFFERS;
+  for (int i = 0; i < nfixed_buf; i++) {
     if (array_view->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_NONE) {
       break;
     }
@@ -720,7 +759,19 @@ static int ArrowArrayViewSetArrayInternal(struct ArrowArrayView* array_view,
     }
   }
 
-  // Check the number of buffers
+  if (array_view->storage_type == NANOARROW_TYPE_STRING_VIEW ||
+      array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW) {
+    const int64_t n_buffers = array->n_buffers;
+    const int32_t nfixed_buf = NANOARROW_BINARY_VIEW_FIXED_BUFFERS;
+
+    const int32_t nvariadic_buf = (int32_t)(n_buffers - nfixed_buf - 1);
+    if (nvariadic_buf > 0) {
+      array_view->n_variadic_buffers = nvariadic_buf;
+      buffers_required += nvariadic_buf + 1;
+      array_view->variadic_buffer_sizes = (int64_t*)array->buffers[n_buffers - 1];
+    }
+  }
+
   if (buffers_required != array->n_buffers) {
     ArrowErrorSet(error,
                   "Expected array with %" PRId64 " buffer(s) but found %" PRId64
@@ -814,6 +865,7 @@ static int ArrowArrayViewValidateMinimal(struct ArrowArrayView* array_view,
         break;
       case NANOARROW_BUFFER_TYPE_TYPE_ID:
       case NANOARROW_BUFFER_TYPE_UNION_OFFSET:
+      case NANOARROW_BUFFER_TYPE_DATA_VIEW:
         min_buffer_size_bytes = element_size_bytes * offset_plus_length;
         break;
       case NANOARROW_BUFFER_TYPE_NONE: