From 4986f1dfbde140f445c60cdb600fc96ee1ec332e Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 15:17:54 -0500 Subject: [PATCH 01/20] Add RecordBatchWriter proxy class --- .../io/ipc/proxy/record_batch_writer.cc | 76 +++++++++++++++++++ .../matlab/io/ipc/proxy/record_batch_writer.h | 42 ++++++++++ 2 files changed, 118 insertions(+) create mode 100644 matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc create mode 100644 matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc new file mode 100644 index 0000000000000..a29cf72ba2e52 --- /dev/null +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/io/file.h" +#include "arrow/matlab/error/error.h" +#include "arrow/matlab/tabular/proxy/record_batch.h" +#include "arrow/matlab/tabular/proxy/schema.h" +#include "arrow/matlab/tabular/proxy/table.h" +#include "arrow/matlab/io/ipc/proxy/record_batch_writer.h" + +#include "libmexclass/proxy/ProxyManager.h" + +namespace arrow::matlab::io::ipc::proxy { + +RecordBatchWriter::RecordBatchWriter( + const std::shared_ptr writer) + : writer{std::move(writer)} { + REGISTER_METHOD(RecordBatchWriter, close); + REGISTER_METHOD(RecordBatchWriter, writeRecordBatch); + REGISTER_METHOD(RecordBatchWriter, writeTable); +} + +void RecordBatchWriter::writeRecordBatch( + libmexclass::proxy::method::Context& context) { + namespace mda = ::matlab::data; + using RecordBatchProxy = ::arrow::matlab::tabular::proxy::RecordBatch; + + mda::StructArray opts = context.inputs[0]; + const mda::TypedArray record_batch_proxy_id_mda = + opts[0]["RecordBatchProxyID"]; + const uint64_t record_batch_proxy_id = record_batch_proxy_id_mda[0]; + + auto proxy = libmexclass::proxy::ProxyManager::getProxy(record_batch_proxy_id); + auto record_batch_proxy = std::static_pointer_cast(proxy); + auto record_batch = record_batch_proxy->unwrap(); + + MATLAB_ERROR_IF_NOT_OK_WITH_CONTEXT(writer->WriteRecordBatch(*record_batch), context, + error::IPC_RECORD_BATCH_WRITE_FAILED); +} + +void RecordBatchWriter::writeTable(libmexclass::proxy::method::Context& context) { + namespace mda = ::matlab::data; + using TableProxy = ::arrow::matlab::tabular::proxy::Table; + + mda::StructArray opts = context.inputs[0]; + const mda::TypedArray table_proxy_id_mda = opts[0]["TableProxyID"]; + const uint64_t table_proxy_id = table_proxy_id_mda[0]; + + auto proxy = libmexclass::proxy::ProxyManager::getProxy(table_proxy_id); + auto table_proxy = std::static_pointer_cast(proxy); + auto table = table_proxy->unwrap(); + + MATLAB_ERROR_IF_NOT_OK_WITH_CONTEXT(writer->WriteTable(*table), context, + error::IPC_RECORD_BATCH_WRITE_FAILED); +} + +void RecordBatchWriter::close(libmexclass::proxy::method::Context& context) { + MATLAB_ERROR_IF_NOT_OK_WITH_CONTEXT(writer->Close(), context, + error::IPC_RECORD_BATCH_WRITE_CLOSE_FAILED); +} + +} // namespace arrow::matlab::io::ipc::proxy diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h new file mode 100644 index 0000000000000..4ecaad41b05d6 --- /dev/null +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/ipc/writer.h" +#include "libmexclass/proxy/Proxy.h" + +namespace arrow::matlab::io::ipc::proxy { + +class RecordBatchWriter : public libmexclass::proxy::Proxy { + public: + RecordBatchWriter(std::shared_ptr writer); + + virtual ~RecordBatchWriter() = default; + + static libmexclass::proxy::MakeResult make( + const libmexclass::proxy::FunctionArguments& constructor_arguments); + + protected: + std::shared_ptr writer; + + void writeRecordBatch(libmexclass::proxy::method::Context& context); + + void writeTable(libmexclass::proxy::method::Context& context); + + void close(libmexclass::proxy::method::Context& context); +}; + +} // namespace arrow::matlab::io::ipc::proxy From 2e7c2f23f0db89132b66ecbf4d93b753a28f6753 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 15:18:49 -0500 Subject: [PATCH 02/20] Add record_batch_writer.cc to CMake sources --- matlab/tools/cmake/BuildMatlabArrowInterface.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matlab/tools/cmake/BuildMatlabArrowInterface.cmake b/matlab/tools/cmake/BuildMatlabArrowInterface.cmake index 8016cbf261b7c..b9475fa51d1cc 100644 --- a/matlab/tools/cmake/BuildMatlabArrowInterface.cmake +++ b/matlab/tools/cmake/BuildMatlabArrowInterface.cmake @@ -81,7 +81,8 @@ set(MATLAB_ARROW_LIBMEXCLASS_CLIENT_PROXY_SOURCES "${CMAKE_SOURCE_DIR}/src/cpp/a "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/c/proxy/schema.cc" "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/c/proxy/record_batch_importer.cc" "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_reader.cc" - "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc") + "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc" + "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc") set(MATLAB_ARROW_LIBMEXCLASS_CLIENT_PROXY_FACTORY_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/proxy") From b4eba33a60b168c8e8bcdb982f41b914cc36e844 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 15:34:38 -0500 Subject: [PATCH 03/20] Re-implement proxy::RecordBatchFileWriter to inherit from proxy::RecordBatchWriter --- .../io/ipc/proxy/record_batch_file_writer.cc | 51 ++----------------- .../io/ipc/proxy/record_batch_file_writer.h | 16 ++---- 2 files changed, 8 insertions(+), 59 deletions(-) diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc index ed1052e0a8076..e583a4f55a97a 100644 --- a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc @@ -15,12 +15,10 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/matlab/io/ipc/proxy/record_batch_file_writer.h" #include "arrow/io/file.h" #include "arrow/matlab/error/error.h" -#include "arrow/matlab/tabular/proxy/record_batch.h" +#include "arrow/matlab/io/ipc/proxy/record_batch_file_writer.h" #include "arrow/matlab/tabular/proxy/schema.h" -#include "arrow/matlab/tabular/proxy/table.h" #include "arrow/util/utf8.h" #include "libmexclass/proxy/ProxyManager.h" @@ -29,11 +27,7 @@ namespace arrow::matlab::io::ipc::proxy { RecordBatchFileWriter::RecordBatchFileWriter( const std::shared_ptr writer) - : writer{std::move(writer)} { - REGISTER_METHOD(RecordBatchFileWriter, close); - REGISTER_METHOD(RecordBatchFileWriter, writeRecordBatch); - REGISTER_METHOD(RecordBatchFileWriter, writeTable); -} + : RecordBatchWriter(std::move(writer)) {} libmexclass::proxy::MakeResult RecordBatchFileWriter::make( const libmexclass::proxy::FunctionArguments& constructor_arguments) { @@ -65,43 +59,4 @@ libmexclass::proxy::MakeResult RecordBatchFileWriter::make( return std::make_shared(std::move(writer)); } -void RecordBatchFileWriter::writeRecordBatch( - libmexclass::proxy::method::Context& context) { - namespace mda = ::matlab::data; - using RecordBatchProxy = ::arrow::matlab::tabular::proxy::RecordBatch; - - mda::StructArray opts = context.inputs[0]; - const mda::TypedArray record_batch_proxy_id_mda = - opts[0]["RecordBatchProxyID"]; - const uint64_t record_batch_proxy_id = record_batch_proxy_id_mda[0]; - - auto proxy = libmexclass::proxy::ProxyManager::getProxy(record_batch_proxy_id); - auto record_batch_proxy = std::static_pointer_cast(proxy); - auto record_batch = record_batch_proxy->unwrap(); - - MATLAB_ERROR_IF_NOT_OK_WITH_CONTEXT(writer->WriteRecordBatch(*record_batch), context, - error::IPC_RECORD_BATCH_WRITE_FAILED); -} - -void RecordBatchFileWriter::writeTable(libmexclass::proxy::method::Context& context) { - namespace mda = ::matlab::data; - using TableProxy = ::arrow::matlab::tabular::proxy::Table; - - mda::StructArray opts = context.inputs[0]; - const mda::TypedArray table_proxy_id_mda = opts[0]["TableProxyID"]; - const uint64_t table_proxy_id = table_proxy_id_mda[0]; - - auto proxy = libmexclass::proxy::ProxyManager::getProxy(table_proxy_id); - auto table_proxy = std::static_pointer_cast(proxy); - auto table = table_proxy->unwrap(); - - MATLAB_ERROR_IF_NOT_OK_WITH_CONTEXT(writer->WriteTable(*table), context, - error::IPC_RECORD_BATCH_WRITE_FAILED); -} - -void RecordBatchFileWriter::close(libmexclass::proxy::method::Context& context) { - MATLAB_ERROR_IF_NOT_OK_WITH_CONTEXT(writer->Close(), context, - error::IPC_RECORD_BATCH_WRITE_CLOSE_FAILED); -} - -} // namespace arrow::matlab::io::ipc::proxy \ No newline at end of file +} // namespace arrow::matlab::io::ipc::proxy diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.h b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.h index bfd83504f190a..c9851cba2eb66 100644 --- a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.h +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.h @@ -16,27 +16,21 @@ // under the License. #include "arrow/ipc/writer.h" +#include "arrow/matlab/io/ipc/proxy/record_batch_writer.h" + #include "libmexclass/proxy/Proxy.h" namespace arrow::matlab::io::ipc::proxy { -class RecordBatchFileWriter : public libmexclass::proxy::Proxy { +class RecordBatchFileWriter : public RecordBatchWriter { public: RecordBatchFileWriter(std::shared_ptr writer); - ~RecordBatchFileWriter() = default; + virtual ~RecordBatchFileWriter() = default; static libmexclass::proxy::MakeResult make( const libmexclass::proxy::FunctionArguments& constructor_arguments); - protected: - std::shared_ptr writer; - - void writeRecordBatch(libmexclass::proxy::method::Context& context); - - void writeTable(libmexclass::proxy::method::Context& context); - - void close(libmexclass::proxy::method::Context& context); }; -} // namespace arrow::matlab::io::ipc::proxy \ No newline at end of file +} // namespace arrow::matlab::io::ipc::proxy From a72d5add4976c97b9b1abf57d542af62ca3e735b Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 15:39:02 -0500 Subject: [PATCH 04/20] Add RecordBatchWriter MATLAB class --- .../+arrow/+io/+ipc/RecordBatchWriter.m | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m diff --git a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m new file mode 100644 index 0000000000000..77b248439f744 --- /dev/null +++ b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m @@ -0,0 +1,75 @@ +%RECORDBATCHFILEWRITER Class for serializing record batches to the Arrow +% IPC format. + +% Licensed to the Apache Software Foundation (ASF) under one or more +% contributor license agreements. See the NOTICE file distributed with +% this work for additional information regarding copyright ownership. +% The ASF licenses this file to you under the Apache License, Version +% 2.0 (the "License"); you may not use this file except in compliance +% with the License. You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +% implied. See the License for the specific language governing +% permissions and limitations under the License. + + +classdef (Abstract) RecordBatchWriter < matlab.mixin.Scalar + + properties(SetAccess=private, GetAccess=public, Hidden) + Proxy + end + + methods + function obj = RecordBatchWriter(proxy) + arguments + proxy(1, 1) libmexclass.proxy.Proxy + end + obj.Proxy = proxy; + end + + function writeRecordBatch(obj, recordBatch) + arguments + obj(1, 1) arrow.io.ipc.RecordBatchWriter + recordBatch(1, 1) arrow.tabular.RecordBatch + end + + args = struct(RecordBatchProxyID=recordBatch.Proxy.ID); + obj.Proxy.writeRecordBatch(args); + end + + function writeTable(obj, arrowTable) + arguments + obj(1, 1) arrow.io.ipc.RecordBatchWriter + arrowTable(1, 1) arrow.tabular.Table + end + + args = struct(TableProxyID=arrowTable.Proxy.ID); + obj.Proxy.writeTable(args); + end + + function write(obj, tabularObj) + arguments + obj(1, 1) arrow.io.ipc.RecordBatchWriter + tabularObj(1, 1) + end + if isa(tabularObj, "arrow.tabular.RecordBatch") + obj.writeRecordBatch(tabularObj); + elseif isa(tabularObj, "arrow.tabular.Table") + obj.writeTable(tabularObj); + else + id = "arrow:matlab:ipc:write:InvalidType"; + msg = "tabularObj input argument must be an instance of " + ... + "either arrow.tabular.RecordBatch or arrow.tabular.Table."; + error(id, msg); + end + end + + function close(obj) + obj.Proxy.close(); + end + end +end \ No newline at end of file From 1160fa7c9632ab86e4633d6e0326f00b265b64a6 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 15:43:55 -0500 Subject: [PATCH 05/20] Update RecordBatchFileWriter to inherit from RecordBatchWriter --- .../+arrow/+io/+ipc/RecordBatchFileWriter.m | 56 ++----------------- 1 file changed, 6 insertions(+), 50 deletions(-) diff --git a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchFileWriter.m b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchFileWriter.m index aee4acf5c16e6..bfe48c02b6490 100644 --- a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchFileWriter.m +++ b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchFileWriter.m @@ -1,5 +1,5 @@ -%RECORDBATCHFILEWRITER Class for serializing record batches to a file using -% the IPC format. +%RECORDBATCHFILEWRITER Class for serializing record batches the IPC File +% format. % Licensed to the Apache Software Foundation (ASF) under one or more % contributor license agreements. See the NOTICE file distributed with @@ -16,11 +16,7 @@ % implied. See the License for the specific language governing % permissions and limitations under the License. -classdef RecordBatchFileWriter < matlab.mixin.Scalar - - properties(SetAccess=private, GetAccess=public, Hidden) - Proxy - end +classdef RecordBatchFileWriter < arrow.io.ipc.RecordBatchWriter methods function obj = RecordBatchFileWriter(filename, schema) @@ -30,48 +26,8 @@ end args = struct(Filename=filename, SchemaProxyID=schema.Proxy.ID); proxyName = "arrow.io.ipc.proxy.RecordBatchFileWriter"; - obj.Proxy = arrow.internal.proxy.create(proxyName, args); - end - - function writeRecordBatch(obj, recordBatch) - arguments - obj(1, 1) arrow.io.ipc.RecordBatchFileWriter - recordBatch(1, 1) arrow.tabular.RecordBatch - end - - args = struct(RecordBatchProxyID=recordBatch.Proxy.ID); - obj.Proxy.writeRecordBatch(args); - end - - function writeTable(obj, arrowTable) - arguments - obj(1, 1) arrow.io.ipc.RecordBatchFileWriter - arrowTable(1, 1) arrow.tabular.Table - end - - args = struct(TableProxyID=arrowTable.Proxy.ID); - obj.Proxy.writeTable(args); - end - - function write(obj, tabularObj) - arguments - obj(1, 1) arrow.io.ipc.RecordBatchFileWriter - tabularObj(1, 1) - end - if isa(tabularObj, "arrow.tabular.RecordBatch") - obj.writeRecordBatch(tabularObj); - elseif isa(tabularObj, "arrow.tabular.Table") - obj.writeTable(tabularObj); - else - id = "arrow:matlab:ipc:write:InvalidType"; - msg = "tabularObj input argument must be an instance of " + ... - "either arrow.tabular.RecordBatch or arrow.tabular.Table."; - error(id, msg); - end - end - - function close(obj) - obj.Proxy.close(); + proxy = arrow.internal.proxy.create(proxyName, args); + obj@arrow.io.ipc.RecordBatchWriter(proxy); end end -end +end \ No newline at end of file From 81f64f772f0fcdca8c9516c92eee7d9af49a310e Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 15:54:29 -0500 Subject: [PATCH 06/20] Add RecordBatchStreamWriter Proxy class --- .../ipc/proxy/record_batch_stream_writer.cc | 63 +++++++++++++++++++ .../io/ipc/proxy/record_batch_stream_writer.h | 36 +++++++++++ 2 files changed, 99 insertions(+) create mode 100644 matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc create mode 100644 matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc new file mode 100644 index 0000000000000..950e0afa57e33 --- /dev/null +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/io/file.h" +#include "arrow/ipc/writer.h" +#include "arrow/matlab/error/error.h" +#include "arrow/matlab/io/ipc/proxy/record_batch_file_writer.h" +#include "arrow/matlab/tabular/proxy/schema.h" +#include "arrow/util/utf8.h" + +#include "libmexclass/proxy/ProxyManager.h" + +namespace arrow::matlab::io::ipc::proxy { + +RecordBatchStreamWriter::RecordBatchStreamWriter( + const std::shared_ptr writer) + : RecordBatchWriter(std::move(writer)) {} + +libmexclass::proxy::MakeResult RecordBatchStreamWriter::make( + const libmexclass::proxy::FunctionArguments& constructor_arguments) { + namespace mda = ::matlab::data; + using RecordBatchStreamWriterProxy = arrow::matlab::io::ipc::proxy::RecordBatchStreamWriter; + using SchemaProxy = arrow::matlab::tabular::proxy::Schema; + + const mda::StructArray opts = constructor_arguments[0]; + + const mda::StringArray filename_mda = opts[0]["Filename"]; + const auto filename_utf16 = std::u16string(filename_mda[0]); + MATLAB_ASSIGN_OR_ERROR(const auto filename_utf8, + arrow::util::UTF16StringToUTF8(filename_utf16), + error::UNICODE_CONVERSION_ERROR_ID); + + const mda::TypedArray arrow_schema_proxy_id_mda = opts[0]["SchemaProxyID"]; + auto proxy = libmexclass::proxy::ProxyManager::getProxy(arrow_schema_proxy_id_mda[0]); + auto arrow_schema_proxy = std::static_pointer_cast(proxy); + auto arrow_schema = arrow_schema_proxy->unwrap(); + + MATLAB_ASSIGN_OR_ERROR(auto output_stream, + arrow::io::FileOutputStream::Open(filename_utf8), + error::FAILED_TO_OPEN_FILE_FOR_WRITE); + + MATLAB_ASSIGN_OR_ERROR(auto writer, + arrow::ipc::MakeStreamWriter(output_stream, arrow_schema), + "arrow:matlab:MakeFailed"); + + return std::make_shared(std::move(writer)); +} + +} // namespace arrow::matlab::io::ipc::proxy diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h new file mode 100644 index 0000000000000..7f3067e3ba581 --- /dev/null +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/ipc/writer.h" +#include "arrow/matlab/io/ipc/proxy/record_batch_writer.h" + +#include "libmexclass/proxy/Proxy.h" + +namespace arrow::matlab::io::ipc::proxy { + +class RecordBatchStreamWriter : public RecordBatchWriter { + public: + RecordBatchStreamWriter(std::shared_ptr writer); + + virtual ~RecordBatchStreamWriter() = default; + + static libmexclass::proxy::MakeResult make( + const libmexclass::proxy::FunctionArguments& constructor_arguments); + +}; + +} // namespace arrow::matlab::io::ipc::proxy From b2402feaf773c5a4a75ebe0845afae5d372c71be Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 15:55:18 -0500 Subject: [PATCH 07/20] Add record_batch_stream_writer.cc to CMake sources --- matlab/tools/cmake/BuildMatlabArrowInterface.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matlab/tools/cmake/BuildMatlabArrowInterface.cmake b/matlab/tools/cmake/BuildMatlabArrowInterface.cmake index b9475fa51d1cc..29a737a6ecf25 100644 --- a/matlab/tools/cmake/BuildMatlabArrowInterface.cmake +++ b/matlab/tools/cmake/BuildMatlabArrowInterface.cmake @@ -82,7 +82,8 @@ set(MATLAB_ARROW_LIBMEXCLASS_CLIENT_PROXY_SOURCES "${CMAKE_SOURCE_DIR}/src/cpp/a "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/c/proxy/record_batch_importer.cc" "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_reader.cc" "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc" - "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc") + "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc" + "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc") set(MATLAB_ARROW_LIBMEXCLASS_CLIENT_PROXY_FACTORY_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/proxy") From db64729b48500e8edaa065edf8486824bf6d681c Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 15:56:58 -0500 Subject: [PATCH 08/20] Fix RecordBatchStreamWriter compiler errors --- .../cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc index 950e0afa57e33..a6f7f9064033f 100644 --- a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc @@ -18,7 +18,7 @@ #include "arrow/io/file.h" #include "arrow/ipc/writer.h" #include "arrow/matlab/error/error.h" -#include "arrow/matlab/io/ipc/proxy/record_batch_file_writer.h" +#include "arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h" #include "arrow/matlab/tabular/proxy/schema.h" #include "arrow/util/utf8.h" From 2caefcdfffca90575f5d550ec7563ee54092b038 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 16:03:39 -0500 Subject: [PATCH 09/20] Add RecordBatchStreamWriter MATLAB class --- .../+arrow/+io/+ipc/RecordBatchStreamWriter.m | 34 +++++++++++++++++++ .../+arrow/+io/+ipc/RecordBatchWriter.m | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 matlab/src/matlab/+arrow/+io/+ipc/RecordBatchStreamWriter.m diff --git a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchStreamWriter.m b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchStreamWriter.m new file mode 100644 index 0000000000000..17fe7184a8df8 --- /dev/null +++ b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchStreamWriter.m @@ -0,0 +1,34 @@ +%RECORDBATCHSTREAMWRITER Class for serializing record batches to the Arrow +% IPC Streaming format. + +% Licensed to the Apache Software Foundation (ASF) under one or more +% contributor license agreements. See the NOTICE file distributed with +% this work for additional information regarding copyright ownership. +% The ASF licenses this file to you under the Apache License, Version +% 2.0 (the "License"); you may not use this file except in compliance +% with the License. You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +% implied. See the License for the specific language governing +% permissions and limitations under the License. + +classdef RecordBatchStreamWriter < arrow.io.ipc.RecordBatchWriter + + methods + function obj = RecordBatchStreamWriter(filename, schema) + arguments + filename(1, 1) string {mustBeNonzeroLengthText} + schema(1, 1) arrow.tabular.Schema + end + args = struct(Filename=filename, SchemaProxyID=schema.Proxy.ID); + proxyName = "arrow.io.ipc.proxy.RecordBatchStreamWriter"; + proxy = arrow.internal.proxy.create(proxyName, args); + obj@arrow.io.ipc.RecordBatchWriter(proxy); + end + end +end + diff --git a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m index 77b248439f744..97d5d41a837a5 100644 --- a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m +++ b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m @@ -1,4 +1,4 @@ -%RECORDBATCHFILEWRITER Class for serializing record batches to the Arrow +%RECORDBATCHWRITER Class for serializing record batches to the Arrow % IPC format. % Licensed to the Apache Software Foundation (ASF) under one or more From 8ba77099262d5cf280359d1ec3b95a30f29b0288 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 16:11:59 -0500 Subject: [PATCH 10/20] Add #pragma once --- matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h index 4ecaad41b05d6..bb31ddfcb34d7 100644 --- a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#pragma once + #include "arrow/ipc/writer.h" #include "libmexclass/proxy/Proxy.h" From 5f888fd140530640f5dccc424003d7b9f415de25 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 16:12:22 -0500 Subject: [PATCH 11/20] Register arrow.io.ipc.proxy.RecordBatchStreamReader class --- matlab/src/cpp/arrow/matlab/proxy/factory.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/matlab/src/cpp/arrow/matlab/proxy/factory.cc b/matlab/src/cpp/arrow/matlab/proxy/factory.cc index 8326b4371917a..a08a7495c00c9 100644 --- a/matlab/src/cpp/arrow/matlab/proxy/factory.cc +++ b/matlab/src/cpp/arrow/matlab/proxy/factory.cc @@ -36,6 +36,7 @@ #include "arrow/matlab/io/feather/proxy/writer.h" #include "arrow/matlab/io/ipc/proxy/record_batch_file_reader.h" #include "arrow/matlab/io/ipc/proxy/record_batch_file_writer.h" +#include "arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h" #include "arrow/matlab/tabular/proxy/record_batch.h" #include "arrow/matlab/tabular/proxy/schema.h" #include "arrow/matlab/tabular/proxy/table.h" @@ -111,6 +112,8 @@ libmexclass::proxy::MakeResult Factory::make_proxy( REGISTER_PROXY(arrow.c.proxy.RecordBatchImporter , arrow::matlab::c::proxy::RecordBatchImporter); REGISTER_PROXY(arrow.io.ipc.proxy.RecordBatchFileReader , arrow::matlab::io::ipc::proxy::RecordBatchFileReader); REGISTER_PROXY(arrow.io.ipc.proxy.RecordBatchFileWriter , arrow::matlab::io::ipc::proxy::RecordBatchFileWriter); + REGISTER_PROXY(arrow.io.ipc.proxy.RecordBatchStreamWriter , arrow::matlab::io::ipc::proxy::RecordBatchStreamWriter); + // clang-format on return libmexclass::error::Error{error::UNKNOWN_PROXY_ERROR_ID, From ce2b6a3365f20d7868fcd4958a2097592716364f Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 16:14:11 -0500 Subject: [PATCH 12/20] Parameterize tRecordBatchFileWriter to test both RecordBatchFileWriter and RecordBatchStreamWriter --- .../arrow/io/ipc/tRecordBatchFileWriter.m | 76 +++++++++++-------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/matlab/test/arrow/io/ipc/tRecordBatchFileWriter.m b/matlab/test/arrow/io/ipc/tRecordBatchFileWriter.m index 25bbf4474edd4..b78e44ac1bf1d 100644 --- a/matlab/test/arrow/io/ipc/tRecordBatchFileWriter.m +++ b/matlab/test/arrow/io/ipc/tRecordBatchFileWriter.m @@ -1,4 +1,5 @@ -%TRECORDBATCHFILEWRITER Unit tests for arrow.io.ipc.RecordBatchFileWriter. +%TRECORDBATCHFILEWRITER Unit tests for arrow.io.ipc.RecordBatchFileWriter +% and arrow.io.ipc.RecordBatchStreamWriter. % Licensed to the Apache Software Foundation (ASF) under one or more % contributor license agreements. See the NOTICE file distributed with @@ -17,6 +18,15 @@ classdef tRecordBatchFileWriter < matlab.unittest.TestCase + properties(TestParameter) + WriterConstructor = struct(... + RecordBatchFileWriter=@arrow.io.ipc.RecordBatchFileWriter,... + RecordBatchStreamWriter=@arrow.io.ipc.RecordBatchStreamWriter... + ); + end + + + methods function folder = setupTemporaryFolder(testCase) import matlab.unittest.fixtures.TemporaryFolderFixture @@ -26,45 +36,45 @@ end methods (Test) - function ZeroLengthFilenameError(testCase) - % Verify RecordBatchFileWriter throws an exception with the + function ZeroLengthFilenameError(testCase, WriterConstructor) + % Verify RecordBatchWriter throws an exception with the % identifier MATLAB:validators:mustBeNonzeroLengthText if the % filename input argument given is a zero length string. schema = arrow.schema(arrow.field("A", arrow.float64())); - fcn = @() arrow.io.ipc.RecordBatchFileWriter("", schema); + fcn = @() WriterConstructor("", schema); testCase.verifyError(fcn, "MATLAB:validators:mustBeNonzeroLengthText"); end - function MissingStringFilenameError(testCase) - % Verify RecordBatchFileWriter throws an exception with the + function MissingStringFilenameError(testCase, WriterConstructor) + % Verify RecordBatchWriter throws an exception with the % identifier MATLAB:validators:mustBeNonzeroLengthText if the % filename input argument given is a missing string. schema = arrow.schema(arrow.field("A", arrow.float64())); - fcn = @() arrow.io.ipc.RecordBatchFileWriter(string(missing), schema); + fcn = @() WriterConstructor(string(missing), schema); testCase.verifyError(fcn, "MATLAB:validators:mustBeNonzeroLengthText"); end - function FilenameInvalidTypeError(testCase) - % Verify RecordBatchFileWriter throws an exception with the + function FilenameInvalidTypeError(testCase, WriterConstructor) + % Verify RecordBatchWriter throws an exception with the % identifier MATLAB:validators:UnableToConvert if the filename % input argument is neither a scalar string nor a char vector. schema = arrow.schema(arrow.field("A", arrow.float64())); - fcn = @() arrow.io.ipc.RecordBatchFileWriter(table, schema); + fcn = @() WriterConstructor(table, schema); testCase.verifyError(fcn, "MATLAB:validation:UnableToConvert"); end - function InvalidSchemaType(testCase) - % Verify RecordBatchFileWriter throws an exception with the + function InvalidSchemaType(testCase, WriterConstructor) + % Verify RecordBatchWriter throws an exception with the % identifier MATLAB:validators:UnableToConvert if the schema % input argument is not an arrow.tabular.Schema instance. folder = testCase.setupTemporaryFolder(); fname = fullfile(folder, "data.arrow"); schema = arrow.field("A", arrow.float64()); - fcn = @() arrow.io.ipc.RecordBatchFileWriter(fname, schema); + fcn = @() WriterConstructor(fname, schema); testCase.verifyError(fcn, "MATLAB:validation:UnableToConvert"); end - function writeRecordBatchInvalidType(testCase) + function writeRecordBatchInvalidType(testCase, WriterConstructor) % Verify writeRecordBatch throws an exception with the % identifier MATLAB:validators:UnableToConvert if the % recordBatch input argument given is not an @@ -72,26 +82,26 @@ function writeRecordBatchInvalidType(testCase) folder = testCase.setupTemporaryFolder(); fname = fullfile(folder, "data.arrow"); schema = arrow.schema(arrow.field("A", arrow.float64())); - writer = arrow.io.ipc.RecordBatchFileWriter(fname, schema); + writer = WriterConstructor(fname, schema); arrowTable = arrow.table(table([1 2 3 4]', VariableNames="A")); fcn = @() writer.writeRecordBatch(arrowTable); testCase.verifyError(fcn, "MATLAB:validation:UnableToConvert"); end - function writeTableInvalidType(testCase) + function writeTableInvalidType(testCase, WriterConstructor) % Verify writeTable throws an exception with the % identifier MATLAB:validators:UnableToConvert if the table % input argument given is not an arrow.tabular.Table instance. folder = testCase.setupTemporaryFolder(); fname = fullfile(folder, "data.arrow"); schema = arrow.schema(arrow.field("A", arrow.float64())); - writer = arrow.io.ipc.RecordBatchFileWriter(fname, schema); + writer = WriterConstructor(fname, schema); arrowRecordBatch = arrow.recordBatch(table([1 2 3 4]', VariableNames="A")); fcn = @() writer.writeTable(arrowRecordBatch); testCase.verifyError(fcn, "MATLAB:validation:UnableToConvert"); end - function writeInvalidType(testCase) + function writeInvalidType(testCase, WriterConstructor) % Verify writeTable throws an exception with the % identifier arrow:matlab:ipc:write:InvalidType if the % tabularObj input argument given is neither an @@ -99,12 +109,12 @@ function writeInvalidType(testCase) folder = testCase.setupTemporaryFolder(); fname = fullfile(folder, "data.arrow"); schema = arrow.schema(arrow.field("A", arrow.float64())); - writer = arrow.io.ipc.RecordBatchFileWriter(fname, schema); + writer = WriterConstructor(fname, schema); fcn = @() writer.write(schema); testCase.verifyError(fcn, "arrow:matlab:ipc:write:InvalidType"); end - function writeRecordBatchInvalidSchema(testCase) + function writeRecordBatchInvalidSchema(testCase, WriterConstructor) % Verify writeRecordBatch throws an exception with the % identifier arrow:io:ipc:FailedToWriteRecordBatch if the % schema of the given record batch does match the expected @@ -112,28 +122,28 @@ function writeRecordBatchInvalidSchema(testCase) folder = testCase.setupTemporaryFolder(); fname = fullfile(folder, "data.arrow"); schema = arrow.schema(arrow.field("A", arrow.float64())); - writer = arrow.io.ipc.RecordBatchFileWriter(fname, schema); + writer = WriterConstructor(fname, schema); arrowRecordBatch = arrow.recordBatch(table([1 2 3 4]', VariableNames="B")); fcn = @() writer.writeRecordBatch(arrowRecordBatch); testCase.verifyError(fcn, "arrow:io:ipc:FailedToWriteRecordBatch"); end - function writeTableInvalidSchema(testCase) + function writeTableInvalidSchema(testCase, WriterConstructor) % Verify writeTable throws an exception with the % identifier arrow:io:ipc:FailedToWriteRecordBatch if the % schema of the given table does match the expected schema. folder = testCase.setupTemporaryFolder(); fname = fullfile(folder, "data.arrow"); schema = arrow.schema(arrow.field("A", arrow.float64())); - writer = arrow.io.ipc.RecordBatchFileWriter(fname, schema); + writer = WriterConstructor(fname, schema); arrowTable = arrow.table(table([1 2 3 4]', VariableNames="B")); fcn = @() writer.writeTable(arrowTable); testCase.verifyError(fcn, "arrow:io:ipc:FailedToWriteRecordBatch"); end - function writeInvalidSchema(testCase) + function writeInvalidSchema(testCase, WriterConstructor) % Verify write throws an exception with the % identifier arrow:io:ipc:FailedToWriteRecordBatch if the % schema of the given record batch or table does match the @@ -141,7 +151,7 @@ function writeInvalidSchema(testCase) folder = testCase.setupTemporaryFolder(); fname = fullfile(folder, "data.arrow"); schema = arrow.schema(arrow.field("A", arrow.float64())); - writer = arrow.io.ipc.RecordBatchFileWriter(fname, schema); + writer = WriterConstructor(fname, schema); arrowTable = arrow.table(table([1 2 3 4]', VariableNames="B")); fcn = @() writer.write(arrowTable); @@ -152,39 +162,39 @@ function writeInvalidSchema(testCase) testCase.verifyError(fcn, "arrow:io:ipc:FailedToWriteRecordBatch"); end - function writeRecordBatchSmoke(testCase) + function writeRecordBatchSmoke(testCase, WriterConstructor) % Verify writeRecordBatch does not error or issue a warning % if it successfully writes the record batch to the file. folder = testCase.setupTemporaryFolder(); fname = fullfile(folder, "data.arrow"); schema = arrow.schema(arrow.field("A", arrow.float64())); - writer = arrow.io.ipc.RecordBatchFileWriter(fname, schema); + writer = WriterConstructor(fname, schema); arrowRecordBatch = arrow.recordBatch(table([1 2 3 4]', VariableNames="A")); fcn = @() writer.writeRecordBatch(arrowRecordBatch); testCase.verifyWarningFree(fcn); end - function writeTableBatchSmoke(testCase) + function writeTableBatchSmoke(testCase, WriterConstructor) % Verify writeTable does not error or issue a warning % if it successfully writes the table to the file. folder = testCase.setupTemporaryFolder(); fname = fullfile(folder, "data.arrow"); schema = arrow.schema(arrow.field("A", arrow.float64())); - writer = arrow.io.ipc.RecordBatchFileWriter(fname, schema); + writer = WriterConstructor(fname, schema); arrowTable = arrow.table(table([1 2 3 4]', VariableNames="A")); fcn = @() writer.writeTable(arrowTable); testCase.verifyWarningFree(fcn); end - function writeSmoke(testCase) + function writeSmoke(testCase, WriterConstructor) % Verify write does not error or issue a warning if it % successfully writes the record batch or table to the file. folder = testCase.setupTemporaryFolder(); fname = fullfile(folder, "data.arrow"); schema = arrow.schema(arrow.field("A", arrow.float64())); - writer = arrow.io.ipc.RecordBatchFileWriter(fname, schema); + writer = WriterConstructor(fname, schema); arrowRecordBatch = arrow.recordBatch(table([1 2 3 4]', VariableNames="A")); fcn = @() writer.write(arrowRecordBatch); @@ -195,13 +205,13 @@ function writeSmoke(testCase) testCase.verifyWarningFree(fcn); end - function closeSmoke(testCase) + function closeSmoke(testCase, WriterConstructor) % Verify close does not error or issue a warning if it was % successful. folder = testCase.setupTemporaryFolder(); fname = fullfile(folder, "data.arrow"); schema = arrow.schema(arrow.field("A", arrow.float64())); - writer = arrow.io.ipc.RecordBatchFileWriter(fname, schema); + writer = WriterConstructor(fname, schema); arrowTable = arrow.table(table([1 2 3 4]', VariableNames="A")); writer.write(arrowTable); fcn = @() writer.close(); From b8dc29bb0b537c57a5c2b0ea3dd1c6c364d50806 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 16:15:32 -0500 Subject: [PATCH 13/20] Rename tRecordBatchFileWriter.m to tRecordBatchWriter.m --- .../io/ipc/{tRecordBatchFileWriter.m => tRecordBatchWriter.m} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename matlab/test/arrow/io/ipc/{tRecordBatchFileWriter.m => tRecordBatchWriter.m} (98%) diff --git a/matlab/test/arrow/io/ipc/tRecordBatchFileWriter.m b/matlab/test/arrow/io/ipc/tRecordBatchWriter.m similarity index 98% rename from matlab/test/arrow/io/ipc/tRecordBatchFileWriter.m rename to matlab/test/arrow/io/ipc/tRecordBatchWriter.m index b78e44ac1bf1d..d1544a97f0415 100644 --- a/matlab/test/arrow/io/ipc/tRecordBatchFileWriter.m +++ b/matlab/test/arrow/io/ipc/tRecordBatchWriter.m @@ -1,4 +1,4 @@ -%TRECORDBATCHFILEWRITER Unit tests for arrow.io.ipc.RecordBatchFileWriter +%TRECORDBATCHWRITER Unit tests for arrow.io.ipc.RecordBatchFileWriter % and arrow.io.ipc.RecordBatchStreamWriter. % Licensed to the Apache Software Foundation (ASF) under one or more @@ -16,7 +16,7 @@ % implied. See the License for the specific language governing % permissions and limitations under the License. -classdef tRecordBatchFileWriter < matlab.unittest.TestCase +classdef tRecordBatchWriter < matlab.unittest.TestCase properties(TestParameter) WriterConstructor = struct(... From 87a7b0dfa2609816de2c51fef3ce97f8263b6edc Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 3 Dec 2024 16:58:14 -0500 Subject: [PATCH 14/20] Ran clang-format --- .../arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc | 2 +- .../cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.h | 1 - .../arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc | 5 +++-- .../arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h | 1 - .../src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc | 5 ++--- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc index e583a4f55a97a..69ba734bd0ef9 100644 --- a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.cc @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. +#include "arrow/matlab/io/ipc/proxy/record_batch_file_writer.h" #include "arrow/io/file.h" #include "arrow/matlab/error/error.h" -#include "arrow/matlab/io/ipc/proxy/record_batch_file_writer.h" #include "arrow/matlab/tabular/proxy/schema.h" #include "arrow/util/utf8.h" diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.h b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.h index c9851cba2eb66..ac76afaf23957 100644 --- a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.h +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_file_writer.h @@ -30,7 +30,6 @@ class RecordBatchFileWriter : public RecordBatchWriter { static libmexclass::proxy::MakeResult make( const libmexclass::proxy::FunctionArguments& constructor_arguments); - }; } // namespace arrow::matlab::io::ipc::proxy diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc index a6f7f9064033f..4640a54819b83 100644 --- a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.cc @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. +#include "arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h" #include "arrow/io/file.h" #include "arrow/ipc/writer.h" #include "arrow/matlab/error/error.h" -#include "arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h" #include "arrow/matlab/tabular/proxy/schema.h" #include "arrow/util/utf8.h" @@ -33,7 +33,8 @@ RecordBatchStreamWriter::RecordBatchStreamWriter( libmexclass::proxy::MakeResult RecordBatchStreamWriter::make( const libmexclass::proxy::FunctionArguments& constructor_arguments) { namespace mda = ::matlab::data; - using RecordBatchStreamWriterProxy = arrow::matlab::io::ipc::proxy::RecordBatchStreamWriter; + using RecordBatchStreamWriterProxy = + arrow::matlab::io::ipc::proxy::RecordBatchStreamWriter; using SchemaProxy = arrow::matlab::tabular::proxy::Schema; const mda::StructArray opts = constructor_arguments[0]; diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h index 7f3067e3ba581..484d1aa252c57 100644 --- a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_stream_writer.h @@ -30,7 +30,6 @@ class RecordBatchStreamWriter : public RecordBatchWriter { static libmexclass::proxy::MakeResult make( const libmexclass::proxy::FunctionArguments& constructor_arguments); - }; } // namespace arrow::matlab::io::ipc::proxy diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc index a29cf72ba2e52..beffcca0245f0 100644 --- a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.cc @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. +#include "arrow/matlab/io/ipc/proxy/record_batch_writer.h" #include "arrow/io/file.h" #include "arrow/matlab/error/error.h" #include "arrow/matlab/tabular/proxy/record_batch.h" #include "arrow/matlab/tabular/proxy/schema.h" #include "arrow/matlab/tabular/proxy/table.h" -#include "arrow/matlab/io/ipc/proxy/record_batch_writer.h" #include "libmexclass/proxy/ProxyManager.h" @@ -34,8 +34,7 @@ RecordBatchWriter::RecordBatchWriter( REGISTER_METHOD(RecordBatchWriter, writeTable); } -void RecordBatchWriter::writeRecordBatch( - libmexclass::proxy::method::Context& context) { +void RecordBatchWriter::writeRecordBatch(libmexclass::proxy::method::Context& context) { namespace mda = ::matlab::data; using RecordBatchProxy = ::arrow::matlab::tabular::proxy::RecordBatch; From e0417e13ccb059007786422cfaab95dcea9a3d27 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore <74676073+sgilmore10@users.noreply.github.com> Date: Wed, 4 Dec 2024 09:12:47 -0500 Subject: [PATCH 15/20] Update matlab/src/matlab/+arrow/+io/+ipc/RecordBatchFileWriter.m Co-authored-by: Sutou Kouhei --- matlab/src/matlab/+arrow/+io/+ipc/RecordBatchFileWriter.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchFileWriter.m b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchFileWriter.m index bfe48c02b6490..ee1298c23706f 100644 --- a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchFileWriter.m +++ b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchFileWriter.m @@ -1,4 +1,4 @@ -%RECORDBATCHFILEWRITER Class for serializing record batches the IPC File +%RECORDBATCHFILEWRITER Class for serializing record batches to the Arrow IPC File % format. % Licensed to the Apache Software Foundation (ASF) under one or more From bfed7884d0292694e44b7b479bf13dd572e41c9d Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 4 Dec 2024 09:17:21 -0500 Subject: [PATCH 16/20] Deleted declaration of un-implemented static method RecordBatchWriter::make --- matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h index bb31ddfcb34d7..885a0cbf207fe 100644 --- a/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h +++ b/matlab/src/cpp/arrow/matlab/io/ipc/proxy/record_batch_writer.h @@ -28,9 +28,6 @@ class RecordBatchWriter : public libmexclass::proxy::Proxy { virtual ~RecordBatchWriter() = default; - static libmexclass::proxy::MakeResult make( - const libmexclass::proxy::FunctionArguments& constructor_arguments); - protected: std::shared_ptr writer; From 86092fcf50837c5677bdd2dfc67ea1fd6370fe95 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore <74676073+sgilmore10@users.noreply.github.com> Date: Wed, 4 Dec 2024 10:38:02 -0500 Subject: [PATCH 17/20] Update matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m Co-authored-by: Kevin Gurney --- matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m index 97d5d41a837a5..ce9032896f6c2 100644 --- a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m +++ b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m @@ -61,7 +61,7 @@ function write(obj, tabularObj) elseif isa(tabularObj, "arrow.tabular.Table") obj.writeTable(tabularObj); else - id = "arrow:matlab:ipc:write:InvalidType"; + id = "arrow:matlab:io:ipc:write:InvalidType"; msg = "tabularObj input argument must be an instance of " + ... "either arrow.tabular.RecordBatch or arrow.tabular.Table."; error(id, msg); From 6dbb881d5124d2af13a7102063ac2a51a7d61041 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore <74676073+sgilmore10@users.noreply.github.com> Date: Wed, 4 Dec 2024 10:38:09 -0500 Subject: [PATCH 18/20] Update matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m Co-authored-by: Kevin Gurney --- matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m index ce9032896f6c2..6b39270a4ae9b 100644 --- a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m +++ b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m @@ -62,7 +62,7 @@ function write(obj, tabularObj) obj.writeTable(tabularObj); else id = "arrow:matlab:io:ipc:write:InvalidType"; - msg = "tabularObj input argument must be an instance of " + ... + msg = "Input must be an instance of " + ... "either arrow.tabular.RecordBatch or arrow.tabular.Table."; error(id, msg); end From ac2c18aec5eb714b6c606106c6b62da5eb1ad01c Mon Sep 17 00:00:00 2001 From: Sarah Gilmore <74676073+sgilmore10@users.noreply.github.com> Date: Wed, 4 Dec 2024 10:38:56 -0500 Subject: [PATCH 19/20] Update matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m Co-authored-by: Kevin Gurney --- matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m | 1 - 1 file changed, 1 deletion(-) diff --git a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m index 6b39270a4ae9b..68fb1467fbec6 100644 --- a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m +++ b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m @@ -16,7 +16,6 @@ % implied. See the License for the specific language governing % permissions and limitations under the License. - classdef (Abstract) RecordBatchWriter < matlab.mixin.Scalar properties(SetAccess=private, GetAccess=public, Hidden) From cd08378684274fe3be67820db1f5b03cb964ed4a Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 4 Dec 2024 10:48:52 -0500 Subject: [PATCH 20/20] Update error message ID --- matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m | 2 +- matlab/test/arrow/io/ipc/tRecordBatchWriter.m | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m index 68fb1467fbec6..a662392cc6f47 100644 --- a/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m +++ b/matlab/src/matlab/+arrow/+io/+ipc/RecordBatchWriter.m @@ -60,7 +60,7 @@ function write(obj, tabularObj) elseif isa(tabularObj, "arrow.tabular.Table") obj.writeTable(tabularObj); else - id = "arrow:matlab:io:ipc:write:InvalidType"; + id = "arrow:io:ipc:write:InvalidType"; msg = "Input must be an instance of " + ... "either arrow.tabular.RecordBatch or arrow.tabular.Table."; error(id, msg); diff --git a/matlab/test/arrow/io/ipc/tRecordBatchWriter.m b/matlab/test/arrow/io/ipc/tRecordBatchWriter.m index d1544a97f0415..55802e31f885d 100644 --- a/matlab/test/arrow/io/ipc/tRecordBatchWriter.m +++ b/matlab/test/arrow/io/ipc/tRecordBatchWriter.m @@ -111,7 +111,7 @@ function writeInvalidType(testCase, WriterConstructor) schema = arrow.schema(arrow.field("A", arrow.float64())); writer = WriterConstructor(fname, schema); fcn = @() writer.write(schema); - testCase.verifyError(fcn, "arrow:matlab:ipc:write:InvalidType"); + testCase.verifyError(fcn, "arrow:io:ipc:write:InvalidType"); end function writeRecordBatchInvalidSchema(testCase, WriterConstructor)