From 879ff324aac9f03c0e93145d87b92be27c5692bb Mon Sep 17 00:00:00 2001 From: Ruoxuan Wang <52271048+ruowan@users.noreply.github.com> Date: Tue, 5 Dec 2023 21:02:39 +0800 Subject: [PATCH 01/51] test(csharp/test/Drivers/Snowflake): refactor snowflake test framework (#1323) --- csharp/test/Drivers/Snowflake/ClientTests.cs | 68 +++++++++--------- .../Snowflake/Resources/snowflakeconfig.json | 21 ++++-- .../Snowflake/SnowflakeTestConfiguration.cs | 70 ++++++++++++++----- .../Snowflake/SnowflakeTestingUtils.cs | 11 ++- 4 files changed, 111 insertions(+), 59 deletions(-) diff --git a/csharp/test/Drivers/Snowflake/ClientTests.cs b/csharp/test/Drivers/Snowflake/ClientTests.cs index 77d12fb462..e895865024 100644 --- a/csharp/test/Drivers/Snowflake/ClientTests.cs +++ b/csharp/test/Drivers/Snowflake/ClientTests.cs @@ -39,7 +39,7 @@ public class ClientTests { public ClientTests() { - Skip.IfNot(Utils.CanExecuteTestConfig(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE)); + Skip.IfNot(Utils.CanExecuteTestConfig(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE)); } /// @@ -50,7 +50,7 @@ public void CanClientExecuteUpdate() { SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE); - using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration)) { string[] queries = SnowflakeTestingUtils.GetQueries(testConfiguration); @@ -68,7 +68,7 @@ public void CanClientExecuteUpdateUsingExecuteReader() { SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE); - using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration)) { adbcConnection.Open(); @@ -104,7 +104,7 @@ public void CanClientGetSchema() { SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE); - using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration)) { Tests.ClientTests.CanClientGetSchema(adbcConnection, testConfiguration); } @@ -119,7 +119,7 @@ public void CanClientExecuteQuery() { SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE); - using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration)) { Tests.ClientTests.CanClientExecuteQuery(adbcConnection, testConfiguration); } @@ -136,7 +136,7 @@ public void CanClientExecuteQueryWithNoResults() testConfiguration.Query = "SELECT * WHERE 0=1"; testConfiguration.ExpectedResultsCount = 0; - using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration)) { Tests.ClientTests.CanClientExecuteQuery(adbcConnection, testConfiguration); } @@ -151,7 +151,9 @@ public void CanClientExecuteQueryUsingPrivateKey() { SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE); - using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration)) + Skip.If(testConfiguration.Authentication.SnowflakeJwt is null, "JWT authentication is not configured"); + + using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration, SnowflakeAuthentication.AuthJwt)) { Tests.ClientTests.CanClientExecuteQuery(adbcConnection, testConfiguration); } @@ -166,7 +168,7 @@ public void VerifyTypesAndValues() { SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE); - using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration)) { SampleDataBuilder sampleDataBuilder = SnowflakeData.GetSampleData(); @@ -179,7 +181,7 @@ public void VerifySchemaTables() { SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE); - using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration)) { adbcConnection.Open(); @@ -218,7 +220,7 @@ public void VerifySchemaTables() } } - private Adbc.Client.AdbcConnection GetSnowflakeAdbcConnectionUsingConnectionString(SnowflakeTestConfiguration testConfiguration) + private Adbc.Client.AdbcConnection GetSnowflakeAdbcConnectionUsingConnectionString(SnowflakeTestConfiguration testConfiguration, string authType = null) { // see https://arrow.apache.org/adbc/0.5.1/driver/snowflake.html @@ -228,22 +230,32 @@ private Adbc.Client.AdbcConnection GetSnowflakeAdbcConnectionUsingConnectionStri builder[SnowflakeParameters.HOST] = testConfiguration.Host; builder[SnowflakeParameters.DATABASE] = testConfiguration.Database; builder[SnowflakeParameters.USERNAME] = testConfiguration.User; - if (!string.IsNullOrEmpty(testConfiguration.AuthenticationTokenPath)) + if (authType == SnowflakeAuthentication.AuthJwt) { - builder[SnowflakeParameters.AUTH_TYPE] = testConfiguration.AuthenticationType; - string privateKey = File.ReadAllText(testConfiguration.AuthenticationTokenPath); - if (testConfiguration.AuthenticationType.Equals("auth_jwt", StringComparison.OrdinalIgnoreCase)) + string privateKey = testConfiguration.Authentication.SnowflakeJwt.PrivateKey; + builder[SnowflakeParameters.AUTH_TYPE] = SnowflakeAuthentication.AuthJwt; + builder[SnowflakeParameters.PKCS8_VALUE] = privateKey; + builder[SnowflakeParameters.USERNAME] = testConfiguration.Authentication.SnowflakeJwt.User; + if (!string.IsNullOrEmpty(testConfiguration.Authentication.SnowflakeJwt.PrivateKeyPassPhrase)) { - builder[SnowflakeParameters.PKCS8_VALUE] = privateKey; - if(!string.IsNullOrEmpty(testConfiguration.Pkcs8Passcode)) - { - builder[SnowflakeParameters.PKCS8_PASS] = testConfiguration.Pkcs8Passcode; - } + builder[SnowflakeParameters.PKCS8_PASS] = testConfiguration.Authentication.SnowflakeJwt.PrivateKeyPassPhrase; + } + } + else if (authType == SnowflakeAuthentication.AuthOAuth) + { + builder[SnowflakeParameters.AUTH_TYPE] = SnowflakeAuthentication.AuthOAuth; + builder[SnowflakeParameters.AUTH_TOKEN] = testConfiguration.Authentication.OAuth.Token; + if (testConfiguration.Authentication.OAuth.User != null) + { + builder[SnowflakeParameters.USERNAME] = testConfiguration.Authentication.OAuth.User; } } - else + else if (string.IsNullOrEmpty(authType) || authType == SnowflakeAuthentication.AuthSnowflake) { - builder[SnowflakeParameters.PASSWORD] = testConfiguration.Password; + // if no auth type is specified, use the snowflake auth + builder[SnowflakeParameters.AUTH_TYPE] = SnowflakeAuthentication.AuthSnowflake; + builder[SnowflakeParameters.USERNAME] = testConfiguration.Authentication.Default.User; + builder[SnowflakeParameters.PASSWORD] = testConfiguration.Authentication.Default.Password; } AdbcDriver snowflakeDriver = SnowflakeTestingUtils.GetSnowflakeAdbcDriver(testConfiguration); return new Adbc.Client.AdbcConnection(builder.ConnectionString) @@ -251,19 +263,5 @@ private Adbc.Client.AdbcConnection GetSnowflakeAdbcConnectionUsingConnectionStri AdbcDriver = snowflakeDriver }; } - private Adbc.Client.AdbcConnection GetSnowflakeAdbcConnection(SnowflakeTestConfiguration testConfiguration) - { - Dictionary parameters = new Dictionary(); - - AdbcDriver snowflakeDriver = SnowflakeTestingUtils.GetSnowflakeAdbcDriver(testConfiguration, out parameters); - - Adbc.Client.AdbcConnection adbcConnection = new Adbc.Client.AdbcConnection( - snowflakeDriver, - parameters: parameters, - options: new Dictionary() - ); - - return adbcConnection; - } } } diff --git a/csharp/test/Drivers/Snowflake/Resources/snowflakeconfig.json b/csharp/test/Drivers/Snowflake/Resources/snowflakeconfig.json index d3b4358143..dec0c492e3 100644 --- a/csharp/test/Drivers/Snowflake/Resources/snowflakeconfig.json +++ b/csharp/test/Drivers/Snowflake/Resources/snowflakeconfig.json @@ -4,19 +4,28 @@ "account": "", "host": "", "database": "", - "user": "", - "password": "", "warehouse": "", - "authenticationType": "", - "authenticationTokenPath": "", - "pkcs8Passcode": "", - "useHighPrecision": true, + "useHighPrecision": true, "metadata": { "catalog": "", "schema": "", "table": "", "expectedColumnCount": 0 }, + "authentication": { + "auth_oauth": { + "token": "" + }, + "auth_jwt": { + "private_key_file": "", + "private_key_pwd": "", + "user": "" + }, + "auth_snowflake": { + "user": "", + "password": "" + } + }, "query": "", "expectedResults": 0 } diff --git a/csharp/test/Drivers/Snowflake/SnowflakeTestConfiguration.cs b/csharp/test/Drivers/Snowflake/SnowflakeTestConfiguration.cs index 96bdcc22b5..b2f7b2ba89 100644 --- a/csharp/test/Drivers/Snowflake/SnowflakeTestConfiguration.cs +++ b/csharp/test/Drivers/Snowflake/SnowflakeTestConfiguration.cs @@ -73,27 +73,65 @@ internal class SnowflakeTestConfiguration : TestConfiguration public string Warehouse { get; set; } /// - /// The Snowflake authentication type. + /// The Snowflake use high precision /// - [JsonPropertyName("authenticationType")] - public string AuthenticationType { get; set; } + [JsonPropertyName("useHighPrecision")] + public bool UseHighPrecision { get; set; } = true; /// - /// The file location of the authentication token (if using). + /// The snowflake Authentication /// - [JsonPropertyName("authenticationTokenPath"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public string AuthenticationTokenPath { get; set; } + [JsonPropertyName("authentication")] + public SnowflakeAuthentication Authentication { get; set; } - /// - /// The passcode to use if the JWT token is encrypted. - /// - [JsonPropertyName("pkcs8Passcode"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public string Pkcs8Passcode { get; set; } + } - /// - /// The Snowflake authentication type. - /// - [JsonPropertyName("useHighPrecision")] - public bool UseHighPrecision { get; set; } = true; + public class SnowflakeAuthentication + { + public const string AuthOAuth = "auth_oauth"; + public const string AuthJwt = "auth_jwt"; + public const string AuthSnowflake = "auth_snowflake"; + + [JsonPropertyName(AuthOAuth)] + public OAuthAuthentication OAuth { get; set; } + + [JsonPropertyName(AuthJwt)] + public JwtAuthentication SnowflakeJwt { get; set; } + + [JsonPropertyName(AuthSnowflake)] + public DefaultAuthentication Default { get; set; } + } + + public class OAuthAuthentication + { + [JsonPropertyName("token")] + public string Token { get; set; } + + [JsonPropertyName("user")] + public string User { get; set; } + } + + public class JwtAuthentication + { + [JsonPropertyName("private_key")] + public string PrivateKey { get; set; } + + [JsonPropertyName("private_key_file")] + public string PrivateKeyFile { get; set; } + + [JsonPropertyName("private_key_pwd")] + public string PrivateKeyPassPhrase{ get; set; } + + [JsonPropertyName("user")] + public string User { get; set; } + } + + public class DefaultAuthentication + { + [JsonPropertyName("user")] + public string User { get; set; } + + [JsonPropertyName("password")] + public string Password { get; set; } } } diff --git a/csharp/test/Drivers/Snowflake/SnowflakeTestingUtils.cs b/csharp/test/Drivers/Snowflake/SnowflakeTestingUtils.cs index 18fcafb65f..a6c63868b9 100644 --- a/csharp/test/Drivers/Snowflake/SnowflakeTestingUtils.cs +++ b/csharp/test/Drivers/Snowflake/SnowflakeTestingUtils.cs @@ -34,6 +34,7 @@ internal class SnowflakeParameters public const string PASSWORD = "password"; public const string WAREHOUSE = "adbc.snowflake.sql.warehouse"; public const string AUTH_TYPE = "adbc.snowflake.sql.auth_type"; + public const string AUTH_TOKEN = "adbc.snowflake.sql.client_option.auth_token"; public const string HOST = "adbc.snowflake.sql.uri.host"; public const string PKCS8_VALUE = "adbc.snowflake.sql.client_option.jwt_private_key_pkcs8_value"; public const string PKCS8_PASS = "adbc.snowflake.sql.client_option.jwt_private_key_pkcs8_password"; @@ -78,11 +79,17 @@ out Dictionary parameters { SnowflakeParameters.USERNAME, testConfiguration.User }, { SnowflakeParameters.PASSWORD, testConfiguration.Password }, { SnowflakeParameters.WAREHOUSE, testConfiguration.Warehouse }, - { SnowflakeParameters.AUTH_TYPE, testConfiguration.AuthenticationType }, { SnowflakeParameters.USE_HIGH_PRECISION, testConfiguration.UseHighPrecision.ToString().ToLowerInvariant() } }; - if (!string.IsNullOrWhiteSpace(testConfiguration.Host)) + if(testConfiguration.Authentication.Default is not null) + { + parameters[SnowflakeParameters.AUTH_TYPE] = SnowflakeAuthentication.AuthSnowflake; + parameters[SnowflakeParameters.USERNAME] = testConfiguration.Authentication.Default.User; + parameters[SnowflakeParameters.PASSWORD] = testConfiguration.Authentication.Default.Password; + } + + if(!string.IsNullOrWhiteSpace(testConfiguration.Host)) { parameters[SnowflakeParameters.HOST] = testConfiguration.Host; } From 491ab8d4638391e5b60c32f1e29c5fe3bacbd0f5 Mon Sep 17 00:00:00 2001 From: William Date: Tue, 5 Dec 2023 20:20:34 +0100 Subject: [PATCH 02/51] fix(go/adbc/sqldriver): Fix nil pointer panics for query parameters (#1342) Thought I would contribute some fixes I've been using locally for the issues described in #1341 I have no previous experience with this repository or with the Arrow memory model so I would say it's likely I've gotten something wrong. Feel free to ask me to improve on my contributions or merely take them as inspiration for some other fix. Resolves #1341 --- go/adbc/sqldriver/driver.go | 11 ++- go/adbc/sqldriver/driver_internals_test.go | 95 ++++++++++++++++++++++ go/adbc/sqldriver/driver_test.go | 6 +- 3 files changed, 106 insertions(+), 6 deletions(-) diff --git a/go/adbc/sqldriver/driver.go b/go/adbc/sqldriver/driver.go index 4b83495f9b..775f3f78a8 100644 --- a/go/adbc/sqldriver/driver.go +++ b/go/adbc/sqldriver/driver.go @@ -22,6 +22,7 @@ import ( "database/sql" "database/sql/driver" "errors" + "fmt" "io" "reflect" "strconv" @@ -463,16 +464,20 @@ func arrFromVal(val any) arrow.Array { case []byte: dt = arrow.BinaryTypes.Binary buffers[1] = memory.NewBufferBytes(arrow.Int32Traits.CastToBytes([]int32{0, int32(len(v))})) - buffers[2] = memory.NewBufferBytes(v) + buffers = append(buffers, memory.NewBufferBytes(v)) case string: dt = arrow.BinaryTypes.String buffers[1] = memory.NewBufferBytes(arrow.Int32Traits.CastToBytes([]int32{0, int32(len(v))})) var buf = *(*[]byte)(unsafe.Pointer(&v)) (*reflect.SliceHeader)(unsafe.Pointer(&buf)).Cap = len(v) - buffers[2] = memory.NewBufferBytes(buf) + buffers = append(buffers, memory.NewBufferBytes(buf)) + default: + panic(fmt.Sprintf("unsupported type %T", val)) } for _, b := range buffers { - defer b.Release() + if b != nil { + defer b.Release() + } } data := array.NewData(dt, 1, buffers, nil, 0, 0) defer data.Release() diff --git a/go/adbc/sqldriver/driver_internals_test.go b/go/adbc/sqldriver/driver_internals_test.go index 8e9ce56535..9981a40ddb 100644 --- a/go/adbc/sqldriver/driver_internals_test.go +++ b/go/adbc/sqldriver/driver_internals_test.go @@ -19,6 +19,7 @@ package sqldriver import ( "database/sql/driver" + "encoding/base64" "fmt" "strings" "testing" @@ -273,3 +274,97 @@ func TestNextRowTypes(t *testing.T) { }) } } + +func TestArrFromVal(t *testing.T) { + tests := []struct { + value any + expectedDataType arrow.DataType + expectedStringValue string + }{ + { + value: true, + expectedDataType: arrow.FixedWidthTypes.Boolean, + expectedStringValue: "true", + }, + { + value: int8(1), + expectedDataType: arrow.PrimitiveTypes.Int8, + expectedStringValue: "1", + }, + { + value: uint8(1), + expectedDataType: arrow.PrimitiveTypes.Uint8, + expectedStringValue: "1", + }, + { + value: int16(1), + expectedDataType: arrow.PrimitiveTypes.Int16, + expectedStringValue: "1", + }, + { + value: uint16(1), + expectedDataType: arrow.PrimitiveTypes.Uint16, + expectedStringValue: "1", + }, + { + value: int32(1), + expectedDataType: arrow.PrimitiveTypes.Int32, + expectedStringValue: "1", + }, + { + value: uint32(1), + expectedDataType: arrow.PrimitiveTypes.Uint32, + expectedStringValue: "1", + }, + { + value: int64(1), + expectedDataType: arrow.PrimitiveTypes.Int64, + expectedStringValue: "1", + }, + { + value: uint64(1), + expectedDataType: arrow.PrimitiveTypes.Uint64, + expectedStringValue: "1", + }, + { + value: float32(1), + expectedDataType: arrow.PrimitiveTypes.Float32, + expectedStringValue: "1", + }, + { + value: float64(1), + expectedDataType: arrow.PrimitiveTypes.Float64, + expectedStringValue: "1", + }, + { + value: arrow.Date32FromTime(testTime), + expectedDataType: arrow.PrimitiveTypes.Date32, + expectedStringValue: testTime.UTC().Truncate(24 * time.Hour).Format("2006-01-02"), + }, + { + value: arrow.Date64FromTime(testTime), + expectedDataType: arrow.PrimitiveTypes.Date64, + expectedStringValue: testTime.UTC().Truncate(24 * time.Hour).Format("2006-01-02"), + }, + { + value: []byte("my-string"), + expectedDataType: arrow.BinaryTypes.Binary, + expectedStringValue: base64.StdEncoding.EncodeToString([]byte("my-string")), + }, + { + value: "my-string", + expectedDataType: arrow.BinaryTypes.String, + expectedStringValue: "my-string", + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("%d-%T", i, test.value), func(t *testing.T) { + arr := arrFromVal(test.value) + + assert.Equal(t, test.expectedDataType, arr.DataType()) + require.Equal(t, 1, arr.Len()) + assert.True(t, arr.IsValid(0)) + assert.Equal(t, test.expectedStringValue, arr.ValueStr(0)) + }) + } +} diff --git a/go/adbc/sqldriver/driver_test.go b/go/adbc/sqldriver/driver_test.go index 121e04f3fe..511ce9c921 100644 --- a/go/adbc/sqldriver/driver_test.go +++ b/go/adbc/sqldriver/driver_test.go @@ -38,7 +38,7 @@ func Example() { panic(err) } - rows, err := db.Query("SELECT 1") + rows, err := db.Query("SELECT ?", 1) if err != nil { panic(err) } @@ -66,8 +66,8 @@ func Example() { } // Output: - // [1] - // 1 + // [?] + // ? // true true // 1 } From 414379d59023a25f938b372495398347dbf3a191 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Wed, 6 Dec 2023 09:49:56 -0800 Subject: [PATCH 03/51] ci: temporary fix for Python version (#1337) I don't think this is the better long term option of what is described in https://github.com/apache/arrow-adbc/issues/1333#issuecomment-1836083476 but should get CI green for now --- .github/workflows/integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index acab5b053e..d0f403e6f0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -229,6 +229,7 @@ jobs: BUILD_ALL: "0" BUILD_DRIVER_MANAGER: "1" BUILD_DRIVER_POSTGRESQL: "1" + SETUPTOOLS_SCM_PRETEND_VERSION: "0.9.0" run: | ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" From cf3b5c0ec6279d2c3f8d48328cfe6dd92e250116 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Wed, 6 Dec 2023 10:06:30 -0800 Subject: [PATCH 04/51] build(python): add CMake build target for Python (#1257) --- c/CMakeLists.txt | 48 ++++++++++++++++ c/cmake_modules/GoUtils.cmake | 2 +- ci/scripts/python_build.ps1 | 76 +++---------------------- ci/scripts/python_build.sh | 55 ++++-------------- python/adbc_driver_flightsql/README.md | 12 ++++ python/adbc_driver_manager/README.md | 11 ++++ python/adbc_driver_postgresql/README.md | 11 ++++ python/adbc_driver_snowflake/README.md | 11 ++++ python/adbc_driver_sqlite/README.md | 11 ++++ 9 files changed, 123 insertions(+), 114 deletions(-) diff --git a/c/CMakeLists.txt b/c/CMakeLists.txt index 9b1ec5baa3..dac330b955 100644 --- a/c/CMakeLists.txt +++ b/c/CMakeLists.txt @@ -67,5 +67,53 @@ if(ADBC_INTEGRATION_DUCKDB) add_subdirectory(integration/duckdb) endif() +if(ADBC_BUILD_PYTHON) + find_package(Python3 REQUIRED COMPONENTS Interpreter Development) + + if(NOT ADBC_BUILD_SHARED) + message(FATAL_ERROR "Building Python requires ADBC_BUILD_SHARED=ON") + endif() + + # NB: the Python packages require the driver manager to be installed, + # but you don't technically need -DADBC_DRIVER_MANAGER=ON when installing + # other Python packages. To be safe then, we always install the driver + # manager package, regardless of the value of -DABC_DRIVER_MANAGER + # --config-settings eidtable_mode=compat required due to + # https://github.com/python/mypy/issues/13392 + add_custom_target(python + COMMAND ${Python3_EXECUTABLE} -m pip install --no-deps -e + "${REPOSITORY_ROOT}/python/adbc_driver_manager" + --config-settings editable_mode=compat) + + macro(adbc_install_python_package TARGET) + string(TOUPPER ${TARGET} ${TARGET}_LIB_upper) + add_custom_command(TARGET python + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E env "ADBC_${${TARGET}_LIB_upper}_\ +LIBRARY=$" ${Python3_EXECUTABLE} -m pip install + --no-deps -e + "${REPOSITORY_ROOT}/python/adbc_driver_${TARGET}" + COMMENT "pip installing the adbc_driver_${TARGET} library..." + DEPENDS $ + WORKING_DIRECTORY ${REPOSITORY_ROOT}) + endmacro() + + if(ADBC_DRIVER_POSTGRESQL) + adbc_install_python_package(postgresql) + endif() + + if(ADBC_DRIVER_SQLITE) + adbc_install_python_package(sqlite) + endif() + + if(ADBC_DRIVER_FLIGHTSQL) + adbc_install_python_package(flightsql) + endif() + + if(ADBC_DRIVER_SNOWFLAKE) + adbc_install_python_package(snowflake) + endif() +endif() + validate_config() config_summary_message() diff --git a/c/cmake_modules/GoUtils.cmake b/c/cmake_modules/GoUtils.cmake index aac6f5acf2..085d46fefd 100644 --- a/c/cmake_modules/GoUtils.cmake +++ b/c/cmake_modules/GoUtils.cmake @@ -153,7 +153,7 @@ function(add_go_lib GO_MOD_DIR GO_LIBNAME) add_custom_target(${GO_LIBNAME}_target ALL DEPENDS "${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}" "${LIBOUT_SHARED}.${ADBC_SO_VERSION}" "${LIBOUT_SHARED}") - add_library(${GO_LIBNAME}_shared SHARED IMPORTED) + add_library(${GO_LIBNAME}_shared SHARED IMPORTED GLOBAL) set_target_properties(${GO_LIBNAME}_shared PROPERTIES IMPORTED_LOCATION "${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}" diff --git a/ci/scripts/python_build.ps1 b/ci/scripts/python_build.ps1 index d1a5bffa43..14deea332b 100755 --- a/ci/scripts/python_build.ps1 +++ b/ci/scripts/python_build.ps1 @@ -20,7 +20,6 @@ $ErrorActionPreference = "Stop" $SourceDir = $Args[0] $BuildDir = $Args[1] -$InstallDir = if ($Args[2] -ne $null) { $Args[2] } else { Join-Path $BuildDir "local/" } $BuildAll = $env:BUILD_ALL -ne "0" $BuildDriverFlightSql = ($BuildAll -and (-not ($env:BUILD_DRIVER_FLIGHTSQL -eq "0"))) -or ($env:BUILD_DRIVER_FLIGHTSQL -eq "1") @@ -29,70 +28,11 @@ $BuildDriverPostgreSQL = ($BuildAll -and (-not ($env:BUILD_DRIVER_POSTGRESQL -eq $BuildDriverSqlite = ($BuildAll -and (-not ($env:BUILD_DRIVER_SQLITE -eq "0"))) -or ($env:BUILD_DRIVER_SQLITE -eq "1") $BuildDriverSnowflake = ($BuildAll -and (-not ($env:BUILD_DRIVER_SNOWFLAKE -eq "0"))) -or ($env:BUILD_DRIVER_SNOWFLAKE -eq "1") -function Build-Subproject { - $Subproject = $Args[0] - $SubprojectBuild = Join-Path $SourceDir "python\$($Subproject)" - - echo "============================================================" - echo "Building $($Subproject)" - echo "============================================================" - - pip install -e $SubprojectBuild - if (-not $?) { exit 1 } -} - -if ($BuildDriverManager) { - Build-Subproject adbc_driver_manager -} -if ($BuildDriverFlightSql) { - $env:ADBC_FLIGHTSQL_LIBRARY = Get-Childitem ` - -ErrorAction SilentlyContinue ` - -Path $InstallDir ` - -Recurse ` - -Include "adbc_driver_flightsql.dll","libadbc_driver_flightsql.so" | % {$_.FullName} - echo $env:ADBC_FLIGHTSQL_LIBRARY - if ($env:ADBC_FLIGHTSQL_LIBRARY -eq $null) { - echo "Could not find Flight SQL driver in $($InstallDir)" - exit 1 - } - Build-Subproject adbc_driver_flightsql -} -if ($BuildDriverPostgreSQL) { - $env:ADBC_POSTGRESQL_LIBRARY = Get-Childitem ` - -ErrorAction SilentlyContinue ` - -Path $InstallDir ` - -Recurse ` - -Include "adbc_driver_postgresql.dll","libadbc_driver_postgresql.so" | % {$_.FullName} - echo $env:ADBC_POSTGRESQL_LIBRARY - if ($env:ADBC_POSTGRESQL_LIBRARY -eq $null) { - echo "Could not find libpq driver in $($InstallDir)" - exit 1 - } - Build-Subproject adbc_driver_postgresql -} -if ($BuildDriverSqlite) { - $env:ADBC_SQLITE_LIBRARY = Get-Childitem ` - -ErrorAction SilentlyContinue ` - -Path $InstallDir ` - -Recurse ` - -Include "adbc_driver_sqlite.dll","libadbc_driver_sqlite.so" | % {$_.FullName} - echo $env:ADBC_SQLITE_LIBRARY - if ($env:ADBC_SQLITE_LIBRARY -eq $null) { - echo "Could not find SQLite driver in $($InstallDir)" - exit 1 - } - Build-Subproject adbc_driver_sqlite -} -if ($BuildDriverSnowflake) { - $env:ADBC_SNOWFLAKE_LIBRARY = Get-Childitem ` - -ErrorAction SilentlyContinue ` - -Path $InstallDir ` - -Recurse ` - -Include "adbc_driver_snowflake.dll","libadbc_driver_snowflake.so" | % {$_.FullName} - echo $env:ADBC_SNOWFLAKE_LIBRARY - if ($env:ADBC_SNOWFLAKE_LIBRARY -eq $null) { - echo "Could not find Snowflake driver in $($InstallDir)" - exit 1 - } - Build-Subproject adbc_driver_snowflake -} +cmake -S "$($SourceDir)\c" -B $BuildDir ` + -DADBC_DRIVER_MANAGER=$BuildDriverManager ` + -DADBC_DRIVER_FLIGHTSQL=$BuildDriverFlightSql ` + -DADBC_DRIVER_POSTGRESQL=$BuildDriverPostgreSQL ` + -DADBC_DRIVER_SQLITE=$BuildDriverSqlite ` + -DADBC_DRIVER_SNOWFLAKE=$BuildDriverSnowflake ` + -DADBC_BUILD_PYTHON=ON +cmake --build $BuildDir --target python diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh index 79aeaa28b9..533a82a38d 100755 --- a/ci/scripts/python_build.sh +++ b/ci/scripts/python_build.sh @@ -25,58 +25,23 @@ set -e : ${BUILD_DRIVER_SQLITE:=${BUILD_ALL}} : ${BUILD_DRIVER_SNOWFLAKE:=${BUILD_ALL}} -if [[ $(uname) = "Darwin" ]]; then - ADBC_LIBRARY_SUFFIX="dylib" -else - ADBC_LIBRARY_SUFFIX="so" -fi - -build_subproject() { - local -r source_dir="${1}" - local -r install_dir="${2}" - local -r subproject="${3}" - - if [[ "${subproject}" = "adbc_driver_flightsql" ]]; then - export ADBC_FLIGHTSQL_LIBRARY="${install_dir}/lib/libadbc_driver_flightsql.${ADBC_LIBRARY_SUFFIX}" - elif [[ "${subproject}" = "adbc_driver_postgresql" ]]; then - export ADBC_POSTGRESQL_LIBRARY="${install_dir}/lib/libadbc_driver_postgresql.${ADBC_LIBRARY_SUFFIX}" - elif [[ "${subproject}" = "adbc_driver_sqlite" ]]; then - export ADBC_SQLITE_LIBRARY="${install_dir}/lib/libadbc_driver_sqlite.${ADBC_LIBRARY_SUFFIX}" - elif [[ "${subproject}" = "adbc_driver_snowflake" ]]; then - export ADBC_SNOWFLAKE_LIBRARY="${install_dir}/lib/libadbc_driver_snowflake.${ADBC_LIBRARY_SUFFIX}" - fi - - python -m pip install --no-deps "${source_dir}/python/${subproject}" -} main() { local -r source_dir="${1}" local -r build_dir="${2}" - local install_dir="${3}" - - if [[ -z "${install_dir}" ]]; then - install_dir="${build_dir}/local" - fi - - if [[ "${BUILD_DRIVER_FLIGHTSQL}" -gt 0 ]]; then - build_subproject "${source_dir}" "${install_dir}" adbc_driver_flightsql - fi - - if [[ "${BUILD_DRIVER_MANAGER}" -gt 0 ]]; then - build_subproject "${source_dir}" "${install_dir}" adbc_driver_manager - fi - if [[ "${BUILD_DRIVER_POSTGRESQL}" -gt 0 ]]; then - build_subproject "${source_dir}" "${install_dir}" adbc_driver_postgresql - fi + set -x - if [[ "${BUILD_DRIVER_SQLITE}" -gt 0 ]]; then - build_subproject "${source_dir}" "${install_dir}" adbc_driver_sqlite - fi + cmake -S "${source_dir}/c" -B ${build_dir} \ + -DADBC_DRIVER_MANAGER=${BUILD_DRIVER_MANAGER} \ + -DADBC_DRIVER_FLIGHTSQL=${BUILD_DRIVER_FLIGHTSQL} \ + -DADBC_DRIVER_POSTGRESQL=${BUILD_DRIVER_POSTGRESQL} \ + -DADBC_DRIVER_SQLITE=${BUILD_DRIVER_SQLITE} \ + -DADBC_DRIVER_SNOWFLAKE=${BUILD_DRIVER_SNOWFLAKE} \ + -DADBC_BUILD_PYTHON=ON + cmake --build ${build_dir} --target python - if [[ "${BUILD_DRIVER_SNOWFLAKE}" -gt 0 ]]; then - build_subproject "${source_dir}" "${install_dir}" adbc_driver_snowflake - fi + set +x } main "$@" diff --git a/python/adbc_driver_flightsql/README.md b/python/adbc_driver_flightsql/README.md index 3320c6d853..ea3de1d23b 100644 --- a/python/adbc_driver_flightsql/README.md +++ b/python/adbc_driver_flightsql/README.md @@ -44,6 +44,18 @@ export ADBC_FLIGHTSQL_LIBRARY=/path/to/libadbc_driver_flightsql.so pip install --no-deps -e . ``` +For users building from the arrow-adbc source repository, you can alternately use CMake to manage library dependencies and set environment variables for you. Assuming you specify ``-DADBC_DRIVER_FLIGHTSQL=ON`` you can also add ``-DADBC_BUILD_PYTHON=ON`` to define a ``python`` target. + +For example, assuming you run cmake from the project root: + +```shell +cmake -S c -B build --preset debug -DADBC_BUILD_PYTHON=ON +cmake --build build --target python +``` + +will properly build and install the Python library for you. + + See [CONTRIBUTING.md](../../CONTRIBUTING.md) for details on the general build process. diff --git a/python/adbc_driver_manager/README.md b/python/adbc_driver_manager/README.md index bde5d400d8..e74e9d06f0 100644 --- a/python/adbc_driver_manager/README.md +++ b/python/adbc_driver_manager/README.md @@ -38,6 +38,17 @@ row-oriented API of the base DBAPI interface. Dependencies: a C++ compiler. +For users building from the arrow-adbc source repository, you can alternately use CMake to manage library dependencies and set environment variables for you. You can add ``-DADBC_BUILD_PYTHON=ON`` to define a ``python`` target. + +For example, assuming you run cmake from the project root: + +```shell +cmake -S c -B build --preset debug -DADBC_BUILD_PYTHON=ON +cmake --build build --target python +``` + +will properly build and install the Python library for you. + See [CONTRIBUTING.md](../../CONTRIBUTING.md) for details. ## Testing diff --git a/python/adbc_driver_postgresql/README.md b/python/adbc_driver_postgresql/README.md index cfa9ac99dd..2181b071d6 100644 --- a/python/adbc_driver_postgresql/README.md +++ b/python/adbc_driver_postgresql/README.md @@ -57,6 +57,17 @@ export ADBC_POSTGRESQL_LIBRARY=/path/to/libadbc_driver_postgresql.so pip install --no-deps -e . ``` +For users building from the arrow-adbc source repository, you can alternately use CMake to manage library dependencies and set environment variables for you. Assuming you specify ``-DADBC_DRIVER_POSTGRESQL=ON`` you can also add ``-DADBC_BUILD_PYTHON=ON`` to define a ``python`` target. + +For example, assuming you run cmake from the project root: + +```shell +cmake -S c -B build --preset debug -DADBC_BUILD_PYTHON=ON +cmake --build build --target python +``` + +will properly build and install the Python library for you. + See [CONTRIBUTING.md](../../CONTRIBUTING.md) for details on the general build process. diff --git a/python/adbc_driver_snowflake/README.md b/python/adbc_driver_snowflake/README.md index 0ca23f4852..17db5d69e6 100644 --- a/python/adbc_driver_snowflake/README.md +++ b/python/adbc_driver_snowflake/README.md @@ -44,6 +44,17 @@ export ADBC_SNOWFLAKE_LIBRARY=/path/to/libadbc_driver_snowflake.so pip install --no-deps -e . ``` +For users building from the arrow-adbc source repository, you can alternately use CMake to manage library dependencies and set environment variables for you. Assuming you specify ``-DADBC_DRIVER_SNOWFLAKE=ON`` you can also add ``-DADBC_BUILD_PYTHON=ON`` to define a ``python`` target. + +For example, assuming you run cmake from the project root: + +```shell +cmake -S c -B build --preset debug -DADBC_BUILD_PYTHON=ON +cmake --build build --target python +``` + +will properly build and install the Python library for you. + See [CONTRIBUTING.md](../../CONTRIBUTING.md) for details on the general build process. diff --git a/python/adbc_driver_sqlite/README.md b/python/adbc_driver_sqlite/README.md index 802bcdc79e..a324853a16 100644 --- a/python/adbc_driver_sqlite/README.md +++ b/python/adbc_driver_sqlite/README.md @@ -55,6 +55,17 @@ export ADBC_SQLITE_LIBRARY=/path/to/libadbc_driver_sqlite.so pip install --no-deps -e . ``` +For users building from the arrow-adbc source repository, you can alternately use CMake to manage library dependencies and set environment variables for you. Assuming you specify ``-DADBC_DRIVER_SQLITE=ON`` you can also add ``-DADBC_BUILD_PYTHON=ON`` to define a ``python`` target. + +For example, assuming you run cmake from the project root: + +```shell +cmake -S c -B build --preset debug -DADBC_BUILD_PYTHON=ON +cmake --build build --target python +``` + +will properly build and install the Python library for you. + See [CONTRIBUTING.md](../../CONTRIBUTING.md) for details on the general build process. From fd705e062a45d64491e8fbcc47ce6a53d494dc20 Mon Sep 17 00:00:00 2001 From: davidhcoe <13318837+davidhcoe@users.noreply.github.com> Date: Tue, 12 Dec 2023 16:17:34 -0500 Subject: [PATCH 05/51] build(csharp): script build, test, package and smoke test (#1345) Down payment toward https://github.com/apache/arrow-adbc/issues/1121: - Structures the projects for NuGet packaging --------- Co-authored-by: David Coe --- .github/CODEOWNERS | 2 +- csharp/.gitignore | 1 + csharp/Apache.Arrow.Adbc.sln | 29 ++++++--- csharp/Directory.Build.targets | 29 +++++++++ .../Apache.Arrow.Adbc.csproj | 11 +++- csharp/src/Apache.Arrow.Adbc/readme.md | 21 +++++++ .../Client/Apache.Arrow.Adbc.Client.csproj | 8 +++ .../Apache.Arrow.Adbc.Drivers.BigQuery.csproj | 8 +++ .../Drivers/BigQuery/BigQueryConnection.cs | 7 ++- ...Apache.Arrow.Adbc.Drivers.FlightSql.csproj | 2 +- ...rrow.Adbc.Drivers.Interop.Snowflake.csproj | 39 ++++++++++++ .../Snowflake/Build-SnowflakeDriver.ps1 | 36 +++++++++++ .../Snowflake/SnowflakeDriverLoader.cs | 63 +++++++++++++++++++ .../src/Drivers/Interop/Snowflake/readme.md | 30 +++++++++ .../Apache.Arrow.Adbc.Tests/DriverTests.cs | 2 +- csharp/test/Drivers/BigQuery/ClientTests.cs | 4 +- csharp/test/Drivers/BigQuery/DriverTests.cs | 2 +- .../BigQuery/Resources/BigQueryData.sql | 21 ------- csharp/test/Drivers/FlightSql/ClientTests.cs | 2 - ...dbc.Tests.Drivers.Interop.Snowflake.csproj | 9 ++- .../{ => Interop}/Snowflake/ClientTests.cs | 0 .../{ => Interop}/Snowflake/DriverTests.cs | 0 .../Snowflake/Resources/SnowflakeData.sql | 0 .../Snowflake/Resources/snowflakeconfig.json | 0 .../{ => Interop}/Snowflake/SnowflakeData.cs | 0 .../Snowflake/SnowflakeTestConfiguration.cs | 0 .../Snowflake/SnowflakeTestingUtils.cs | 16 +++-- .../{ => Interop}/Snowflake/ValueTests.cs | 0 .../Drivers/{ => Interop}/Snowflake/readme.md | 0 .../Apache.Arrow.Adbc.SmokeTests.csproj | 44 +++++++++++++ ...ow.Adbc.SmokeTests.Drivers.BigQuery.csproj | 33 ++++++++++ ...w.Adbc.SmokeTests.Drivers.FlightSql.csproj | 28 +++++++++ ...mokeTests.Drivers.Interop.Snowflake.csproj | 33 ++++++++++ csharp/test/SmokeTests/build.props | 8 +++ csharp/test/SmokeTests/readme.md | 28 +++++++++ dev/release/rat_exclude_files.txt | 1 + 36 files changed, 466 insertions(+), 51 deletions(-) create mode 100644 csharp/Directory.Build.targets create mode 100644 csharp/src/Apache.Arrow.Adbc/readme.md create mode 100644 csharp/src/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Drivers.Interop.Snowflake.csproj create mode 100644 csharp/src/Drivers/Interop/Snowflake/Build-SnowflakeDriver.ps1 create mode 100644 csharp/src/Drivers/Interop/Snowflake/SnowflakeDriverLoader.cs create mode 100644 csharp/src/Drivers/Interop/Snowflake/readme.md rename csharp/test/Drivers/{ => Interop}/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj (68%) rename csharp/test/Drivers/{ => Interop}/Snowflake/ClientTests.cs (100%) rename csharp/test/Drivers/{ => Interop}/Snowflake/DriverTests.cs (100%) rename csharp/test/Drivers/{ => Interop}/Snowflake/Resources/SnowflakeData.sql (100%) rename csharp/test/Drivers/{ => Interop}/Snowflake/Resources/snowflakeconfig.json (100%) rename csharp/test/Drivers/{ => Interop}/Snowflake/SnowflakeData.cs (100%) rename csharp/test/Drivers/{ => Interop}/Snowflake/SnowflakeTestConfiguration.cs (100%) rename csharp/test/Drivers/{ => Interop}/Snowflake/SnowflakeTestingUtils.cs (91%) rename csharp/test/Drivers/{ => Interop}/Snowflake/ValueTests.cs (100%) rename csharp/test/Drivers/{ => Interop}/Snowflake/readme.md (100%) create mode 100644 csharp/test/SmokeTests/Apache.Arrow.Adbc.SmokeTests/Apache.Arrow.Adbc.SmokeTests.csproj create mode 100644 csharp/test/SmokeTests/BigQuery/Apache.Arrow.Adbc.SmokeTests.Drivers.BigQuery.csproj create mode 100644 csharp/test/SmokeTests/FlightSql/Apache.Arrow.Adbc.SmokeTests.Drivers.FlightSql.csproj create mode 100644 csharp/test/SmokeTests/Interop/Snowflake/Apache.Arrow.Adbc.SmokeTests.Drivers.Interop.Snowflake.csproj create mode 100644 csharp/test/SmokeTests/build.props create mode 100644 csharp/test/SmokeTests/readme.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9d8d9a4039..1dd38f0869 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -27,7 +27,7 @@ adbc.h @lidavidm ## Implementations /c/ @lidavidm -/csharp/ @lidavidm +/csharp/ @lidavidm @CurtHagenlocher /glib/ @kou /java/ @lidavidm /go/ @zeroshade diff --git a/csharp/.gitignore b/csharp/.gitignore index e2b09a58d0..fbd47dd0ea 100644 --- a/csharp/.gitignore +++ b/csharp/.gitignore @@ -29,3 +29,4 @@ x64 artifacts/ TestResults/ +packages/ diff --git a/csharp/Apache.Arrow.Adbc.sln b/csharp/Apache.Arrow.Adbc.sln index 5dee7405ae..75719a275b 100644 --- a/csharp/Apache.Arrow.Adbc.sln +++ b/csharp/Apache.Arrow.Adbc.sln @@ -1,4 +1,3 @@ - Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.4.33403.182 @@ -25,12 +24,16 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Client", "Client", "{B61116 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Client", "src\Client\Apache.Arrow.Adbc.Client.csproj", "{A405F4A0-5938-4139-B2DF-ED9A05EC3D7C}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake", "test\Drivers\Snowflake\Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj", "{DEE943D1-A2D0-40CD-ABE0-2CB38E7CA156}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Drivers.BigQuery", "src\Drivers\BigQuery\Apache.Arrow.Adbc.Drivers.BigQuery.csproj", "{A748041C-EF9A-4E88-B6FB-9F2D6CB79170}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Tests.Drivers.BigQuery", "test\Drivers\BigQuery\Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj", "{EA43BB7C-BC00-4701-BDF4-367880C2495C}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Drivers.Interop.Snowflake", "src\Drivers\Interop\Snowflake\Apache.Arrow.Adbc.Drivers.Interop.Snowflake.csproj", "{30024B6F-7BC1-4574-BE5A-924FBD6EAF83}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake", "test\Drivers\Interop\Snowflake\Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj", "{8BE1EECC-3ACF-41B2-AF7D-1A67196FF6C7}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Flight", "src\arrow\csharp\src\Apache.Arrow.Flight\Apache.Arrow.Flight.csproj", "{0F41F702-A691-4E51-96A9-DF9813E47D8F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -61,10 +64,6 @@ Global {A405F4A0-5938-4139-B2DF-ED9A05EC3D7C}.Debug|Any CPU.Build.0 = Debug|Any CPU {A405F4A0-5938-4139-B2DF-ED9A05EC3D7C}.Release|Any CPU.ActiveCfg = Release|Any CPU {A405F4A0-5938-4139-B2DF-ED9A05EC3D7C}.Release|Any CPU.Build.0 = Release|Any CPU - {DEE943D1-A2D0-40CD-ABE0-2CB38E7CA156}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DEE943D1-A2D0-40CD-ABE0-2CB38E7CA156}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DEE943D1-A2D0-40CD-ABE0-2CB38E7CA156}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DEE943D1-A2D0-40CD-ABE0-2CB38E7CA156}.Release|Any CPU.Build.0 = Release|Any CPU {A748041C-EF9A-4E88-B6FB-9F2D6CB79170}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A748041C-EF9A-4E88-B6FB-9F2D6CB79170}.Debug|Any CPU.Build.0 = Debug|Any CPU {A748041C-EF9A-4E88-B6FB-9F2D6CB79170}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -73,6 +72,18 @@ Global {EA43BB7C-BC00-4701-BDF4-367880C2495C}.Debug|Any CPU.Build.0 = Debug|Any CPU {EA43BB7C-BC00-4701-BDF4-367880C2495C}.Release|Any CPU.ActiveCfg = Release|Any CPU {EA43BB7C-BC00-4701-BDF4-367880C2495C}.Release|Any CPU.Build.0 = Release|Any CPU + {30024B6F-7BC1-4574-BE5A-924FBD6EAF83}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {30024B6F-7BC1-4574-BE5A-924FBD6EAF83}.Debug|Any CPU.Build.0 = Debug|Any CPU + {30024B6F-7BC1-4574-BE5A-924FBD6EAF83}.Release|Any CPU.ActiveCfg = Release|Any CPU + {30024B6F-7BC1-4574-BE5A-924FBD6EAF83}.Release|Any CPU.Build.0 = Release|Any CPU + {8BE1EECC-3ACF-41B2-AF7D-1A67196FF6C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8BE1EECC-3ACF-41B2-AF7D-1A67196FF6C7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8BE1EECC-3ACF-41B2-AF7D-1A67196FF6C7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8BE1EECC-3ACF-41B2-AF7D-1A67196FF6C7}.Release|Any CPU.Build.0 = Release|Any CPU + {0F41F702-A691-4E51-96A9-DF9813E47D8F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0F41F702-A691-4E51-96A9-DF9813E47D8F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0F41F702-A691-4E51-96A9-DF9813E47D8F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0F41F702-A691-4E51-96A9-DF9813E47D8F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -84,9 +95,11 @@ Global {C7290227-E925-47E7-8B6B-A8B171645D58} = {5BD04C26-CE52-4893-8C1A-479705195CEF} {5C15E79C-19C4-4FF4-BB82-28754FE3966B} = {C7290227-E925-47E7-8B6B-A8B171645D58} {A405F4A0-5938-4139-B2DF-ED9A05EC3D7C} = {B6111602-2DC4-4B2F-9598-E3EE1972D3E4} - {DEE943D1-A2D0-40CD-ABE0-2CB38E7CA156} = {C7290227-E925-47E7-8B6B-A8B171645D58} {A748041C-EF9A-4E88-B6FB-9F2D6CB79170} = {FEB257A0-4FD3-495E-9A47-9E1649755445} {EA43BB7C-BC00-4701-BDF4-367880C2495C} = {C7290227-E925-47E7-8B6B-A8B171645D58} + {30024B6F-7BC1-4574-BE5A-924FBD6EAF83} = {FEB257A0-4FD3-495E-9A47-9E1649755445} + {8BE1EECC-3ACF-41B2-AF7D-1A67196FF6C7} = {C7290227-E925-47E7-8B6B-A8B171645D58} + {0F41F702-A691-4E51-96A9-DF9813E47D8F} = {9ADA26E0-F328-4466-908A-2FA506DBDF7D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {4795CF16-0FDB-4BE0-9768-5CF31564DC03} diff --git a/csharp/Directory.Build.targets b/csharp/Directory.Build.targets new file mode 100644 index 0000000000..bc66bcd405 --- /dev/null +++ b/csharp/Directory.Build.targets @@ -0,0 +1,29 @@ + + + + + + + $([System.IO.Path]::Combine('$(IntermediateOutputPath)','$(TargetFrameworkMoniker).AssemblyAttributes$(DefaultLanguageSourceExtension)')) + + + + + + + diff --git a/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj b/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj index c49113e729..b64b32d778 100644 --- a/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj +++ b/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj @@ -3,14 +3,19 @@ netstandard2.0;net6.0 true + readme.md - - - + + + true + \ + PreserveNewest + + diff --git a/csharp/src/Apache.Arrow.Adbc/readme.md b/csharp/src/Apache.Arrow.Adbc/readme.md new file mode 100644 index 0000000000..afd97fe652 --- /dev/null +++ b/csharp/src/Apache.Arrow.Adbc/readme.md @@ -0,0 +1,21 @@ + +# Apache Arrow ADBC + +An implementation of Arrow ADBC targeting .NET Standard 2.0 and .NET 6 or later. diff --git a/csharp/src/Client/Apache.Arrow.Adbc.Client.csproj b/csharp/src/Client/Apache.Arrow.Adbc.Client.csproj index 013c4a53d5..f44be6f229 100644 --- a/csharp/src/Client/Apache.Arrow.Adbc.Client.csproj +++ b/csharp/src/Client/Apache.Arrow.Adbc.Client.csproj @@ -1,8 +1,16 @@ netstandard2.0;net6.0 + readme.md + + + true + \ + PreserveNewest + + diff --git a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj index 368ff8181e..a7451c7bd1 100644 --- a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj +++ b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj @@ -1,6 +1,7 @@ netstandard2.0;net6.0 + readme.md @@ -11,4 +12,11 @@ + + + true + \ + PreserveNewest + + diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index dea7c9caaa..8e33596671 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -868,11 +868,16 @@ private ListArray CreateNestedListArray(List arrayList, IArrowType public override AdbcStatement CreateStatement() { - if (this.client == null || this.credential == null) + if (this.credential == null) { throw new InvalidOperationException(); } + if(this.client == null) + { + Open(); + } + BigQueryStatement statement = new BigQueryStatement(this.client, this.credential); statement.Options = ParseOptions(); return statement; diff --git a/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj b/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj index 836932bb6f..b6a7fb4c68 100644 --- a/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj +++ b/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj @@ -3,10 +3,10 @@ netstandard2.0;net6.0 - + diff --git a/csharp/src/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Drivers.Interop.Snowflake.csproj b/csharp/src/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Drivers.Interop.Snowflake.csproj new file mode 100644 index 0000000000..ece287a7c9 --- /dev/null +++ b/csharp/src/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Drivers.Interop.Snowflake.csproj @@ -0,0 +1,39 @@ + + + netstandard2.0;net472;net6.0 + readme.md + + + + + + + + + + true + lib\netstandard2.0 + PreserveNewest + + + true + lib\net472 + PreserveNewest + + + true + lib\net6.0 + PreserveNewest + + + + + true + \ + PreserveNewest + + + + + + diff --git a/csharp/src/Drivers/Interop/Snowflake/Build-SnowflakeDriver.ps1 b/csharp/src/Drivers/Interop/Snowflake/Build-SnowflakeDriver.ps1 new file mode 100644 index 0000000000..cbb69e343a --- /dev/null +++ b/csharp/src/Drivers/Interop/Snowflake/Build-SnowflakeDriver.ps1 @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Write-Host "Building the Snowflake ADBC Go driver" + +$location = Get-Location + +$file = "libadbc_driver_snowflake.dll" + +cd ..\..\..\..\..\go\adbc\pkg + +if(Test-Path $file) +{ + #because each framework build will run the script, avoid building it each time + $diff=((ls $file).LastWriteTime - (Get-Date)).TotalSeconds + if ($diff -gt -30) + { + Write-Output "Skipping build of $file because it is too recent" + exit + } +} +make $file +COPY $file $location diff --git a/csharp/src/Drivers/Interop/Snowflake/SnowflakeDriverLoader.cs b/csharp/src/Drivers/Interop/Snowflake/SnowflakeDriverLoader.cs new file mode 100644 index 0000000000..c2d247d7d3 --- /dev/null +++ b/csharp/src/Drivers/Interop/Snowflake/SnowflakeDriverLoader.cs @@ -0,0 +1,63 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.IO; +using Apache.Arrow.Adbc.C; + +namespace Apache.Arrow.Adbc.Drivers.Interop.Snowflake +{ + /// + /// Lightweight class for loading the Snowflake Go driver to .NET. + /// + public class SnowflakeDriverLoader + { + /// + /// Loads the Snowflake Go driver from the current directory using the default name and entry point. + /// + /// An based on the Snowflake Go driver. + /// + public static AdbcDriver LoadDriver() + { + string file = "libadbc_driver_snowflake.dll"; + + if(File.Exists(file)) + { + // get the full path because some .NET versions need it + file = Path.GetFullPath(file); + } + else + { + throw new FileNotFoundException($"Cound not find {file}"); + } + + return LoadDriver(file, "SnowflakeDriverInit"); + } + + /// + /// Loads the Snowflake Go driver from the current directory using the default name and entry point. + /// + /// The file to load. + /// The entry point of the file. + /// An based on the Snowflake Go driver. + public static AdbcDriver LoadDriver(string file, string entryPoint) + { + AdbcDriver snowflakeDriver = CAdbcDriverImporter.Load(file, entryPoint); + + return snowflakeDriver; + } + } +} diff --git a/csharp/src/Drivers/Interop/Snowflake/readme.md b/csharp/src/Drivers/Interop/Snowflake/readme.md new file mode 100644 index 0000000000..53a01bd773 --- /dev/null +++ b/csharp/src/Drivers/Interop/Snowflake/readme.md @@ -0,0 +1,30 @@ + + +# About +This project generates a NuGet package containing the Snowflake ADBC Go Driver for use in other .NET projects. It contains a lightweight loader for loading the driver. + +For details, see: + +[Snowflake Driver](https://arrow.apache.org/adbc/main/driver/snowflake.html) for docs + +[GitHub](https://github.com/apache/arrow-adbc/tree/main/go/adbc/driver/snowflake) for source code + +## Build the Snowflake Driver +Run the `Build-SnowflakeDriver.ps1` script to build the Snowflake driver. diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/DriverTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/DriverTests.cs index 70c0acac58..c9a5bea2ca 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/DriverTests.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/DriverTests.cs @@ -45,7 +45,7 @@ public static void CanExecuteQuery(QueryResult queryResult, long expectedNumberO count += nextBatch.Length; } - Assert.True(expectedNumberOfResults == count, "The parsed records differ from the specified amount"); + Assert.True(expectedNumberOfResults == count, $"The parsed records ({count}) differ from the expected amount ({expectedNumberOfResults})"); // if the values were set, make sure they are correct if (queryResult.RowCount != -1) diff --git a/csharp/test/Drivers/BigQuery/ClientTests.cs b/csharp/test/Drivers/BigQuery/ClientTests.cs index 3e9e761d9c..089951d382 100644 --- a/csharp/test/Drivers/BigQuery/ClientTests.cs +++ b/csharp/test/Drivers/BigQuery/ClientTests.cs @@ -17,8 +17,6 @@ using System; using System.Collections.Generic; -using System.Data; -using Apache.Arrow.Adbc.Client; using Apache.Arrow.Adbc.Drivers.BigQuery; using Apache.Arrow.Adbc.Tests.Xunit; using Xunit; @@ -54,7 +52,7 @@ public void CanClientExecuteUpdate() string[] queries = BigQueryTestingUtils.GetQueries(testConfiguration); - List expectedResults = new List() { -1, 1, 1, 1 }; + List expectedResults = new List() { -1, 1, 1 }; Tests.ClientTests.CanClientExecuteUpdate(adbcConnection, testConfiguration, queries, expectedResults); } diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 9d470475b6..c94c737e23 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -53,7 +53,7 @@ public void CanExecuteUpdate() string[] queries = BigQueryTestingUtils.GetQueries(testConfiguration); - List expectedResults = new List() { -1, 1, 1, 1 }; + List expectedResults = new List() { -1, 1, 1 }; for (int i = 0; i < queries.Length; i++) { diff --git a/csharp/test/Drivers/BigQuery/Resources/BigQueryData.sql b/csharp/test/Drivers/BigQuery/Resources/BigQueryData.sql index 242a2f92f4..96e37dcdff 100644 --- a/csharp/test/Drivers/BigQuery/Resources/BigQueryData.sql +++ b/csharp/test/Drivers/BigQuery/Resources/BigQueryData.sql @@ -77,24 +77,3 @@ VALUES ( ARRAY[4, 5, 6], STRUCT('Jane Doe', 40) ); - -INSERT INTO {ADBC_CATALOG}.{ADBC_DATASET}.{ADBC_TABLE} ( - id, number, decimal, - big_decimal, - is_active, - name, data, - date, time, datetime, timestamp, - point, numbers, - person -) -VALUES ( - null, null, null, - null, - null, - null, - null, - null, null, null, null, - null, - null, - null -); diff --git a/csharp/test/Drivers/FlightSql/ClientTests.cs b/csharp/test/Drivers/FlightSql/ClientTests.cs index 4b413c9be0..b87c8b7cea 100644 --- a/csharp/test/Drivers/FlightSql/ClientTests.cs +++ b/csharp/test/Drivers/FlightSql/ClientTests.cs @@ -15,9 +15,7 @@ * limitations under the License. */ -using System; using System.Collections.Generic; -using Apache.Arrow.Adbc.Client; using Apache.Arrow.Adbc.Drivers.FlightSql; using Xunit; diff --git a/csharp/test/Drivers/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj similarity index 68% rename from csharp/test/Drivers/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj rename to csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj index 6fa40d80b0..a7dba73d0e 100644 --- a/csharp/test/Drivers/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj +++ b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj @@ -12,11 +12,10 @@ - - - - - + + + + PreserveNewest diff --git a/csharp/test/Drivers/Snowflake/ClientTests.cs b/csharp/test/Drivers/Interop/Snowflake/ClientTests.cs similarity index 100% rename from csharp/test/Drivers/Snowflake/ClientTests.cs rename to csharp/test/Drivers/Interop/Snowflake/ClientTests.cs diff --git a/csharp/test/Drivers/Snowflake/DriverTests.cs b/csharp/test/Drivers/Interop/Snowflake/DriverTests.cs similarity index 100% rename from csharp/test/Drivers/Snowflake/DriverTests.cs rename to csharp/test/Drivers/Interop/Snowflake/DriverTests.cs diff --git a/csharp/test/Drivers/Snowflake/Resources/SnowflakeData.sql b/csharp/test/Drivers/Interop/Snowflake/Resources/SnowflakeData.sql similarity index 100% rename from csharp/test/Drivers/Snowflake/Resources/SnowflakeData.sql rename to csharp/test/Drivers/Interop/Snowflake/Resources/SnowflakeData.sql diff --git a/csharp/test/Drivers/Snowflake/Resources/snowflakeconfig.json b/csharp/test/Drivers/Interop/Snowflake/Resources/snowflakeconfig.json similarity index 100% rename from csharp/test/Drivers/Snowflake/Resources/snowflakeconfig.json rename to csharp/test/Drivers/Interop/Snowflake/Resources/snowflakeconfig.json diff --git a/csharp/test/Drivers/Snowflake/SnowflakeData.cs b/csharp/test/Drivers/Interop/Snowflake/SnowflakeData.cs similarity index 100% rename from csharp/test/Drivers/Snowflake/SnowflakeData.cs rename to csharp/test/Drivers/Interop/Snowflake/SnowflakeData.cs diff --git a/csharp/test/Drivers/Snowflake/SnowflakeTestConfiguration.cs b/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestConfiguration.cs similarity index 100% rename from csharp/test/Drivers/Snowflake/SnowflakeTestConfiguration.cs rename to csharp/test/Drivers/Interop/Snowflake/SnowflakeTestConfiguration.cs diff --git a/csharp/test/Drivers/Snowflake/SnowflakeTestingUtils.cs b/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs similarity index 91% rename from csharp/test/Drivers/Snowflake/SnowflakeTestingUtils.cs rename to csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs index a6c63868b9..d1e7b9dc74 100644 --- a/csharp/test/Drivers/Snowflake/SnowflakeTestingUtils.cs +++ b/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs @@ -20,8 +20,7 @@ using System.IO; using System.Linq; using System.Text; -using Apache.Arrow.Adbc.C; -using Xunit; +using Apache.Arrow.Adbc.Drivers.Interop.Snowflake; namespace Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake { @@ -100,7 +99,7 @@ out Dictionary parameters } Dictionary options = new Dictionary() { }; - AdbcDriver snowflakeDriver = CAdbcDriverImporter.Load(testConfiguration.DriverPath, testConfiguration.DriverEntryPoint); + AdbcDriver snowflakeDriver = GetSnowflakeAdbcDriver(testConfiguration); return snowflakeDriver; } @@ -116,7 +115,16 @@ internal static AdbcDriver GetSnowflakeAdbcDriver( SnowflakeTestConfiguration testConfiguration ) { - AdbcDriver snowflakeDriver = CAdbcDriverImporter.Load(testConfiguration.DriverPath, testConfiguration.DriverEntryPoint); + AdbcDriver snowflakeDriver; + + if (testConfiguration == null || string.IsNullOrEmpty(testConfiguration.DriverPath) || string.IsNullOrEmpty(testConfiguration.DriverEntryPoint)) + { + snowflakeDriver = SnowflakeDriverLoader.LoadDriver(); + } + else + { + snowflakeDriver = SnowflakeDriverLoader.LoadDriver(testConfiguration.DriverPath, testConfiguration.DriverEntryPoint); + } return snowflakeDriver; } diff --git a/csharp/test/Drivers/Snowflake/ValueTests.cs b/csharp/test/Drivers/Interop/Snowflake/ValueTests.cs similarity index 100% rename from csharp/test/Drivers/Snowflake/ValueTests.cs rename to csharp/test/Drivers/Interop/Snowflake/ValueTests.cs diff --git a/csharp/test/Drivers/Snowflake/readme.md b/csharp/test/Drivers/Interop/Snowflake/readme.md similarity index 100% rename from csharp/test/Drivers/Snowflake/readme.md rename to csharp/test/Drivers/Interop/Snowflake/readme.md diff --git a/csharp/test/SmokeTests/Apache.Arrow.Adbc.SmokeTests/Apache.Arrow.Adbc.SmokeTests.csproj b/csharp/test/SmokeTests/Apache.Arrow.Adbc.SmokeTests/Apache.Arrow.Adbc.SmokeTests.csproj new file mode 100644 index 0000000000..08ad0ea09d --- /dev/null +++ b/csharp/test/SmokeTests/Apache.Arrow.Adbc.SmokeTests/Apache.Arrow.Adbc.SmokeTests.csproj @@ -0,0 +1,44 @@ + + + + + net472;net6.0 + disable + false + true + True + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/csharp/test/SmokeTests/BigQuery/Apache.Arrow.Adbc.SmokeTests.Drivers.BigQuery.csproj b/csharp/test/SmokeTests/BigQuery/Apache.Arrow.Adbc.SmokeTests.Drivers.BigQuery.csproj new file mode 100644 index 0000000000..d032501e43 --- /dev/null +++ b/csharp/test/SmokeTests/BigQuery/Apache.Arrow.Adbc.SmokeTests.Drivers.BigQuery.csproj @@ -0,0 +1,33 @@ + + + + net472;net6.0 + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + PreserveNewest + + + + + + diff --git a/csharp/test/SmokeTests/FlightSql/Apache.Arrow.Adbc.SmokeTests.Drivers.FlightSql.csproj b/csharp/test/SmokeTests/FlightSql/Apache.Arrow.Adbc.SmokeTests.Drivers.FlightSql.csproj new file mode 100644 index 0000000000..7647af2d76 --- /dev/null +++ b/csharp/test/SmokeTests/FlightSql/Apache.Arrow.Adbc.SmokeTests.Drivers.FlightSql.csproj @@ -0,0 +1,28 @@ + + + net472;net6.0 + disable + False + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + Never + + + diff --git a/csharp/test/SmokeTests/Interop/Snowflake/Apache.Arrow.Adbc.SmokeTests.Drivers.Interop.Snowflake.csproj b/csharp/test/SmokeTests/Interop/Snowflake/Apache.Arrow.Adbc.SmokeTests.Drivers.Interop.Snowflake.csproj new file mode 100644 index 0000000000..fa722ea644 --- /dev/null +++ b/csharp/test/SmokeTests/Interop/Snowflake/Apache.Arrow.Adbc.SmokeTests.Drivers.Interop.Snowflake.csproj @@ -0,0 +1,33 @@ + + + + net472;net6.0 + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + PreserveNewest + + + + + + diff --git a/csharp/test/SmokeTests/build.props b/csharp/test/SmokeTests/build.props new file mode 100644 index 0000000000..2b3620f03f --- /dev/null +++ b/csharp/test/SmokeTests/build.props @@ -0,0 +1,8 @@ + + + + $(Version) + $(Version) + $(Version) + + diff --git a/csharp/test/SmokeTests/readme.md b/csharp/test/SmokeTests/readme.md new file mode 100644 index 0000000000..66f51791e0 --- /dev/null +++ b/csharp/test/SmokeTests/readme.md @@ -0,0 +1,28 @@ + + +# Smoke Tests +The smoke tests are designed to work with the NuGet packages that are generated from the `dotnet pack` command. These are run automatically as part of the `Build-All.ps1` script. + +## Smoke Test Structure +Smoke tests are similar to the unit tests, and actually leverage the code from unit tests, but instead target using the NuGet packages (instead of the project references) to validate they function correctly. + +Since these are smoke tests, they run a scaled down version of what the unit tests run. To do this, each `*.Tests` project has an equivalent `*.SmokeTests` project. For example, there is both `Apache.Arrow.Adbc.Tests` and `Apache.Arrow.Adbc.SmokeTests`. The smoke tests use linking to add items from the `*.Test` project and make them appear to be included, but they are actually located in the original project. + +The smoke tests only run the ClientTests because those exercise multiple NuGet packages and the underlying driver. diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 9e95dcfcff..4bfa977102 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -10,6 +10,7 @@ ci/linux-packages/changelog ci/linux-packages/*.install csharp/*.sln csharp/*.csproj +csharp/*.props dev/release/rat_exclude_files.txt docs/source/format/*.drawio docs/source/format/*.svg From 9544887f37edc30bed43a93ee4be615df3f1b014 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 13 Dec 2023 20:21:28 +0100 Subject: [PATCH 06/51] feat(python/adbc_driver_manager): export handles through python Arrow Capsule interface (#1346) Addresses https://github.com/apache/arrow-adbc/issues/70 This PR adds the dunder methods to the Handle classes of the low-level interface (which already enables using the low-level interface without pyarrow and with the capsule protocol). And secondly, in the places that accept data (eg ingest/bind), it now also accepts objects that implement the dunders in addition to hardcoded support for pyarrow. --------- Co-authored-by: David Li --- .../adbc_driver_manager/_lib.pxd | 9 +- .../adbc_driver_manager/_lib.pyx | 110 +++++++++++++++--- .../adbc_driver_manager/dbapi.py | 51 +++++--- .../adbc_driver_manager/tests/test_dbapi.py | 28 +++++ .../tests/test_lowlevel.py | 72 ++++++++++++ 5 files changed, 237 insertions(+), 33 deletions(-) diff --git a/python/adbc_driver_manager/adbc_driver_manager/_lib.pxd b/python/adbc_driver_manager/adbc_driver_manager/_lib.pxd index 358a09aa19..e9ea833c36 100644 --- a/python/adbc_driver_manager/adbc_driver_manager/_lib.pxd +++ b/python/adbc_driver_manager/adbc_driver_manager/_lib.pxd @@ -22,10 +22,15 @@ from libc.stdint cimport int32_t, int64_t, uint8_t, uint32_t cdef extern from "adbc.h" nogil: # C ABI + + ctypedef void (*CArrowSchemaRelease)(void*) + ctypedef void (*CArrowArrayRelease)(void*) + cdef struct CArrowSchema"ArrowSchema": - pass + CArrowSchemaRelease release + cdef struct CArrowArray"ArrowArray": - pass + CArrowArrayRelease release ctypedef int (*CArrowArrayStreamGetLastError)(void*) ctypedef int (*CArrowArrayStreamGetNext)(void*, CArrowArray*) diff --git a/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx b/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx index ced8870ec9..91139100bb 100644 --- a/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx +++ b/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx @@ -24,10 +24,15 @@ import threading import typing from typing import List, Tuple +cimport cpython import cython from cpython.bytes cimport PyBytes_FromStringAndSize +from cpython.pycapsule cimport ( + PyCapsule_GetPointer, PyCapsule_New, PyCapsule_CheckExact +) from libc.stdint cimport int32_t, int64_t, uint8_t, uint32_t, uintptr_t -from libc.string cimport memset +from libc.stdlib cimport malloc, free +from libc.string cimport memcpy, memset from libcpp.vector cimport vector as c_vector if typing.TYPE_CHECKING: @@ -304,9 +309,29 @@ cdef class _AdbcHandle: f"with open {self._child_type}") +cdef void pycapsule_schema_deleter(object capsule) noexcept: + cdef CArrowSchema* allocated = PyCapsule_GetPointer( + capsule, "arrow_schema" + ) + if allocated.release != NULL: + allocated.release(allocated) + free(allocated) + + +cdef void pycapsule_stream_deleter(object capsule) noexcept: + cdef CArrowArrayStream* allocated = PyCapsule_GetPointer( + capsule, "arrow_array_stream" + ) + if allocated.release != NULL: + allocated.release(allocated) + free(allocated) + + cdef class ArrowSchemaHandle: """ A wrapper for an allocated ArrowSchema. + + This object implements the Arrow PyCapsule interface. """ cdef: CArrowSchema schema @@ -316,23 +341,42 @@ cdef class ArrowSchemaHandle: """The address of the ArrowSchema.""" return &self.schema + def __arrow_c_schema__(self) -> object: + """Consume this object to get a PyCapsule.""" + # Reference: + # https://arrow.apache.org/docs/dev/format/CDataInterface/PyCapsuleInterface.html#create-a-pycapsule + cdef CArrowSchema* allocated = malloc(sizeof(CArrowSchema)) + allocated.release = NULL + capsule = PyCapsule_New( + allocated, "arrow_schema", &pycapsule_schema_deleter, + ) + memcpy(allocated, &self.schema, sizeof(CArrowSchema)) + self.schema.release = NULL + return capsule + cdef class ArrowArrayHandle: """ A wrapper for an allocated ArrowArray. + + This object implements the Arrow PyCapsule interface. """ cdef: CArrowArray array @property def address(self) -> int: - """The address of the ArrowArray.""" + """ + The address of the ArrowArray. + """ return &self.array cdef class ArrowArrayStreamHandle: """ A wrapper for an allocated ArrowArrayStream. + + This object implements the Arrow PyCapsule interface. """ cdef: CArrowArrayStream stream @@ -342,6 +386,21 @@ cdef class ArrowArrayStreamHandle: """The address of the ArrowArrayStream.""" return &self.stream + def __arrow_c_stream__(self, requested_schema=None) -> object: + """Consume this object to get a PyCapsule.""" + if requested_schema is not None: + raise NotImplementedError("requested_schema") + + cdef CArrowArrayStream* allocated = \ + malloc(sizeof(CArrowArrayStream)) + allocated.release = NULL + capsule = PyCapsule_New( + allocated, "arrow_array_stream", &pycapsule_stream_deleter, + ) + memcpy(allocated, &self.stream, sizeof(CArrowArrayStream)) + self.stream.release = NULL + return capsule + class GetObjectsDepth(enum.IntEnum): ALL = ADBC_OBJECT_DEPTH_ALL @@ -1000,32 +1059,47 @@ cdef class AdbcStatement(_AdbcHandle): connection._open_child() - def bind(self, data, schema) -> None: + def bind(self, data, schema=None) -> None: """ Bind an ArrowArray to this statement. Parameters ---------- - data : int or ArrowArrayHandle - schema : int or ArrowSchemaHandle + data : PyCapsule or int or ArrowArrayHandle + schema : PyCapsule or int or ArrowSchemaHandle """ cdef CAdbcError c_error = empty_error() cdef CArrowArray* c_array cdef CArrowSchema* c_schema - if isinstance(data, ArrowArrayHandle): + if hasattr(data, "__arrow_c_array__") and not isinstance(data, ArrowArrayHandle): + if schema is not None: + raise ValueError( + "Can not provide a schema when passing Arrow-compatible " + "data that implements the Arrow PyCapsule Protocol" + ) + schema, data = data.__arrow_c_array__() + + if PyCapsule_CheckExact(data): + c_array = PyCapsule_GetPointer(data, "arrow_array") + elif isinstance(data, ArrowArrayHandle): c_array = &( data).array elif isinstance(data, int): c_array = data else: - raise TypeError(f"data must be int or ArrowArrayHandle, not {type(data)}") - - if isinstance(schema, ArrowSchemaHandle): + raise TypeError( + "data must be Arrow-compatible data (implementing the Arrow PyCapsule " + f"Protocol), a PyCapsule, int or ArrowArrayHandle, not {type(data)}" + ) + + if PyCapsule_CheckExact(schema): + c_schema = PyCapsule_GetPointer(schema, "arrow_schema") + elif isinstance(schema, ArrowSchemaHandle): c_schema = &( schema).schema elif isinstance(schema, int): c_schema = schema else: - raise TypeError(f"schema must be int or ArrowSchemaHandle, " + raise TypeError("schema must be a PyCapsule, int or ArrowSchemaHandle, " f"not {type(schema)}") with nogil: @@ -1042,17 +1116,27 @@ cdef class AdbcStatement(_AdbcHandle): Parameters ---------- - stream : int or ArrowArrayStreamHandle + stream : PyCapsule or int or ArrowArrayStreamHandle """ cdef CAdbcError c_error = empty_error() cdef CArrowArrayStream* c_stream - if isinstance(stream, ArrowArrayStreamHandle): + if ( + hasattr(stream, "__arrow_c_stream__") + and not isinstance(stream, ArrowArrayStreamHandle) + ): + stream = stream.__arrow_c_stream__() + + if PyCapsule_CheckExact(stream): + c_stream = PyCapsule_GetPointer( + stream, "arrow_array_stream" + ) + elif isinstance(stream, ArrowArrayStreamHandle): c_stream = &( stream).stream elif isinstance(stream, int): c_stream = stream else: - raise TypeError(f"data must be int or ArrowArrayStreamHandle, " + raise TypeError(f"data must be a PyCapsule, int or ArrowArrayStreamHandle, " f"not {type(stream)}") with nogil: diff --git a/python/adbc_driver_manager/adbc_driver_manager/dbapi.py b/python/adbc_driver_manager/adbc_driver_manager/dbapi.py index 8edcdf4f58..4c36ad5cbd 100644 --- a/python/adbc_driver_manager/adbc_driver_manager/dbapi.py +++ b/python/adbc_driver_manager/adbc_driver_manager/dbapi.py @@ -612,17 +612,21 @@ def close(self): self._closed = True def _bind(self, parameters) -> None: - if isinstance(parameters, pyarrow.RecordBatch): + if hasattr(parameters, "__arrow_c_array__"): + self._stmt.bind(parameters) + elif hasattr(parameters, "__arrow_c_stream__"): + self._stmt.bind_stream(parameters) + elif isinstance(parameters, pyarrow.RecordBatch): arr_handle = _lib.ArrowArrayHandle() sch_handle = _lib.ArrowSchemaHandle() parameters._export_to_c(arr_handle.address, sch_handle.address) self._stmt.bind(arr_handle, sch_handle) - return - if isinstance(parameters, pyarrow.Table): - parameters = parameters.to_reader() - stream_handle = _lib.ArrowArrayStreamHandle() - parameters._export_to_c(stream_handle.address) - self._stmt.bind_stream(stream_handle) + else: + if isinstance(parameters, pyarrow.Table): + parameters = parameters.to_reader() + stream_handle = _lib.ArrowArrayStreamHandle() + parameters._export_to_c(stream_handle.address) + self._stmt.bind_stream(stream_handle) def _prepare_execute(self, operation, parameters=None) -> None: self._results = None @@ -639,9 +643,7 @@ def _prepare_execute(self, operation, parameters=None) -> None: # Not all drivers support it pass - if isinstance( - parameters, (pyarrow.RecordBatch, pyarrow.Table, pyarrow.RecordBatchReader) - ): + if _is_arrow_data(parameters): self._bind(parameters) elif parameters: rb = pyarrow.record_batch( @@ -668,7 +670,6 @@ def execute(self, operation: Union[bytes, str], parameters=None) -> None: self._prepare_execute(operation, parameters) handle, self._rowcount = self._stmt.execute_query() self._results = _RowIterator( - # pyarrow.RecordBatchReader._import_from_c(handle.address) _reader.AdbcRecordBatchReader._import_from_c(handle.address) ) @@ -683,7 +684,7 @@ def executemany(self, operation: Union[bytes, str], seq_of_parameters) -> None: operation : bytes or str The query to execute. Pass SQL queries as strings, (serialized) Substrait plans as bytes. - parameters + seq_of_parameters Parameters to bind. Can be a list of Python sequences, or an Arrow record batch, table, or record batch reader. If None, then the query will be executed once, else it will @@ -695,10 +696,7 @@ def executemany(self, operation: Union[bytes, str], seq_of_parameters) -> None: self._stmt.set_sql_query(operation) self._stmt.prepare() - if isinstance( - seq_of_parameters, - (pyarrow.RecordBatch, pyarrow.Table, pyarrow.RecordBatchReader), - ): + if _is_arrow_data(seq_of_parameters): arrow_parameters = seq_of_parameters elif seq_of_parameters: arrow_parameters = pyarrow.RecordBatch.from_pydict( @@ -806,7 +804,10 @@ def adbc_ingest( table_name The table to insert into. data - The Arrow data to insert. + The Arrow data to insert. This can be a pyarrow RecordBatch, Table + or RecordBatchReader, or any Arrow-compatible data that implements + the Arrow PyCapsule Protocol (i.e. has an ``__arrow_c_array__`` + or ``__arrow_c_stream__ ``method). mode How to deal with existing data: @@ -878,7 +879,11 @@ def adbc_ingest( except NotSupportedError: pass - if isinstance(data, pyarrow.RecordBatch): + if hasattr(data, "__arrow_c_array__"): + self._stmt.bind(data) + elif hasattr(data, "__arrow_c_stream__"): + self._stmt.bind_stream(data) + elif isinstance(data, pyarrow.RecordBatch): array = _lib.ArrowArrayHandle() schema = _lib.ArrowSchemaHandle() data._export_to_c(array.address, schema.address) @@ -1151,3 +1156,13 @@ def _warn_unclosed(name): category=ResourceWarning, stacklevel=2, ) + + +def _is_arrow_data(data): + return ( + hasattr(data, "__arrow_c_array__") + or hasattr(data, "__arrow_c_stream__") + or isinstance( + data, (pyarrow.RecordBatch, pyarrow.Table, pyarrow.RecordBatchReader) + ) + ) diff --git a/python/adbc_driver_manager/tests/test_dbapi.py b/python/adbc_driver_manager/tests/test_dbapi.py index 52b8e1316c..20990eff43 100644 --- a/python/adbc_driver_manager/tests/test_dbapi.py +++ b/python/adbc_driver_manager/tests/test_dbapi.py @@ -134,6 +134,22 @@ def test_get_table_types(sqlite): assert sqlite.adbc_get_table_types() == ["table", "view"] +class ArrayWrapper: + def __init__(self, array): + self.array = array + + def __arrow_c_array__(self, requested_schema=None): + return self.array.__arrow_c_array__(requested_schema=requested_schema) + + +class StreamWrapper: + def __init__(self, stream): + self.stream = stream + + def __arrow_c_stream__(self, requested_schema=None): + return self.stream.__arrow_c_stream__(requested_schema=requested_schema) + + @pytest.mark.parametrize( "data", [ @@ -142,6 +158,12 @@ def test_get_table_types(sqlite): lambda: pyarrow.table( [[1, 2], ["foo", ""]], names=["ints", "strs"] ).to_reader(), + lambda: ArrayWrapper( + pyarrow.record_batch([[1, 2], ["foo", ""]], names=["ints", "strs"]) + ), + lambda: StreamWrapper( + pyarrow.table([[1, 2], ["foo", ""]], names=["ints", "strs"]) + ), ], ) @pytest.mark.sqlite @@ -237,6 +259,8 @@ def test_query_fetch_df(sqlite): (1.0, 2), pyarrow.record_batch([[1.0], [2]], names=["float", "int"]), pyarrow.table([[1.0], [2]], names=["float", "int"]), + ArrayWrapper(pyarrow.record_batch([[1.0], [2]], names=["float", "int"])), + StreamWrapper(pyarrow.table([[1.0], [2]], names=["float", "int"])), ], ) def test_execute_parameters(sqlite, parameters): @@ -253,6 +277,10 @@ def test_execute_parameters(sqlite, parameters): pyarrow.record_batch([[1, 3], ["a", None]], names=["float", "str"]), pyarrow.table([[1, 3], ["a", None]], names=["float", "str"]), pyarrow.table([[1, 3], ["a", None]], names=["float", "str"]).to_batches()[0], + ArrayWrapper( + pyarrow.record_batch([[1, 3], ["a", None]], names=["float", "str"]) + ), + StreamWrapper(pyarrow.table([[1, 3], ["a", None]], names=["float", "str"])), ((x, y) for x, y in ((1, "a"), (3, None))), ], ) diff --git a/python/adbc_driver_manager/tests/test_lowlevel.py b/python/adbc_driver_manager/tests/test_lowlevel.py index 15d98e5389..98c8721ca0 100644 --- a/python/adbc_driver_manager/tests/test_lowlevel.py +++ b/python/adbc_driver_manager/tests/test_lowlevel.py @@ -390,3 +390,75 @@ def test_child_tracking(sqlite): RuntimeError, match="Cannot close AdbcDatabase with open AdbcConnection" ): db.close() + + +@pytest.mark.sqlite +def test_pycapsule(sqlite): + _, conn = sqlite + handle = conn.get_table_types() + with pyarrow.RecordBatchReader._import_from_c_capsule( + handle.__arrow_c_stream__() + ) as reader: + reader.read_all() + + # set up some data + data = pyarrow.record_batch( + [ + [1, 2, 3, 4], + ["a", "b", "c", "d"], + ], + names=["ints", "strs"], + ) + table = pyarrow.Table.from_batches([data]) + + with adbc_driver_manager.AdbcStatement(conn) as stmt: + stmt.set_options(**{adbc_driver_manager.INGEST_OPTION_TARGET_TABLE: "foo"}) + schema_capsule, array_capsule = data.__arrow_c_array__() + stmt.bind(array_capsule, schema_capsule) + stmt.execute_update() + + with adbc_driver_manager.AdbcStatement(conn) as stmt: + stmt.set_options(**{adbc_driver_manager.INGEST_OPTION_TARGET_TABLE: "bar"}) + stream_capsule = data.__arrow_c_stream__() + stmt.bind_stream(stream_capsule) + stmt.execute_update() + + # importing a schema + + handle = conn.get_table_schema(catalog=None, db_schema=None, table_name="foo") + assert data.schema == pyarrow.schema(handle) + # ensure consumed schema was marked as such + with pytest.raises(ValueError, match="Cannot import released ArrowSchema"): + pyarrow.schema(handle) + + # smoke test for the capsule calling release + capsule = conn.get_table_schema( + catalog=None, db_schema=None, table_name="foo" + ).__arrow_c_schema__() + del capsule + + # importing a stream + + with adbc_driver_manager.AdbcStatement(conn) as stmt: + stmt.set_sql_query("SELECT * FROM foo") + handle, _ = stmt.execute_query() + + result = pyarrow.table(handle) + assert result == table + + with adbc_driver_manager.AdbcStatement(conn) as stmt: + stmt.set_sql_query("SELECT * FROM bar") + handle, _ = stmt.execute_query() + + result = pyarrow.table(handle) + assert result == table + + # ensure consumed schema was marked as such + with pytest.raises(ValueError, match="Cannot import released ArrowArrayStream"): + pyarrow.table(handle) + + # smoke test for the capsule calling release + with adbc_driver_manager.AdbcStatement(conn) as stmt: + stmt.set_sql_query("SELECT * FROM foo") + capsule = stmt.execute_query()[0].__arrow_c_stream__() + del capsule From d9490aece5836997d71ded6fd752d1b503347e06 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 18 Dec 2023 15:04:57 -0500 Subject: [PATCH 07/51] docs: generate Javadocs (#1362) In #1357 I'd like to explore hacking Javadocs into Intersphinx for better cross-linking, but first we have to actually generate the Javadocs. Fixes #1361. --- ci/conda_env_java.txt | 21 ++++++ ci/scripts/docs_build.sh | 6 ++ docker-compose.yml | 2 +- docs/source/java/api/adbc_driver_manager.rst | 79 -------------------- docs/source/java/api/index.rst | 10 +-- docs/source/java/driver_manager.rst | 2 +- java/pom.xml | 9 ++- 7 files changed, 39 insertions(+), 90 deletions(-) create mode 100644 ci/conda_env_java.txt delete mode 100644 docs/source/java/api/adbc_driver_manager.rst diff --git a/ci/conda_env_java.txt b/ci/conda_env_java.txt new file mode 100644 index 0000000000..2b06fdddd8 --- /dev/null +++ b/ci/conda_env_java.txt @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +maven +# maven javadocs build appears to fail under Java 21 +# maven javadocs build appears to emit the wrong stylesheet under Java 8 +openjdk=17.* diff --git a/ci/scripts/docs_build.sh b/ci/scripts/docs_build.sh index 261786b99a..dcdf27a617 100755 --- a/ci/scripts/docs_build.sh +++ b/ci/scripts/docs_build.sh @@ -30,6 +30,12 @@ main() { make doctest popd + pushd "$source_dir/java" + mvn site + rm -rf "$source_dir/docs/build/java/api" + cp -r target/site/apidocs "$source_dir/docs/build/java/api" + popd + for desc_file in $(find "${source_dir}/r" -name DESCRIPTION); do local pkg=$(dirname "$desc_file") local pkg_name=$(basename $pkg) diff --git a/docker-compose.yml b/docker-compose.yml index a25fb6653b..89394e598f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: volumes: - .:/adbc:delegated command: | - /bin/bash -c 'git config --global --add safe.directory /adbc && source /opt/conda/etc/profile.d/conda.sh && mamba create -y -n adbc -c conda-forge go --file /adbc/ci/conda_env_cpp.txt --file /adbc/ci/conda_env_docs.txt --file /adbc/ci/conda_env_python.txt && conda activate adbc && env ADBC_USE_ASAN=0 ADBC_USE_UBSAN=0 /adbc/ci/scripts/cpp_build.sh /adbc /adbc/build && env CGO_ENABLED=1 /adbc/ci/scripts/go_build.sh /adbc /adbc/build && /adbc/ci/scripts/python_build.sh /adbc /adbc/build && /adbc/ci/scripts/r_build.sh /adbc && /adbc/ci/scripts/docs_build.sh /adbc' + /bin/bash -c 'git config --global --add safe.directory /adbc && source /opt/conda/etc/profile.d/conda.sh && mamba create -y -n adbc -c conda-forge go --file /adbc/ci/conda_env_cpp.txt --file /adbc/ci/conda_env_docs.txt --file /adbc/ci/conda_env_java.txt --file /adbc/ci/conda_env_python.txt && conda activate adbc && env ADBC_USE_ASAN=0 ADBC_USE_UBSAN=0 /adbc/ci/scripts/cpp_build.sh /adbc /adbc/build && env CGO_ENABLED=1 /adbc/ci/scripts/go_build.sh /adbc /adbc/build && /adbc/ci/scripts/python_build.sh /adbc /adbc/build && /adbc/ci/scripts/r_build.sh /adbc && /adbc/ci/scripts/docs_build.sh /adbc' ############################ Java JARs ###################################### diff --git a/docs/source/java/api/adbc_driver_manager.rst b/docs/source/java/api/adbc_driver_manager.rst deleted file mode 100644 index 6c4ec2d10f..0000000000 --- a/docs/source/java/api/adbc_driver_manager.rst +++ /dev/null @@ -1,79 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one -.. or more contributor license agreements. See the NOTICE file -.. distributed with this work for additional information -.. regarding copyright ownership. The ASF licenses this file -.. to you under the Apache License, Version 2.0 (the -.. "License"); you may not use this file except in compliance -.. with the License. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, -.. software distributed under the License is distributed on an -.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -.. KIND, either express or implied. See the License for the -.. specific language governing permissions and limitations -.. under the License. - -============================ -``Java ADBC Driver Manager`` -============================ - -Java ADBC Wrapper for JDBC -========================== - -The Java ADBC Driver Manager provides a means to manage ADBC drivers and facilitate connections to databases using the ADBC API. This particular implementation wraps around the JDBC API. - -Constants ---------- - -.. data:: org.apache.arrow.adbc.driver.jdbc.JdbcDriver.PARAM_DATASOURCE - - A parameter for creating an ``AdbcDatabase`` from a ``DataSource``. - -.. data:: org.apache.arrow.adbc.driver.jdbc.JdbcDriver.PARAM_JDBC_QUIRKS - - A parameter for specifying backend-specific configuration. - -.. data:: org.apache.arrow.adbc.driver.jdbc.JdbcDriver.PARAM_URI - - A parameter for specifying a URI to connect to, aligning with the C/Go implementations. - -Classes -------- - -.. class:: org.apache.arrow.adbc.driver.jdbc.JdbcDriver - - An ADBC driver implementation that wraps around the JDBC API. - - .. method:: open(Map parameters) - - Opens a new database connection using the specified parameters. - -.. class:: org.apache.arrow.adbc.driver.jdbc.JdbcDataSourceDatabase - - Represents an ADBC database backed by a JDBC ``DataSource``. - -Utilities ---------- - -The ``JdbcDriver`` class provides utility methods to fetch and validate parameters from the provided options map. - -.. method:: org.apache.arrow.adbc.driver.jdbc.JdbcDriver.getParam(Class klass, Map parameters, String... choices) - - Retrieves a parameter from the provided map, validating its type and ensuring no duplicates. - -Usage -===== - -The ``JdbcDriver`` class is registered with the ``AdbcDriverManager`` upon class loading. To utilize this driver: - -1. Ensure the necessary dependencies are in place. -2. Create a ``Map`` containing the connection parameters. -3. Use the ``AdbcDriverManager`` to obtain an instance of the ``JdbcDriver``. -4. Open a new database connection using the driver's ``open`` method. - -Exceptions -========== - -Any errors during the driver operations throw the ``AdbcException``. This exception provides detailed messages indicating the nature of the problem. diff --git a/docs/source/java/api/index.rst b/docs/source/java/api/index.rst index e93406f3c4..a316867429 100644 --- a/docs/source/java/api/index.rst +++ b/docs/source/java/api/index.rst @@ -15,11 +15,9 @@ .. specific language governing permissions and limitations .. under the License. -==================== +================== Java API Reference -==================== +================== -.. toctree:: - :maxdepth: 1 - - adbc_driver_manager +This is a stub page for the Javadocs. If you're seeing this page, it means +that the actual Javadocs were not generated. diff --git a/docs/source/java/driver_manager.rst b/docs/source/java/driver_manager.rst index 78215a0e90..3f31421e41 100644 --- a/docs/source/java/driver_manager.rst +++ b/docs/source/java/driver_manager.rst @@ -35,4 +35,4 @@ To include the ADBC Driver Manager in your Maven project, add the following depe API Reference ============= -See the API reference: :doc:`./api/adbc_driver_manager`. +See the `API reference <./api/org/apache/arrow/adbc/drivermanager/package-summary.html>`_. diff --git a/java/pom.xml b/java/pom.xml index 972e55b00b..8e166b59e7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -219,17 +219,20 @@ + org.apache.maven.plugins maven-javadoc-plugin - 3.4.1 + 3.6.3 - - -package all,-missing https://arrow.apache.org/docs/java/reference/ + + 8 + public 1.8 From 0450d52216ae2d0b306fc37979b1e29072845da0 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Tue, 19 Dec 2023 08:46:58 -0500 Subject: [PATCH 08/51] test(python): Fix polars DeprecationWarning in tests (#1366) --- python/adbc_driver_postgresql/tests/test_polars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/adbc_driver_postgresql/tests/test_polars.py b/python/adbc_driver_postgresql/tests/test_polars.py index a0295211f7..218a6d8b21 100644 --- a/python/adbc_driver_postgresql/tests/test_polars.py +++ b/python/adbc_driver_postgresql/tests/test_polars.py @@ -73,7 +73,7 @@ def test_polars_write_database(postgres_uri: str, df: "polars.DataFrame") -> Non connection=postgres_uri, # TODO(apache/arrow-adbc#541): polars doesn't map the semantics # properly here, and one of their modes isn't supported - if_exists="replace", + if_table_exists="replace", engine="adbc", ) finally: From beb47545793dac8c7aaa0cc77dc0076a0d6c6746 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 08:49:14 -0500 Subject: [PATCH 09/51] chore(go): bump golang.org/x/crypto from 0.14.0 to 0.17.0 in /go/adbc (#1367) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.14.0 to 0.17.0.
Commits
  • 9d2ee97 ssh: implement strict KEX protocol changes
  • 4e5a261 ssh: close net.Conn on all NewServerConn errors
  • 152cdb1 x509roots/fallback: update bundle
  • fdfe1f8 ssh: defer channel window adjustment
  • b8ffc16 blake2b: drop Go 1.6, Go 1.8 compatibility
  • 7e6fbd8 ssh: wrap errors from client handshake
  • bda2f3f argon2: avoid clobbering BP
  • 325b735 ssh/test: skip TestSSHCLIAuth on Windows
  • 1eadac5 go.mod: update golang.org/x dependencies
  • b2d7c26 ssh: add (*Client).DialContext method
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/crypto&package-manager=go_modules&previous-version=0.14.0&new-version=0.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/apache/arrow-adbc/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go/adbc/go.mod | 8 ++++---- go/adbc/go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go/adbc/go.mod b/go/adbc/go.mod index 45f2ee2daf..7368231f3d 100644 --- a/go/adbc/go.mod +++ b/go/adbc/go.mod @@ -83,12 +83,12 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go/adbc/go.sum b/go/adbc/go.sum index acc7fae809..edb94077b9 100644 --- a/go/adbc/go.sum +++ b/go/adbc/go.sum @@ -148,8 +148,8 @@ github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= @@ -165,13 +165,13 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From c43e6f36dee99c66c3b98647005adf86492561e0 Mon Sep 17 00:00:00 2001 From: davidhcoe <13318837+davidhcoe@users.noreply.github.com> Date: Tue, 19 Dec 2023 08:55:45 -0500 Subject: [PATCH 10/51] build(csharp): PowerShell build scripts for C# (#1350) Adding the build scripts that were split out from https://github.com/apache/arrow-adbc/pull/1345 --------- Co-authored-by: David Coe --- .github/dependabot.yml | 41 +++++++++++++++++++++++++++ ci/scripts/csharp_build.ps1 | 19 +++++++++++++ ci/scripts/csharp_pack.ps1 | 32 +++++++++++++++++++++ ci/scripts/csharp_smoketest.ps1 | 49 +++++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 ci/scripts/csharp_build.ps1 create mode 100644 ci/scripts/csharp_pack.ps1 create mode 100644 ci/scripts/csharp_smoketest.ps1 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..83c5135e9d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "chore: " + - package-ecosystem: "nuget" + directory: "/csharp/" + schedule: + interval: "weekly" + commit-message: + prefix: "chore(csharp): " + ignore: + - dependency-name: "Microsoft.Extensions.*" + update-types: + - "version-update:semver-major" + - dependency-name: "Microsoft.Bcl.*" + update-types: + - "version-update:semver-major" + - dependency-name: "System.*" + update-types: + - "version-update:semver-major" diff --git a/ci/scripts/csharp_build.ps1 b/ci/scripts/csharp_build.ps1 new file mode 100644 index 0000000000..01b75bf8a0 --- /dev/null +++ b/ci/scripts/csharp_build.ps1 @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd ..\..\csharp + +dotnet build diff --git a/ci/scripts/csharp_pack.ps1 b/ci/scripts/csharp_pack.ps1 new file mode 100644 index 0000000000..b875b60dd3 --- /dev/null +++ b/ci/scripts/csharp_pack.ps1 @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +param ( + [string]$destination=$null +) + +$loc = Get-Location + +if ($loc.ToString().ToLower().EndsWith("csharp") -eq $False) { + cd ..\..\csharp +} + +if ($destination) { + dotnet pack -c Release -o $destination +} +else { + dotnet pack -c Release +} diff --git a/ci/scripts/csharp_smoketest.ps1 b/ci/scripts/csharp_smoketest.ps1 new file mode 100644 index 0000000000..da5edd7743 --- /dev/null +++ b/ci/scripts/csharp_smoketest.ps1 @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +param ( + [string]$destination=".\packages" +) + +$ErrorActionPreference = "Stop" + +Write-Host "This script performs the following steps:" +Write-Host " - Runs unit tests against all projects" +Write-Host " - Packages everything to NuGet packages in $destination" +Write-Host " - Runs smoke tests using the NuGet packages" + +Write-Host "" + +cd $PSScriptRoot +cd ..\..\csharp + +Write-Host "Running dotnet test" + +dotnet test + +Write-Host "Running dotnet pack" + +$loc = Get-Location + +Write-Host $loc + +Invoke-Expression "powershell -executionpolicy bypass -File $PSScriptRoot\csharp_pack.ps1 -destination $destination" + +Write-Host "Running smoke tests" + +cd test\SmokeTests + +dotnet test From 025c78516009d37c553765c1e83470c8ca2bbf22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 09:06:10 -0500 Subject: [PATCH 11/51] chore: bump actions/setup-java from 3 to 4 (#1370) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-java](https://github.com/actions/setup-java) from 3 to 4.
Release notes

Sourced from actions/setup-java's releases.

v4.0.0

What's Changed

In the scope of this release, the version of the Node.js runtime was updated to 20. The majority of dependencies were updated to the latest versions. From now on, the code for the setup-java will run on Node.js 20 instead of Node.js 16.

Breaking changes

Non-breaking changes

New Contributors

Full Changelog: https://github.com/actions/setup-java/compare/v3...v4.0.0

v3.13.0

What's changed

In the scope of this release, support for Dragonwell JDK was added by @​Accelerator1996 in actions/setup-java#532

steps:
 - name: Checkout
   uses: actions/checkout@v3
 - name: Setup-java
   uses: actions/setup-java@v3
   with:
     distribution: 'dragonwell'
     java-version: '17'

Several inaccuracies were also fixed:

New Contributors

Full Changelog: https://github.com/actions/setup-java/compare/v3...v3.13.0

v3.12.0

... (truncated)

Commits
  • 387ac29 Upgrade Node to v20 (#558)
  • 9eda6b5 feat: implement cache-dependency-path option to control caching dependency (#...
  • 78078da Update @​actions/cache dependency and documentation (#549)
  • 5caaba6 add support for microsoft openjdk 21.0.0 (#546)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-java&package-manager=github_actions&previous-version=3&new-version=4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/java.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 9e11038948..a789ef45bc 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -48,7 +48,7 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: cache: "maven" distribution: "temurin" @@ -81,7 +81,7 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: cache: "maven" distribution: "temurin" From 54f2eb8515f168998ccf73eeb10227f6949efc5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 09:07:40 -0500 Subject: [PATCH 12/51] chore: bump actions/github-script from 6 to 7 (#1372) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/github-script](https://github.com/actions/github-script) from 6 to 7.
Release notes

Sourced from actions/github-script's releases.

v7.0.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v6.4.1...v7.0.0

v6.4.1

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v6.4.0...v6.4.1

v6.4.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v6.3.3...v6.4.0

v6.3.3

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v6.3.2...v6.3.3

v6.3.2

What's Changed

... (truncated)

Commits
  • 60a0d83 Merge pull request #440 from actions/joshmgross/v7.0.1
  • b7fb200 Update version to 7.0.1
  • 12e22ed Merge pull request #439 from actions/joshmgross/avoid-setting-base-url
  • d319f8f Avoid setting baseUrl to undefined when input is not provided
  • e69ef54 Merge pull request #425 from actions/joshmgross/node-20
  • ee0914b Update licenses
  • d6fc56f Use @types/node for Node 20
  • 384d6cf Fix quotations in tests
  • 8472492 Only validate GraphQL previews
  • 84903f5 Remove node-fetch from type
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/github-script&package-manager=github_actions&previous-version=6&new-version=7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index ed9852b8e0..baab8bf54f 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -42,7 +42,7 @@ jobs: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'edited') - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | From cf04190b33428407fbc8e03048905ec797559587 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 09:34:54 -0500 Subject: [PATCH 13/51] chore: bump actions/checkout from 3 to 4 (#1373) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4.
Release notes

Sourced from actions/checkout's releases.

v4.0.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3...v4.0.0

v3.6.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3.5.3...v3.6.0

v3.5.3

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3...v3.5.3

v3.5.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v3.5.1...v3.5.2

v3.5.1

What's Changed

New Contributors

... (truncated)

Changelog

Sourced from actions/checkout's changelog.

Changelog

v4.1.0

v4.0.0

v3.6.0

v3.5.3

v3.5.2

v3.5.1

v3.5.0

v3.4.0

v3.3.0

v3.2.0

v3.1.0

v3.0.2

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=3&new-version=4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/dev_pr.yml | 2 +- .github/workflows/integration.yml | 8 ++++---- .github/workflows/java.yml | 4 ++-- .github/workflows/native-unix.yml | 16 ++++++++-------- .github/workflows/native-windows.yml | 8 ++++---- .github/workflows/nightly-verify.yml | 4 ++-- .github/workflows/nightly-website.yml | 6 +++--- .github/workflows/packaging.yml | 8 ++++---- .github/workflows/rust.yml | 2 +- .github/workflows/verify.yml | 4 ++-- 11 files changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 3acb30f6ba..cdbc1296ab 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -33,7 +33,7 @@ jobs: name: "pre-commit" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index baab8bf54f..b79307a159 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -33,7 +33,7 @@ jobs: name: Process runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: persist-credentials: false diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d0f403e6f0..e4b58f927a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -53,7 +53,7 @@ jobs: name: "DuckDB Integration Tests" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -102,7 +102,7 @@ jobs: name: "FlightSQL Integration Tests (Dremio and SQLite)" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -189,7 +189,7 @@ jobs: name: "PostgreSQL Integration Tests" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -322,7 +322,7 @@ jobs: name: "Snowflake Integration Tests" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index a789ef45bc..656bc5e9a8 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -44,7 +44,7 @@ jobs: matrix: java: ['8', '11', '17', '21'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -77,7 +77,7 @@ jobs: matrix: java: ['11', '17', '21'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/native-unix.yml b/.github/workflows/native-unix.yml index 87e13682e4..e62e842d42 100644 --- a/.github/workflows/native-unix.yml +++ b/.github/workflows/native-unix.yml @@ -69,7 +69,7 @@ jobs: # https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk CXXFLAGS: "-D_LIBCPP_DISABLE_AVAILABILITY" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -141,7 +141,7 @@ jobs: # https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk CXXFLAGS: "-D_LIBCPP_DISABLE_AVAILABILITY" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -192,7 +192,7 @@ jobs: name: "clang-tidy" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -240,7 +240,7 @@ jobs: # https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk CXXFLAGS: "-D_LIBCPP_DISABLE_AVAILABILITY" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -294,7 +294,7 @@ jobs: matrix: os: ["macos-latest", "ubuntu-latest", "windows-latest"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -332,7 +332,7 @@ jobs: env: CGO_ENABLED: "1" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -410,7 +410,7 @@ jobs: # https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk CXXFLAGS: "-D_LIBCPP_DISABLE_AVAILABILITY" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -530,7 +530,7 @@ jobs: os: ["ubuntu-latest"] python: ["3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/native-windows.yml b/.github/workflows/native-windows.yml index 94b80dc563..815a52bf92 100644 --- a/.github/workflows/native-windows.yml +++ b/.github/workflows/native-windows.yml @@ -59,7 +59,7 @@ jobs: matrix: os: ["windows-latest"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -116,7 +116,7 @@ jobs: matrix: os: ["windows-latest"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -194,7 +194,7 @@ jobs: matrix: os: ["windows-latest"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -262,7 +262,7 @@ jobs: os: ["windows-latest"] python: ["3.9", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/nightly-verify.yml b/.github/workflows/nightly-verify.yml index 45279e5d6c..525595702e 100644 --- a/.github/workflows/nightly-verify.yml +++ b/.github/workflows/nightly-verify.yml @@ -31,7 +31,7 @@ jobs: if: github.event_name != 'schedule' || github.repository == 'apache/arrow-adbc' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -102,7 +102,7 @@ jobs: matrix: os: ["macos-latest", "ubuntu-latest", "windows-latest"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 path: arrow-adbc diff --git a/.github/workflows/nightly-website.yml b/.github/workflows/nightly-website.yml index 38083c8b7e..428cfb73a5 100644 --- a/.github/workflows/nightly-website.yml +++ b/.github/workflows/nightly-website.yml @@ -36,7 +36,7 @@ jobs: name: "Build Website" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -57,14 +57,14 @@ jobs: runs-on: ubuntu-latest needs: [build] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 path: site # NOTE: needed to push at the end persist-credentials: true ref: asf-site - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 path: scripts diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index f227947d70..f1cbc816c9 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -61,7 +61,7 @@ jobs: # For cron: only run on the main repo, not forks if: github.event_name != 'schedule' || github.repository == 'apache/arrow-adbc' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -228,7 +228,7 @@ jobs: with: name: source - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: apache/arrow path: arrow @@ -890,7 +890,7 @@ jobs: run: shell: bash -l {0} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: true @@ -933,7 +933,7 @@ jobs: - python-windows - python-sdist steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: true diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 248a9ee494..c2cf1edcea 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -48,7 +48,7 @@ jobs: name: "Rust ${{ matrix.os }}" runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index fbde892bd0..20fcfcc83b 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -50,7 +50,7 @@ jobs: matrix: os: ["macos-latest", "ubuntu-latest"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -72,7 +72,7 @@ jobs: matrix: os: ["macos-latest", "ubuntu-latest", "windows-latest"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false From 14a7c97e2461d70d47f55956d128f3a90536b55c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 09:35:36 -0500 Subject: [PATCH 14/51] chore: bump conda-incubator/setup-miniconda from 2 to 3 (#1369) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 2 to 3.
Release notes

Sourced from conda-incubator/setup-miniconda's releases.

Version 3.0.0

Features

  • #308 Update to node20
  • #291 Add conda-solver option (defaults to libmamba)

Fixes

  • #299 Fix condaBasePath when useBundled is false, and there's no pre-existing conda

Documentation

  • #309 Switch to main branch based development
  • #313 Specify team conda-incubator/setup-miniconda as codeowners
  • #318 README: update actions in examples, add security section, similar actions

Tasks and Maintenance

  • #307 Run dependabot against main branch and also update node packages
  • #311 Bump actions/checkout from 2 to 4
  • #310 Bump actions/cache from 1 to 3
  • #314 Strip/update dependencies
  • #315 Split lint into check and build, switch from npm install to npm ci
  • #317 Bump normalize-url from 4.5.1 to 8.0.0
  • #316 Faster workflow response / saving resources via timeout/concurrency policy

#308: conda-incubator/setup-miniconda#308 #291: conda-incubator/setup-miniconda#291 #299: conda-incubator/setup-miniconda#299 #309: conda-incubator/setup-miniconda#309 #313: conda-incubator/setup-miniconda#313 #318: conda-incubator/setup-miniconda#318 #307: conda-incubator/setup-miniconda#307 #311: conda-incubator/setup-miniconda#311 #310: conda-incubator/setup-miniconda#310 #314: conda-incubator/setup-miniconda#314 #315: conda-incubator/setup-miniconda#315 #317: conda-incubator/setup-miniconda#317 #316: conda-incubator/setup-miniconda#316

New Contributors

Full Changelog: https://github.com/conda-incubator/setup-miniconda/compare/v2...v3.0.0

Version 2.3.0

Documentation

  • #263 Update links to GitHub shell docs

... (truncated)

Changelog

Sourced from conda-incubator/setup-miniconda's changelog.

v3.0.1 (2023-11-29)

Fixes

  • #325 Fix environment activation on windows (a v3 regression) due to hard-coded install PATH

#325: conda-incubator/setup-miniconda#325

v3.0.0 (2023-11-27)

Features

  • #308 Update to node20
  • #291 Add conda-solver option (defaults to libmamba)

Fixes

  • #299 Fix condaBasePath when useBundled is false, and there's no pre-existing conda

Documentation

  • #309 Switch to main branch based development
  • #313 Specify team conda-incubator/setup-miniconda as codeowners
  • #318 README: update actions in examples, add security section, similar actions

Tasks and Maintenance

  • #307 Run dependabot against main branch and also update node packages
  • #311 Bump actions/checkout from 2 to 4
  • #310 Bump actions/cache from 1 to 3
  • #314 Strip/update dependencies
  • #315 Split lint into check and build, switch from npm install to npm ci
  • #317 Bump normalize-url from 4.5.1 to 8.0.0
  • #316 Faster workflow response / saving resources via timeout/concurrency policy

#308: conda-incubator/setup-miniconda#308 #291: conda-incubator/setup-miniconda#291 #299: conda-incubator/setup-miniconda#299 #309: conda-incubator/setup-miniconda#309 #313: conda-incubator/setup-miniconda#313 #318: conda-incubator/setup-miniconda#318 #307: conda-incubator/setup-miniconda#307 #311: conda-incubator/setup-miniconda#311 #310: conda-incubator/setup-miniconda#310

... (truncated)

Commits
  • 11b5629 Prepare 3.0.1 (#326)
  • 8706aa7 Fix env activation on win (a v3 regression) due to hard-coded install PATH (#...
  • c585a97 Bump conda-incubator/setup-miniconda from 2.3.0 to 3.0.0 (#321)
  • 2defc80 Prepare release (#320)
  • 0d5a56b Bump actions/checkout from 2 to 4 (#319)
  • 45fd3f9 Merge pull request #316 from dbast/timeout
  • d1e04fc Merge pull request #299 from isuruf/condaBasePath
  • fab0073 Merge pull request #318 from dbast/readme
  • fa6bdf9 Update with npm run build
  • d42f8b8 Fix condaBasePath when useBundled is false, and there's no pre-existing conda
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=conda-incubator/setup-miniconda&package-manager=github_actions&previous-version=2&new-version=3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/integration.yml | 8 ++++---- .github/workflows/native-unix.yml | 14 +++++++------- .github/workflows/native-windows.yml | 8 ++++---- .github/workflows/nightly-verify.yml | 2 +- .github/workflows/packaging.yml | 4 ++-- .github/workflows/verify.yml | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e4b58f927a..ba1ed14448 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -67,7 +67,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -116,7 +116,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -203,7 +203,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -336,7 +336,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest diff --git a/.github/workflows/native-unix.yml b/.github/workflows/native-unix.yml index e62e842d42..4d6c50c50e 100644 --- a/.github/workflows/native-unix.yml +++ b/.github/workflows/native-unix.yml @@ -83,7 +83,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -155,7 +155,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -206,7 +206,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -254,7 +254,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -346,7 +346,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -424,7 +424,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -544,7 +544,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest diff --git a/.github/workflows/native-windows.yml b/.github/workflows/native-windows.yml index 815a52bf92..747495a635 100644 --- a/.github/workflows/native-windows.yml +++ b/.github/workflows/native-windows.yml @@ -76,7 +76,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -133,7 +133,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -211,7 +211,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -279,7 +279,7 @@ jobs: with: path: ~/conda_pkgs_dir key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest diff --git a/.github/workflows/nightly-verify.yml b/.github/workflows/nightly-verify.yml index 525595702e..c4a3fdf63c 100644 --- a/.github/workflows/nightly-verify.yml +++ b/.github/workflows/nightly-verify.yml @@ -125,7 +125,7 @@ jobs: run: | mv apache-arrow-adbc-${{ env.VERSION }}-rc0/KEYS . - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 # The Unix script will set up conda itself if: matrix.os == 'windows-latest' with: diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index f1cbc816c9..496f260906 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -442,7 +442,7 @@ jobs: echo "schedule: ${{ github.event.schedule }}" >> $GITHUB_STEP_SUMMARY echo "ref: ${{ github.ref }}" >> $GITHUB_STEP_SUMMARY - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest @@ -898,7 +898,7 @@ jobs: uses: actions/download-artifact@v3 with: path: conda-packages - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 20fcfcc83b..966167bccf 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -76,7 +76,7 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 # The Unix script will set up conda itself if: matrix.os == 'windows-latest' with: From 6a1bc994309282a0129a76e12b1b1d21f66889b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 09:36:18 -0500 Subject: [PATCH 15/51] chore: bump docker/setup-qemu-action from 2 to 3 (#1371) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 2 to 3.
Release notes

Sourced from docker/setup-qemu-action's releases.

v3.0.0

Full Changelog: https://github.com/docker/setup-qemu-action/compare/v2.2.0...v3.0.0

v2.2.0

Full Changelog: https://github.com/docker/setup-qemu-action/compare/v2.1.0...v2.2.0

v2.1.0

Full Changelog: https://github.com/docker/setup-qemu-action/compare/v2.0.0...v2.1.0

Commits
  • 6882732 Merge pull request #103 from docker/dependabot/npm_and_yarn/actions/core-1.10.1
  • 183f4af chore: update generated content
  • f174935 build(deps): bump @​actions/core from 1.10.0 to 1.10.1
  • 2e423eb Merge pull request #89 from docker/dependabot/npm_and_yarn/semver-6.3.1
  • ecc406a Bump semver from 6.3.0 to 6.3.1
  • 12dec5e Merge pull request #102 from crazy-max/update-node20
  • c29b312 chore: node 20 as default runtime
  • 34ae628 chore: update generated content
  • 1f3d2e1 chore: fix author in package.json
  • 277dbe8 vendor: bump @​docker/actions-toolkit from 0.3.0 to 0.12.0
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/setup-qemu-action&package-manager=github_actions&previous-version=2&new-version=3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/packaging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 496f260906..d6dd0b01ff 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -516,7 +516,7 @@ jobs: echo "ref: ${{ github.ref }}" >> $GITHUB_STEP_SUMMARY - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Build wheel env: From 45eaba6043682affaff8aa97eef46a9e04a215d8 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 19 Dec 2023 09:42:12 -0500 Subject: [PATCH 16/51] docs: fix path in build script (#1379) Fixes #1368. --- ci/scripts/docs_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/docs_build.sh b/ci/scripts/docs_build.sh index dcdf27a617..7294e6d68b 100755 --- a/ci/scripts/docs_build.sh +++ b/ci/scripts/docs_build.sh @@ -32,8 +32,8 @@ main() { pushd "$source_dir/java" mvn site - rm -rf "$source_dir/docs/build/java/api" - cp -r target/site/apidocs "$source_dir/docs/build/java/api" + rm -rf "$source_dir/docs/build/html/java/api" + cp -r target/site/apidocs "$source_dir/docs/build/html/java/api" popd for desc_file in $(find "${source_dir}/r" -name DESCRIPTION); do From c8ec427db7d3ee9435f3407aaab339b65ce0ecbd Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 19 Dec 2023 09:44:24 -0500 Subject: [PATCH 17/51] docs: spruce up landing page (#1360) Fixes #1359. --- ci/conda_env_docs.txt | 3 +- docs/source/_static/banner.png | Bin 0 -> 18127 bytes docs/source/conf.py | 14 ++- docs/source/driver/sqlite.rst | 2 +- docs/source/index.rst | 198 +++++++++++++++++++++++++++++---- 5 files changed, 190 insertions(+), 27 deletions(-) create mode 100644 docs/source/_static/banner.png diff --git a/ci/conda_env_docs.txt b/ci/conda_env_docs.txt index ff2b6df33f..42151b8d29 100644 --- a/ci/conda_env_docs.txt +++ b/ci/conda_env_docs.txt @@ -18,7 +18,7 @@ breathe doxygen # XXX(https://github.com/apache/arrow-adbc/issues/987) -furo=2023.07.26 +furo>=2023.09.10 make # Needed to install mermaid nodejs @@ -28,4 +28,5 @@ sphinx>=5.0 sphinx-autobuild sphinx-copybutton sphinx-design +sphinxext-opengraph r-pkgdown diff --git a/docs/source/_static/banner.png b/docs/source/_static/banner.png new file mode 100644 index 0000000000000000000000000000000000000000..2f58c223e2d2adc43de7d763a31efd953ad23e7a GIT binary patch literal 18127 zcmeIaXH-*5)IP5FD%gOlpeWS_hy{=qdaeZ{(nKjD9i@ie0)*lf5D^0^y(^&z0iv`J zLXj?AdJ9B)Pv`+c$p4VU_xI`l{r&K|mW#!lz4tux>^*zW%sE5YV=V~V37!*&4jp1s zdwBoJp+l_44juaY>#@IqmNkLLYQQgt9&6~T0>2(UeE6@w{`&jxzmFU_a`fm?78Vv( zR@P(3jvYUK{KSb9Cr_R{b?Ouw8yh=2I|m2HKmYu5`t<2DXU=ePa&mETojrT@+_`hy z+}!8SpTBV70uK)lFE1}2A0IzIzkq;%prGKzix)3lx^(&Sgws~>Feto7#KWz_RP@G z@cHxSMn*=)#>OTlCZ?vQW@cvQ=H?a_7M7NlFJ8QO`SPWem6f%%wT+F9t*xz{ot?eC zJroLcaBy&RbaZlZa&~rhadB~Vb#-%db9Z<5@bK{T^z`!bg27!{Kc6RpHuU~U=a&mKXQ7BYiUS58FenCM2 z8jb$;?c4Y7-wO*1i;9Yhi;GK2N=i#hF&Io)Ss50KEiW&};cyid6_u5hRaI5h)zv?K z{HUp^sjaQ8tE;Q8uWx8*Xl!h3YHIrV^JjB&b4yE0Yiny;TU&d3dq+n{XJ_ZHU%$G# zy1Ki&dwP0$dwcu(`uh9(2L=WP2M33ShK7fSM@B~Qc>L(-=-Aj8fk62E`}g?x_{7A- z+2gE z8=IS(TU%RXGI@J@duL~7cXyXUq3rGLQK{7Z{e2pZHu25GAD9EjT^<^`A37v<<>3Ee z?@X}Qp+grBsolS$>z%wj3j2IjF9GyB^^(nLox>&#uM7>ijN9Z{b=%|%d^E3Fb`+G) zXt-|kugTflH*n_%^>nz-{`1fM?n6@4+dcj^PII|r-OZdTPoh#MSc0ryxrv%t zNu4L=Dg+${djG$F{~H*%S%fmS!g6MOb`u0s-0#=+DRyO{+tJF0|U5@%}{m)1wGnQ;Lz+A%*YXT*(-EH|J!CV_r@N)+QQ9 z2Rd$v97=iCT?QYK3S7xKj3!=KB%`Uym7Gm)dfhmhM2GX1 z2WCcpb{QDuJiY5OS2B_=bcP2e&UlvTO8!D``osg{U^L~siU-m$UzH_VG3e=Sti;nX zf5ex?Vd?VSyyP`H0r!I%EA!|NyE0tGZRncP_W0I8Iu-3ao^CtcoybaWM*X^!ScWc0Q>P+(tDQ4A;n&Yr;5cce2cJw-#}t!(+*m(_o~$C zGtK>FiU1?Ec_xld5-Nq(Nb~+@(*&3*d@e-%1^(_eJOnR@&#U4_CeR*`$Qc1!e!jmYV%gjJO_$0 z_^qb&ZF>2pUsLqx9S@b^fEcfRKkC!-*w1QHU$m$Z?Ul53jTgULp*fPen@N+~Ah7#P zH$@m>Rw3MD0dA6W0Dzh1?^1T%%TQXEJs5O zoe6<$j^6ter$T#THe89y;SN@TkH^SA)edGg#SD9u=!j~JHIAA9abXP|*on);W8I1p z=dB4>Ktx)jXrx=1I@QUksMEx!M!eK#b9hy}3ZrUSh_Dx_T|qwAf7gb<6&(QSe>%fa zudVN-uL#AjJ5Le{jNC-ki;%_>PbU%m`y-#s_@!ZxhMrGRh=gN?;;>0#E)X6?n>}oQ zyR?V#?z`4xqGVBNG6n7n+XE0ySai+rg2}s3L8bAyNIu$H^m0@bA|6nLPZGZZic+4` zu5HD&3&=o~5__C^f7y=eBtg5X{Oablpj&~81@1rV%>rNUjo4&}jQqI&(|3m#20HLY z4hZpv@LycDcO5t~e^@M0W=vt9I^}%)X{4n0c3Ini2+cQg9a6oo7xZ!wz=;lO9927D z#tzjZd#XyRRKU+i`V9?$ zbr2=xowiGr?h@WBbA+)c-!0^ze%2K=ragSq){~b>Ulm#cCfQ88E)D?@ABrGwK?g*3 zb3^WtvVv;1kR#-dLF=_Q<9W9+HKSSbE$|1t1*p`dMv#FY=1$>%VIMv^@xb;Ebbf6s z;yC+hlyT&pvhCy6FUIq}le~knHC&_hCd0L>byBt~k+;O~(ou=W)cSNLJI1inZAqN= z$~MwO)AD_SGkBLQu01B1EaS!Rys{@6;a2C05HS!*#U;*^&o{FO-+fSSkH-S3!1}KB z+;-PD-0p6m88w@YR<(746LZQ2ajS{9!uK6g32Ka7c#=?)1xh8WY0FBSmh3j1?7(BE zr8y-PCtdfbU1_F@yFFhxL2Nb%{zl#k+jq$K&YVgp;Wd4Xz>t(Ooe3C1(RntuN8{(x zd$guc_nh_sFLfS7*SQf1@^1V@$Pk?7MiTt9yVT-=0a;9T03`(ILTZHOU-uU0;Q;tHE_RfI;;kxbg0kdVwwv*KAcKP`tQUt+1TwCav)0$uZ(Xk_czjxuS(m>{cjgYM_(qQ~9Zxnw@dZ=9y~|tCme>h@@8m9L({y~~K~ZDw>b<+-gSk{4 zAa69@<4p+(`=C22OY}=I>Zp+aGCYRVBXr_D)W-}Ulcd|6vJuvZzniLQ-CPjtbXlV^ zZ^|1$o*zU6(mY*W9TK#xoD{J3t4li~;hU=jbjhW5n;e-kaSg^dP^O3MiC6cc+%QN= z$`a8E)>AaC(m8eFm*-?hAKj(b^LcKu%h7UU_&Y98=XlgH1!>1HyS3W`HcQeCIz)M~ zLTBW9yn3wV#QkT}MhMHtuR3^PSsk}*#*!b~?Y)oVG-#&`g+2ho+q?6jBDJl7?oRkB z^F(A@r8CMn$?`JirJQnSybqA=ecbP6u8uCE<7q{T?*q2tZpAv7NS!v_#oDwZhRsM4 z?K&F49#9qp+j)q0h(1YZ$ZW%n;(SpoQmDb0t)g>`|&6fxOBZ(`BHkCAZT}|huc|D z8&q9EobXQ+uqsDA7==8v-DJ&PYPQR&s=Ws|!=i9CJIc&>Z#@VC20N{GBEwdCMYMTP z?Gcir{=vd`Ei-|_wmuWAR;pwr{Ug>d5j#8FCGJ!)H(SCvo`_a5PihEx&SynzmL%JH zTo=B}b1l=oH31}|A+gW3wzM3(H}kd2d8}%#DN3Yvy;E)#C+J(_7^iRN8I7G^xP;JZ zxc`uo*+r&`B995UoOot3#65IG4F9DsuWRL696xQvPH$6K@^tq@Q95T9-&;-3N8nT! zPn0Ie0K@7_?BXEw%^7P1OFw-?XY1Ki!s<85aINxi8xJCK4b{a_Jwku3%X~gJyZY6L zqJ479RthSu($e$*tW=aWb1*%Nbb!jU9@9;aw3!ictKLf68nIi}h#81a$95>Ee)Tis z4Hd*EAaQ<&q{{KcUBH?cl(JdkZ)8sso90a<3o!)KCI>9>em>r72;kQ+2>4<@IVGkCm zX^NSLXxbpWAQw&v$c>TkNm_%GI+JDs;J`0hoR{*Mu3;BMWP%6sclWxwWV0Q3oEaJp z6^+*3{oSG)0h2Z#*-6T@Y+xKW7~@F7rF2|i%N*!)^oU#a#tk&4NhixZ7nnW+PgdBU zR}~+!ujwDD^cvj;37W0Ab6$GNjN~LNb}Nutjwn5k{z@4V!U=Wo+=?t5a|9I_y4G@v zm8@nPC`@U)gZHVKA_BRa_q}IPdN^G@RsDAiHVS!RcQT*^IzB)W#h4|A`!wtN-EU-!J;hKH(6h*%A`178JR&>RPR| z6-tm_=pp$w^pIdk^;@ulbmm;T)}e**+dT~iqtiwAmMJ9fr79zFSp5^`<5Ismv!{*2 zShFF;&Q-77ABH`Vdong@`=M!m({JOzYcp|%@)l1V;5Y;7%7jFhj~Sd8U{tJBD=R2p zRIt&|M5xxmE)!^&ayOA`t0e1Hr>&@yC{#YRE<8!kD4pR)hb_}Yg~14;m}&1&tNp;= z^)#iIw7wgcHEcsE{%iQDYk~e`{^y>d5Y-UzDxD}d(z}*Qlur35)Mj2C|5*o{$q0~U zYp2$pPpl)3MSK$TRGWJ?0lRk;V+Dm>S|c+$Uth_t-c)GkSbzI%3CF8Dvby(mIPrma ze-F})*1h5p_h-IiMlvHyU@zr=A06F`bQ#Mg^-l0Z(Y3upt2xyUF;jD_Ajf|UPC*3- zpNIJ;-oorZ5#8^6_r%F6js6dc<>mQ%%i@NKaEN=SPynT^kyP({Hfva{6I|)}fuLM9 zo+(v*TwQki`bLB64+ELw+?65Vf+4IHvp&hbK<)RMf93x$`F6Bjxo&(ul&I?xe|@A< zV%IWnOf3?+QDU{7Kel2z-%!QVq?mQ56a{G*>z~pWZngiS81-xuhWOQ+|!=Gg5@TiAtq@3Nznf z-&OyU=sXPV8CdisUNGHj*${8=J!}2?7CSq-T*G7d?R7Y*=w6)Q%^oJl?{Dei!5R%6DuyXDc4Dno?FHX5q%%@U#!Lk3|o-S==+*3T)680 z+a+MF&Wgwt$jq%V3SSZAJ8j{5hpcx@<2~k( zNJ$&vF&$rYa*mj*?&CR*oRCj{eAQ8-$vVW51)h07lUl6U0#Nl|N_Cgo2UxaJI7P@J zVKFnMzB)T(*LMP1xJ$b8<;t0pnF460qnxS;9`ExJF0aj)An_R4F??2H(@KXYBHYRoU#q5n z#t+biG=TBG9Q`}}5ao?b?U_aY>oQ$KqH!yy#CT(T>>tHs{XsnW(UE148_>N8=(avO zAvJr+gjJr_c;|KBc*>r$ru@f^7|ktpQ5fm#qX>~hVtUNt%bn&dRV5bhl~9?`q0CJ( zdK~}VzgAr@%j<6Ozq;E)OW%)+xMgnq+o|PaVam-VFV*y1kn@jo>)4l5XZ>iR9Ev?y zn{Dc@Q8oOm^~=Sy_KRvBeKy|yGYyJOHfx_Zl+@Cs_@a;xc^Ql0mqq`e6+vB1*-A?e z0N58T$ED7I*AzlS`$GpsZn`vWYm6ld%;p7y+>Ic#@9bLMo+E$ie^6uzuZfKa__9yo zg-&IW0Wr!jN`e=^J&|x#o+`O{TkGA4tA67=i?MF=DJ3eJuqVSIz6J`YpNH6 z(}v*z-MV2BQ^2dfMr{*8U7PIa&eO8F&TnX6FwSl|Ydv*tK{iK)I_3)(nahc@BqvAK z0ZO(aF>f~YDMnnV&9@md);?e#^KPVf#`{?@1#1zG)r^u(3(I9=fv^Zgyi?cs!pQ%5 z9wew@sg`#6JgOVe90eXn%Ev`dqPmzq_2uR?j4`g1Pi zxO0oy@8&z$mD_)FJMiko>oXFLVe?pbz{`~AvYU%9yYNvS&YR}n+Lfzg6Ul#v1YbBc z7XNJOE3}UbJ1WkJQc4$V@lUTriWr;SxZN-)667}C=I5bn?#p!DQOYFO6=?2auT;Bo zT-lw8C3pQzHo?jhQiDRvDbw>cK`Iy}Z_Otuw457-i^H~LRA08Y#*;S)t$eiXc|Kk) zZ!6u4LCh3txeRLm;oB|^=>U;Wv9M86+aXlC!)n`1ypi!p_PVN2N zJrydXIyjZMIqmhHh;}G!(}OY*WhUDH7i6aGK9%g=X^s}mp7M9TV4+XDWJQZ~O!;TJ z8~jjov!VRVxThis73t#l=(W%5MLTEir2yBB?OrA3q&;!glx}PJmphZ$4d>4m&rL#*A-B>tV09qDFP|`p9fln{Is9p;=3{F zmC&kXzEh?b1pPk_hdb+29TO!C|4fu?K zHuYA(>4wVB3n!ukLJU3KieL0JrJ(mMo~@!}aLyM-`^vIr48=evI4bMIJZIPbGoQtP z;C5Ne^@@HxiRH>n@Z3=6;+C!d(w=+g@0xMiUk1d$c-}XZl#vb9YQtx1uleo0I0Vo6 zwl4Um@d)_l+~|9`Q*BQ8ec^BHA7ZB6x}M(K^0_0Okvh2ho4-xjC^#~)pxm_Vm;A}qcBj{-TPtt80f9n{*%orwN932Q{NWc$j7{>nv-$X&;QJ+ z9eauArpo8{R0gNgO-2MN!m+b6UEApL)$b6C0Kw&c>pN2`hM=_kAs@ZsCj$vFAeqM2 z3tB593)i}WKG@IvDqxm@vy)G_S%RjmkB2|)^^4F$hhuFPZS{WS#vpe&t;y!u<1Vyw zhP5JQX~r0`NO~pOpk$U>Ta@q;-Ex({zLjV45P9X#r1E|yp{r%4d>(&^N88g`$lmid z`Bar47uB0o7rc0H|GQ@NXgWlXV{vQM;80xdFm!-=!~g6~P-- zfy*^6)^AbEH%-dGJ&m}_k@Kb}qe!}P`7DWVZ?X}QP7a<`%A z;)Ld}X8iip=ebB zCX2;gcs^5unc4)|bxL!HAo~w(T z4RY8Se#tzddUVe87iRV2FLZdR-_udo>%h47RIcqdR)u1peA8Y$JBXb2>?|0C$IX4p ziPIcj9PMh>O_a+hJ?YdahZ==&?l9f;TD5ae4nEoN8S|dy3GU$Qg0GOb{$rRXmDrWCN=Pp5=|o25NUth3868#ZQ=-5hFs8g3HXWV;GV%V_FyHlW zxogrPlQ!i@JI$NGH#3ZH0p5L5DJEGIa4&m=pHu6HKhZ5)s~O_0xIRe1AFq5lYa4^M zhll$s^6NNuQVEyu?qg@mxmD$Fh-c4f#*b+QhqB<(3SUO{d7g|*u2iLh0=P-V8KxU4 zi{VC9l7F)EU#gCdWu(3QG#*E4&$-iuoytajAikx}MgId6ps=m#;R$7@$Cyn}It50o zxLN}bI)t&?X?e*!;%QlrGJm}1Q`2v7g0(9q5%N=oe4ksE9V*5IVU&D!iT1MruJqk$ z(SW@)C$QbD^6H(Hc#O4va|vq){tY!H=z@eUmhN04rx@2$4kADeKCJ7L=W;(3*X33Hzx5_V3Gf6RS+tLscr z9Z98?qJo;Med>Yr{+mJoTg7`c}}cU{*EfxF_cuu)9mTl#p1pv7AhA@ zsVf)v^46!r*#+^{uB-K%;<4I{b)@Le$@KA+sRsCcgC(}=Zn^taSes2G?UU@!lgD}{ z!&Z=Y{!=CY7*Ij^*BwFya?NA7V@DIga|2HNzJXxD-Q>xn{ zLw2BT&y62)&rL=5XGCzjoqHD>Lx9F5CXgDgGl|z*}klw9#fdcJ@7{s;b|JgVLsrgpZu_ zes`jo`~msYpkA!P?T5 zpfC!{Jj(gBi77La(AXm%)ch^7yjg3Dmf{s-1&sjjqDoqi^~{Dz_;rtM$~uh zJ9|{_x~+rlsgb{b!AtxM6_|5y*QvGo=!@U!>YBs(Ubp^ZBM#nd0ydcDh*ThvY~J5) z{9Mloa@_q;zN~V4%vO*3cyFQNUSRe9Pmy9jJ!@uZ+`#jvUs}m;rcf^plT6y(+B}Fm zL;WEK2E56i>nl2Y38t9QMo4ks_fkAqR&`joY;yMRTV^qa%&AA!59D3{wl=ehQKh69 z!whGh6eYtu${&eZ_$r?!aw)0r)oUgR7y&kper3-=V`zA@83Z|f18y6mUyKW@bK0lbn?caCX%+NXNGa`_bQ$Zls+L@hn@Z9R=WYe{9gan zpm##w@i?Y}{rc66eA{KPH;#dhn)8d24qLOiMy-og&rIOpq6glHg-c z9>)i02V>anD?FX*hN15X#W}ybN~Sx8w=Lk`)>PXkd*Pp>BiE+dlA=_?Q=?=m?)9Z| zaD;ZSB^^l^!n@zEg~e4K?;(Y`B>nza)YReljPi`k>TrE~Vb_-u0m9jTpoRqrB=t4$ za|#4qnw12IsC@wUnifLfR8i-}4DS~7=gp+TGK}kh^#L{Y7^@iy{oSkL+V5Pc#nMDf zsoFO=5`H?9<7A;(_H?-9$;=H$;2mGafay85SBarE1A!4T4%>lT@!w+aq~6&LjRC&KyLF2y%oPXhe4J;NACbojBD)4qFku6GQEHE;brH$L$EN z9y`0x>Th57K)ir3NAh1b`eR`0guz)n#))hc@?y?z-cTVGuH|nTWpi{(Sc4!^3q7hQ z>-Wefviw#+={Ixg6$kCnZ|w$)kvr3|;%WoTL6DSXc#Z@XRPH;4OQln*V^TAFu(Q## zq`|q6dCmAL?cgBrfWeSUm+SKAf6ivl&;hPvY@HM%e|@TKCv@kNM_zQ*!Rzh~0tl(I zGPX+v^DPIiT74?=kR$I+xQ^5}k!<3}w1XkN2J%)^Swg|cgayJ)+2S8DqcZcug#Yme zH{nbmdiO7mV@+KzKGk{1kdwFisgpruSq+j%Eri9YK=!0d@aSU=8uk3b5o*RnTPsc=u0W9c=n* z{g_3Vc2-u{Oe*l1AX$w)LM5frCMgaW$S(M_09P-z?RQ4G86m*6(tLI23JW=x@Vj5n{x@n3mpqq+T~&#Nw)h%%;V=JvnR=Q;{Ot^<*!XrgBru_DotLi#xC%WtZmNyQEhar>-yj^d(VZ9;L==o6;+%D zce)@{X7T*B${SJh%fLtLF}Ys-85FhGRm6)@ILHG1V$A(o^JBA#X-*C>YRP{ zgrK&R(#4z(E_ZE@dsD;EtdvI_bh?L@A(^J1o>(UWK|!kN;9mOZE3166Nc%C=yVY8xv@7a$Ih3cj+=8pHg;iuA z9PEhpXowYucrkgqDb5~R&;L}t!EvQ5TJmYTGF+8hSgEq<;CpW7(Vd*}l*E+52du1~ zolcLTdfw{xA@8hm3VI6hF58q?^n*WEB*rVB&t{<4O>pHuSPay^O7`KUwjX;a;Dpk& zfa6w`Ycfi}p@rj_%19bC$};WU6aDSk=4t(uu0qpwR64M}rss|m?lCSJ$*}?7>oIkf zooZhtNAd1K*J@ioX=fW)ctt!?E&PR-hB zw!j_LbV5VPli0=tC*lRZvr7RppEd(pj`eum`T7$QR1S7}a&u4Ey|6{(1325Hl5KH- z{4Z0zgKRBXDEk~^!)Momo$b(hb1$(%g?gORpP0OT2qzSAd_fj2yda3=Or7h~n|2z` z#R~=cKzA&snECHr5yYEhX006|r72GiRzH$fZW6@(6>Jf}qierg>YBd3*kGop9lb)%?_eg*zx@4$UqDmCCl;?}r@%}1)gS-P*jO}&bIr8h`C?I< z-NA`hzCN;47^RGKi`pwFAGmF$v0EDwTLq^gln4hlm>qz}LC}xm!fi3k-&qwAAPKUI;bh)*%BhlOI-dWI9 zloptzGECu|iMW$8ZqY0vLkNUep}?id*Nl6iud@#f$dPP}D!JsDL%8hkB#eaqw+?%M zR}(mH9pee{c3Hd8n!!Qe_U0&w|7@~(b)xV1edT5;qlz=YX)V2i zU0(jRmROO42S1Xb;@xBB9P7nDffW{Y;-9`&s2j;h?Nu5__(EiEmOcD{aS9IGIxBra z-WX#HyeDy7Rau=+*N8+$EV6MpCd#9j4#>rIynJ=*w8?1v3*vF{p>3}TXfIO2D22Zd zYnxVIl(_!(>%@u(+YNS>k^JM~Vs0VBz7XkK0|AJMo6NNGeGC+Jy0qS|=0odka^AjX z^yTk{TcT|?NJ&k>qSD2*x_9Ev`mUAEA*$R8IaS8%4L+TbDt@FV%m@a)Qb}q4$LM-l z`Ryh=Cs>i#YA60At@f(M@kn(=)c44yqt3%$f3t|cM~VGoJn6pt2B6PbrZZWSsXvf! zBUvx(4fQX$>E5|D{(PPyT=~%dL=nW#rWRcA(L?`A2rg2x6KjHt2jAt+}x{Y@RTa3WY$~ z%_?AbV@yLXR9yL5So3`obLBmLSx_`HF%MtXy!rHz^G57feE+OWF38Uj^{eLwV;NvY zAtanCYPOFHZ` z#;JAiYQPOXv4pZ8!|Ej($|=mzlySGcOxBYt6*DXVYE{p*M&1b?;FZ^T{N-H9tpO2T zf^I-2`Ks7I-)GXEXghsu63Q|r^_JmNc%ymQX*jF7dFCp$o9ZC^cmAW%kyj4xok2*h z*+w$Y#`QL#D!!Nm_m-2wDzD*v?J8cb_F=yRI_(WKm1~7mxZ_5!T`gMUun6EpiN1fy zJjwlXS$|ZscW5T{7VUh)+kx{7dhzObiTTz_wtjx<2CpO5eAa}t_C}&Xc-B;96CW#g zn|dwuYV=7#HWf4Kl?t2=vyzkBT%wbyKawyW=4HVS%V#_QlQl)@=WRkT2f%yATFQDR~53(qI<%% z)xFjR%0AFvG4c~cUg9f|Ix98^1moheJgTHk%!B^ zrKXBnVxBSvrvJ+OQqa={!DG+GEAG2(mn@`6btoHDTx(g~_#>+o&2NhjfEReXRY z=icQ_36jQjGHwIUk#Zfz+@!t~+P0T_JO;SQM=&#UUWH3_T>9!5`U8x3&J>BWD9 zrttPP|9&4fkSweo@R@SuQ;YvKW$zIqmEr3QFWsog6gvfJ@z2Vkb*?`2^PcXIpH20z zCMr=0{thNq=gep%Ib&WN1-n$W~eQwtbHJ&#L}owc0H1iUyt@NfP){@f%}=yu`7q$JHEDj5oJ}piN#~OyUoDb+eaAr=jF6F zOwPDtShC>Ewy*bxrv}<%Qou`HZ%F)9Y@Z_V*~487STU~;4<`1m1n1(fPTS58cvmTj zQVCf;d{3_YG4hOGoP9NHKJwGb!?t&^X>cO;`L=N%TAzeRNrW%)(n1{;&=(4#E(~3+ zGPakiaB{;2Yu^@I-7%j3W8$dVv*|sH9zh|CnmUl?m=qa^Cyn@waw>qq67Da4v%gKd zpm)O~R|vnlLVnkIM0iki{zg|LP3HNZh;HVsJ6t4d_kH6BEjVb)lM6?JgO4ZFCS~nI^v5XQDCwy{Yp~df`mbsWrsD=qrlS}V9 zOf{~$n##L20WX2?SbVxLItsRmSM%!J{M{w4Ns~(m(PHQD!LalZ(_Bu-K;c%*Ew+BF zLH!HiR@FIR&m}Os9i}*w1}~Mn^vKP%FUkW5$-n<=0#mk(e3!nQaVNIi<&Q3B z#D4~`1E1Q^*J)>k{l={NU@*-}DS?!M#A363pd|E4G{bLfE$PLd7^$=!K6^skdo!ZF z^UB)X+}CO+U9mmNy&y&HtUuEt9$FtX!|xN}5x4!YGp*Q+)xDEFF#J; zz4KQ3WH$}YtVUC3HQ%z?Bit2IlggYY?~}$;d%prl?ovBiklEYXjOBu%>%<7IBfVi`BS$9ILV88@jr4E`B%x` z*8Bo#EBrV=;i1iiG!Wdl&aLZ3MJFX&tHUytIqCi~S-mma8+>px4V zw%1sr|u{U`i-T)%!uAX%(?KmbQ$fEk~B6w#^L?*iXinZD}nSwaVfA}xzv>S zuz@Lch&lO=m}f9%z@hhyDB^s@+;{vX!Q4R}Vy4~WKbWTxhtD%n{-2Tm)ttN_C8!Zp z{7RCkEysWS91EeJaUU^XdUcn6H1wB*Y|^g_xOZ&k*83EkoIi-oSUOCxTH ztcMl44z6xtrlmRl@<_)ureK^~?F5d@-M2v~x)Wl1l#cxhE8u|<|5)xVDMu z6axtdcac51%M|BE4(=k0@xZwK4{jTG-I*k2I3HYUUIP)G0IY{RE+1SPq&K;Nhy{!$ z_eo+NLrRuNWHdo1iQgDfvP3ZoUGc>vv6vwxLv&&^8BG$)7*aCC0!Gu*Ng{AKcF?0V zk;rILpG>_&M>u7RFJd4(g@6mw5#IKnOx2`6)EF(Z0Vo76KljKI4FT4Go6*q-xHUt} zWHQx+A=X!BGsqB=CK_(h#kdggEXH^b?(Nche-0vEVDK~2sJvKpaJ$-esLW=Ju6b|? zm<}ynZ_@M)<6%!(9N^>QY+$QA;16e*9Q<8?e?^{46Rm)?6hP?=0)F6Y;DWX5q-hpI zWwWO@H`=m+Zy> zY&mJVMb~@?#kUgZfYlH62?7UUG!2wd7<|hha0ZN~(K1ROT`C5Fvtcy#l~MZXO=l4}7`=&p z>zs~RSe7Wj@IXf6XcHZC{I4?j5MBP@3Ok)-OSMU&9Uab1C%iv{fWJ>LX#M~D>4TuG z^tJM5y#W}Xcy)KI`scyL{A-L@{(nEGC^rwn`_. + Introducing ADBC :octicon:`cross-reference` + + .. button-ref:: format/specification + :ref-type: doc + :color: secondary + :expand: + + Specification + + .. button-ref:: faq + :ref-type: doc + :color: secondary + :expand: + + FAQ + + .. grid-item-card:: + :columns: 12 4 4 4 + + Development + ^^^ + + Report bugs, ask questions, and contribute to Apache Arrow. + + +++ + + .. button-link:: https://github.com/apache/arrow-adbc/issues + :color: secondary + :expand: + + :fab:`github` Issues/Questions + + .. button-link:: https://arrow.apache.org/community/ + :color: secondary + :expand: + + Mailing List :octicon:`cross-reference` + + .. button-link:: https://github.com/apache/arrow-adbc/blob/main/CONTRIBUTING.md + :color: secondary + :expand: + + Contributing :octicon:`cross-reference` + +Why ADBC? +========= + +.. grid:: 1 2 2 2 + :margin: 4 4 0 0 + :gutter: 1 + + .. grid-item-card:: Arrow-native + + Execute queries and get back results in Arrow format, eliminating extra + data copies for Arrow-native backends. + + .. grid-item-card:: Backend-agnostic + :link: driver/status + :link-type: doc + + Connect to all kinds of databases, even ones that aren't Arrow-native. + ADBC drivers optimize conversion to/from Arrow where required, saving + work for developers. + + +++ + See Supported Drivers + + .. grid-item-card:: Cross-language + + Work in C/C++, C#, Go, Java, Python, R, Ruby, and more. + + .. grid-item-card:: Full-featured + + Execute SQL and Substrait, query database catalogs, inspect table + schemas, and more. ADBC handles common tasks without having to pull in + another database client. + + .. grid-item-card:: Language-native + + Use language-native APIs that you're already familiar with, like DBAPI + in Python, ``database/sql`` in Go, or DBI in R. .. toctree:: :maxdepth: 1 + :hidden: faq .. toctree:: :maxdepth: 1 :caption: Supported Environments + :hidden: C/C++ Go @@ -42,6 +210,7 @@ To learn more about ADBC, see the `introductory blog post .. toctree:: :maxdepth: 1 :caption: Drivers + :hidden: driver/installation driver/status @@ -55,6 +224,7 @@ To learn more about ADBC, see the `introductory blog post .. toctree:: :maxdepth: 1 :caption: Specification + :hidden: format/specification format/versioning @@ -63,28 +233,8 @@ To learn more about ADBC, see the `introductory blog post .. toctree:: :maxdepth: 1 :caption: Development + :hidden: development/contributing development/nightly development/releasing - -ADBC (Arrow Database Connectivity) is an API specification for -Arrow-based database access. It provides a set of APIs in C, Go, and -Java that define how to interact with databases, including executing -queries and fetching metadata, that use Arrow data for result sets and -query parameters. These APIs are then implemented by drivers (or a -driver manager) that use some underlying protocol to work with -specific databases. - -ADBC aims to provide applications with a single, Arrow-based API to -work with multiple databases, whether Arrow-native or not. -Application code should not need to juggle conversions from -non-Arrow-native datasources alongside bindings for multiple -Arrow-native database protocols. - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` From 595df25383d6b2ab94d919899dfdaf95023a0836 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 19 Dec 2023 13:17:51 -0400 Subject: [PATCH 18/51] feat(r): Reference count child objects (#1334) This PR adds a `.child_count` for all database, connection, and statement objects to ensure that we do not call the C-level release callback while a child object is available. Since very early commits, we had set `R_ExternalPtrProtected()` to the parent object, which meant that if you never called `adbc_XXX_release()`, R would correctly sort out the object dependencies and call the release callbacks in the correct order. One major exception to that was the returned `ArrowArrayStream` wrappers, for which there was no mechanism to add a dependent SEXP until I added it (in nanoarrow 0.2, I think); however, it wasn't actually used for anything except for the stream returned by `read_adbc()`. @nbenn / @krlmlr you are probably the best candidates to give this a review/ensure that it fixes or makes it easier to debug the issues you have been having in adbi! Note that https://github.com/apache/arrow-nanoarrow/pull/333 may have been the primary culprit (I will push that out to CRAN ASAP as a tweak release). Closes https://github.com/apache/arrow-nanoarrow/issues/323 , https://github.com/apache/arrow-adbc/issues/1128 . Perhaps related: https://github.com/apache/arrow-adbc/issues/1348 . I triggered some extended checks as well (as they include valgrind): https://github.com/paleolimbot/arrow-adbc/actions/runs/7133874110 --- r/adbcdrivermanager/DESCRIPTION | 2 +- r/adbcdrivermanager/NAMESPACE | 1 - r/adbcdrivermanager/R/adbc.R | 40 ++++++++-- r/adbcdrivermanager/R/helpers.R | 79 +++++++++++-------- r/adbcdrivermanager/R/utils.R | 34 +++++++- .../man/adbc_connection_join.Rd | 8 -- .../man/adbc_statement_set_sql_query.Rd | 9 ++- r/adbcdrivermanager/man/adbc_xptr_move.Rd | 5 +- r/adbcdrivermanager/man/with_adbc.Rd | 7 +- r/adbcdrivermanager/src/init.c | 2 + r/adbcdrivermanager/src/radbc.cc | 34 ++++++++ r/adbcdrivermanager/src/radbc.h | 2 + r/adbcdrivermanager/src/utils.c | 11 +++ .../tests/testthat/test-driver_monkey.R | 5 ++ .../tests/testthat/test-helpers.R | 5 +- .../tests/testthat/test-utils.R | 4 +- 16 files changed, 183 insertions(+), 65 deletions(-) diff --git a/r/adbcdrivermanager/DESCRIPTION b/r/adbcdrivermanager/DESCRIPTION index 17c70324d3..a5b69b81fb 100644 --- a/r/adbcdrivermanager/DESCRIPTION +++ b/r/adbcdrivermanager/DESCRIPTION @@ -24,4 +24,4 @@ Config/build/bootstrap: TRUE URL: https://github.com/apache/arrow-adbc BugReports: https://github.com/apache/arrow-adbc/issues Imports: - nanoarrow + nanoarrow (>= 0.3.0) diff --git a/r/adbcdrivermanager/NAMESPACE b/r/adbcdrivermanager/NAMESPACE index 894851a0e6..5c671a2b3d 100644 --- a/r/adbcdrivermanager/NAMESPACE +++ b/r/adbcdrivermanager/NAMESPACE @@ -83,7 +83,6 @@ export(adbc_statement_release) export(adbc_statement_set_options) export(adbc_statement_set_sql_query) export(adbc_statement_set_substrait_plan) -export(adbc_stream_join) export(adbc_xptr_is_valid) export(adbc_xptr_move) export(execute_adbc) diff --git a/r/adbcdrivermanager/R/adbc.R b/r/adbcdrivermanager/R/adbc.R index 60e271d9ce..65a4e6ebb0 100644 --- a/r/adbcdrivermanager/R/adbc.R +++ b/r/adbcdrivermanager/R/adbc.R @@ -70,6 +70,8 @@ adbc_database_init_default <- function(driver, options = NULL, subclass = charac #' @rdname adbc_database_init #' @export adbc_database_release <- function(database) { + stop_for_nonzero_child_count(database) + error <- adbc_allocate_error() status <- .Call(RAdbcDatabaseRelease, database, error) stop_for_error(status, error) @@ -119,6 +121,8 @@ adbc_connection_init_default <- function(database, options = NULL, subclass = ch #' @rdname adbc_connection_init #' @export adbc_connection_release <- function(connection) { + stop_for_nonzero_child_count(connection) + if (isTRUE(connection$.release_database)) { database <- connection$database on.exit(adbc_database_release(database)) @@ -188,7 +192,7 @@ adbc_connection_get_info <- function(connection, info_codes = NULL) { ) stop_for_error(status, error) - out_stream + adbc_child_stream(connection, out_stream) } #' @rdname adbc_connection_get_info @@ -211,7 +215,7 @@ adbc_connection_get_objects <- function(connection, depth = 0L, catalog = NULL, ) stop_for_error(status, error) - out_stream + adbc_child_stream(connection, out_stream) } #' @rdname adbc_connection_get_info @@ -241,7 +245,7 @@ adbc_connection_get_table_types <- function(connection) { status <- .Call(RAdbcConnectionGetTableTypes, connection, out_stream, error) stop_for_error(status, error) - out_stream + adbc_child_stream(connection, out_stream) } #' @rdname adbc_connection_get_info @@ -258,7 +262,7 @@ adbc_connection_read_partition <- function(connection, serialized_partition) { ) stop_for_error(status, error) - out_stream + adbc_child_stream(connection, out_stream) } #' @rdname adbc_connection_get_info @@ -297,7 +301,7 @@ adbc_connection_get_statistic_names <- function(connection) { status <- .Call(RAdbcConnectionGetStatisticNames, connection, out_stream, error) stop_for_error(status, error) - out_stream + adbc_child_stream(connection, out_stream) } #' @rdname adbc_connection_get_info @@ -319,7 +323,7 @@ adbc_connection_get_statistics <- function(connection, catalog, db_schema, ) stop_for_error(status, error) - out_stream + adbc_child_stream(connection, out_stream) } #' @rdname adbc_connection_get_info @@ -384,6 +388,8 @@ adbc_statement_init_default <- function(connection, options = NULL, subclass = c #' @rdname adbc_statement_init #' @export adbc_statement_release <- function(statement) { + stop_for_nonzero_child_count(statement) + if (isTRUE(statement$.release_connection)) { connection <- statement$connection on.exit(adbc_connection_release(connection)) @@ -407,6 +413,8 @@ adbc_statement_release <- function(statement) { #' or object that can be coerced to one. #' @param schema A [nanoarrow_schema][nanoarrow::as_nanoarrow_schema] or object #' that can be coerced to one. +#' @param stream_join_parent Use `TRUE` to invalidate `statement` and tie its +#' lifecycle to `stream`. #' #' @return #' - `adbc_statement_set_sql_query()`, `adbc_statement_set_substrait_plan()`, @@ -483,9 +491,25 @@ adbc_statement_bind_stream <- function(statement, stream, schema = NULL) { #' @rdname adbc_statement_set_sql_query #' @export -adbc_statement_execute_query <- function(statement, stream = NULL) { +adbc_statement_execute_query <- function(statement, stream = NULL, + stream_join_parent = FALSE) { error <- adbc_allocate_error() - result <- .Call(RAdbcStatementExecuteQuery, statement, stream, error) + + if (is.null(stream)) { + result <- .Call(RAdbcStatementExecuteQuery, statement, NULL, error) + } else { + stream_tmp <- nanoarrow::nanoarrow_allocate_array_stream() + result <- .Call(RAdbcStatementExecuteQuery, statement, stream_tmp, error) + if (identical(result$status, 0L)) { + stream_tmp <- adbc_child_stream( + statement, + stream_tmp, + release_parent = stream_join_parent + ) + nanoarrow::nanoarrow_pointer_export(stream_tmp, stream) + } + } + stop_for_error(result$status, error) result$rows_affected } diff --git a/r/adbcdrivermanager/R/helpers.R b/r/adbcdrivermanager/R/helpers.R index 26aadd1e93..eea3a3ee8c 100644 --- a/r/adbcdrivermanager/R/helpers.R +++ b/r/adbcdrivermanager/R/helpers.R @@ -104,11 +104,7 @@ execute_adbc.default <- function(db_or_con, query, ..., bind = NULL, stream = NU adbc_statement_prepare(stmt) } - adbc_statement_execute_query(stmt, stream) - - if (!is.null(stream)) { - adbc_stream_join(stream, stmt) - } + adbc_statement_execute_query(stmt, stream, stream_join_parent = TRUE) invisible(db_or_con) } @@ -150,12 +146,13 @@ write_adbc.default <- function(tbl, db_or_con, target_table, ..., #' it is good practice to explicitly clean up these objects. These helpers #' are designed to make explicit and predictable cleanup easy to accomplish. #' -#' Note that you can use [adbc_connection_join()], -#' [adbc_statement_join()], and [adbc_stream_join()] +#' Note that you can use [adbc_connection_join()] and [adbc_statement_join()] #' to tie the lifecycle of the parent object to that of the child object. #' These functions mark any previous references to the parent object as #' released so you can still use local and with helpers to manage the parent -#' object before it is joined. +#' object before it is joined. Use `stream_join_parent = TRUE` in +#' [adbc_statement_execute_query()] to tie the lifecycle of a statement to +#' the output stream. #' #' @param x An ADBC database, ADBC connection, ADBC statement, or #' nanoarrow_array_stream returned from calls to an ADBC function. @@ -217,8 +214,6 @@ local_adbc <- function(x, .local_envir = parent.frame()) { #' @param database A database created with [adbc_database_init()] #' @param connection A connection created with [adbc_connection_init()] #' @param statement A statement created with [adbc_statement_init()] -#' @param stream A [nanoarrow_array_stream][nanoarrow::as_nanoarrow_array_stream] -#' @inheritParams with_adbc #' #' @return The input, invisibly. #' @export @@ -244,10 +239,15 @@ local_adbc <- function(x, .local_envir = parent.frame()) { #' adbc_connection_join <- function(connection, database) { assert_adbc(connection, "adbc_connection") - assert_adbc(database, "adbc_database") + + stopifnot( + identical(database, connection$database), + identical(database$.child_count, 1L) + ) connection$.release_database <- TRUE - connection$database <- adbc_xptr_move(database) + connection$database <- adbc_xptr_move(database, check_child_count = FALSE) + xptr_set_protected(connection, connection$database) invisible(connection) } @@ -255,42 +255,53 @@ adbc_connection_join <- function(connection, database) { #' @export adbc_statement_join <- function(statement, connection) { assert_adbc(statement, "adbc_statement") - assert_adbc(connection, "adbc_connection") + + stopifnot( + identical(connection, statement$connection), + identical(connection$.child_count, 1L) + ) statement$.release_connection <- TRUE - statement$connection <- adbc_xptr_move(connection) + statement$connection <- adbc_xptr_move(connection, check_child_count = FALSE) + xptr_set_protected(statement, statement$connection) invisible(statement) } -#' @rdname adbc_connection_join -#' @export -adbc_stream_join <- function(stream, x) { - if (utils::packageVersion("nanoarrow") < "0.1.0.9000") { - stop("adbc_stream_join_statement() requires nanoarrow >= 0.2.0") - } - - assert_adbc(stream, "nanoarrow_array_stream") - assert_adbc(x) +adbc_child_stream <- function(parent, stream, release_parent = FALSE) { + assert_adbc(parent) + # This finalizer will run immediately on release (if released explicitly + # on the main R thread) or on garbage collection otherwise. self_contained_finalizer <- function() { - try(adbc_release_non_null(x)) + try({ + parent$.child_count <- parent$.child_count - 1L + if (release_parent) { + adbc_release_non_null(parent) + } + }) } # Make sure we don't keep any variables around that aren't needed - # for the finalizer and make sure we invalidate the original statement + # for the finalizer and make sure we do keep around a strong reference + # to parent. self_contained_finalizer_env <- as.environment( - list(x = adbc_xptr_move(x)) + list( + parent = if (release_parent) adbc_xptr_move(parent) else parent, + release_parent = release_parent + ) ) parent.env(self_contained_finalizer_env) <- asNamespace("adbcdrivermanager") environment(self_contained_finalizer) <- self_contained_finalizer_env - # This finalizer will run immediately on release (if released explicitly - # on the main R thread) or on garbage collection otherwise. - - # Until the release version of nanoarrow contains this we will get a check - # warning for nanoarrow::array_stream_set_finalizer() - set_finalizer <- asNamespace("nanoarrow")[["array_stream_set_finalizer"]] - set_finalizer(stream, self_contained_finalizer) + # Set the finalizer using nanoarrow's method for this + stream_out <- nanoarrow::array_stream_set_finalizer( + stream, + self_contained_finalizer + ) - invisible(stream) + # Once we're sure this will succeed, increment the parent child count + # Use whatever version is in the finalizer env (we might have moved parent) + self_contained_finalizer_env$parent$.child_count <- + self_contained_finalizer_env$parent$.child_count + 1L + stream_out } diff --git a/r/adbcdrivermanager/R/utils.R b/r/adbcdrivermanager/R/utils.R index 1d7a01be1d..c747a612ed 100644 --- a/r/adbcdrivermanager/R/utils.R +++ b/r/adbcdrivermanager/R/utils.R @@ -16,13 +16,23 @@ # under the License. new_env <- function() { - new.env(parent = emptyenv()) + env <- new.env(parent = emptyenv()) + # A previous version of this just did env$.child_count <- 0L, + # which, perhaps because of compilation, results in env$.child_count + # referring to the exact same SEXP for every ADBC object! Use vector() + # to ensure a fresh allocation. + env$.child_count <- vector("integer", length = 1L) + env } xptr_env <- function(xptr) { .Call(RAdbcXptrEnv, xptr) } +xptr_set_protected <- function(xptr, prot) { + .Call(RAdbcXptrSetProtected, xptr, prot) +} + #' @export length.adbc_xptr <- function(x) { length(xptr_env(x)) @@ -80,6 +90,20 @@ str.adbc_xptr <- function(object, ...) { invisible(object) } +stop_for_nonzero_child_count <- function(obj) { + child_count <- obj$.child_count + if (!identical(child_count, 0L)) { + msg <- sprintf( + "<%s> has %d unreleased child object%s", + paste(class(obj), collapse = "/"), + child_count, + if (child_count != 1) "s" else "" + ) + cnd <- simpleError(msg, call = sys.call(-1)) + class(cnd) <- union("adbc_error_child_count_not_zero", class(cnd)) + stop(cnd) + } +} #' Low-level pointer details #' @@ -93,6 +117,8 @@ str.adbc_xptr <- function(object, ...) { #' #' @param x An 'adbc_database', 'adbc_connection', 'adbc_statement', or #' 'nanoarrow_array_stream' +#' @param check_child_count Ensures that `x` has a zero child count before +#' performing the move. This should almost always be `TRUE`. #' #' @return #' - `adbc_xptr_move()`: A freshly-allocated R object identical to `x` @@ -107,7 +133,11 @@ str.adbc_xptr <- function(object, ...) { #' adbc_xptr_is_valid(db) #' adbc_xptr_is_valid(db_new) #' -adbc_xptr_move <- function(x) { +adbc_xptr_move <- function(x, check_child_count = TRUE) { + if (check_child_count && (".child_count" %in% names(x))) { + stop_for_nonzero_child_count(x) + } + if (inherits(x, "adbc_database")) { .Call(RAdbcMoveDatabase, x) } else if (inherits(x, "adbc_connection")) { diff --git a/r/adbcdrivermanager/man/adbc_connection_join.Rd b/r/adbcdrivermanager/man/adbc_connection_join.Rd index 823a33b7be..418f9b1ddc 100644 --- a/r/adbcdrivermanager/man/adbc_connection_join.Rd +++ b/r/adbcdrivermanager/man/adbc_connection_join.Rd @@ -3,14 +3,11 @@ \name{adbc_connection_join} \alias{adbc_connection_join} \alias{adbc_statement_join} -\alias{adbc_stream_join} \title{Join the lifecycle of a unique parent to its child} \usage{ adbc_connection_join(connection, database) adbc_statement_join(statement, connection) - -adbc_stream_join(stream, x) } \arguments{ \item{connection}{A connection created with \code{\link[=adbc_connection_init]{adbc_connection_init()}}} @@ -18,11 +15,6 @@ adbc_stream_join(stream, x) \item{database}{A database created with \code{\link[=adbc_database_init]{adbc_database_init()}}} \item{statement}{A statement created with \code{\link[=adbc_statement_init]{adbc_statement_init()}}} - -\item{stream}{A \link[nanoarrow:as_nanoarrow_array_stream]{nanoarrow_array_stream}} - -\item{x}{An ADBC database, ADBC connection, ADBC statement, or -nanoarrow_array_stream returned from calls to an ADBC function.} } \value{ The input, invisibly. diff --git a/r/adbcdrivermanager/man/adbc_statement_set_sql_query.Rd b/r/adbcdrivermanager/man/adbc_statement_set_sql_query.Rd index f83f61c7c2..71a85751bf 100644 --- a/r/adbcdrivermanager/man/adbc_statement_set_sql_query.Rd +++ b/r/adbcdrivermanager/man/adbc_statement_set_sql_query.Rd @@ -24,7 +24,11 @@ adbc_statement_bind(statement, values, schema = NULL) adbc_statement_bind_stream(statement, stream, schema = NULL) -adbc_statement_execute_query(statement, stream = NULL) +adbc_statement_execute_query( + statement, + stream = NULL, + stream_join_parent = FALSE +) adbc_statement_execute_schema(statement) @@ -45,6 +49,9 @@ that can be coerced to one.} \item{stream}{A \link[nanoarrow:as_nanoarrow_array_stream]{nanoarrow_array_stream} or object that can be coerced to one.} + +\item{stream_join_parent}{Use \code{TRUE} to invalidate \code{statement} and tie its +lifecycle to \code{stream}.} } \value{ \itemize{ diff --git a/r/adbcdrivermanager/man/adbc_xptr_move.Rd b/r/adbcdrivermanager/man/adbc_xptr_move.Rd index 2dc2ceab24..bc7a92e7f3 100644 --- a/r/adbcdrivermanager/man/adbc_xptr_move.Rd +++ b/r/adbcdrivermanager/man/adbc_xptr_move.Rd @@ -5,13 +5,16 @@ \alias{adbc_xptr_is_valid} \title{Low-level pointer details} \usage{ -adbc_xptr_move(x) +adbc_xptr_move(x, check_child_count = TRUE) adbc_xptr_is_valid(x) } \arguments{ \item{x}{An 'adbc_database', 'adbc_connection', 'adbc_statement', or 'nanoarrow_array_stream'} + +\item{check_child_count}{Ensures that \code{x} has a zero child count before +performing the move. This should almost always be \code{TRUE}.} } \value{ \itemize{ diff --git a/r/adbcdrivermanager/man/with_adbc.Rd b/r/adbcdrivermanager/man/with_adbc.Rd index 5e16a9cad5..ba05dbef84 100644 --- a/r/adbcdrivermanager/man/with_adbc.Rd +++ b/r/adbcdrivermanager/man/with_adbc.Rd @@ -33,12 +33,13 @@ it is good practice to explicitly clean up these objects. These helpers are designed to make explicit and predictable cleanup easy to accomplish. } \details{ -Note that you can use \code{\link[=adbc_connection_join]{adbc_connection_join()}}, -\code{\link[=adbc_statement_join]{adbc_statement_join()}}, and \code{\link[=adbc_stream_join]{adbc_stream_join()}} +Note that you can use \code{\link[=adbc_connection_join]{adbc_connection_join()}} and \code{\link[=adbc_statement_join]{adbc_statement_join()}} to tie the lifecycle of the parent object to that of the child object. These functions mark any previous references to the parent object as released so you can still use local and with helpers to manage the parent -object before it is joined. +object before it is joined. Use \code{stream_join_parent = TRUE} in +\code{\link[=adbc_statement_execute_query]{adbc_statement_execute_query()}} to tie the lifecycle of a statement to +the output stream. } \examples{ # Using with_adbc(): diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c index 77c097b877..ad7ff6dcb9 100644 --- a/r/adbcdrivermanager/src/init.c +++ b/r/adbcdrivermanager/src/init.c @@ -99,6 +99,7 @@ SEXP RAdbcStatementExecutePartitions(SEXP statement_xptr, SEXP out_schema_xptr, SEXP partitions_xptr, SEXP error_xptr); SEXP RAdbcStatementCancel(SEXP statement_xptr, SEXP error_xptr); SEXP RAdbcXptrEnv(SEXP xptr); +SEXP RAdbcXptrSetProtected(SEXP xptr, SEXP prot); static const R_CallMethodDef CallEntries[] = { {"RAdbcVoidDriverInitFunc", (DL_FUNC)&RAdbcVoidDriverInitFunc, 0}, @@ -160,6 +161,7 @@ static const R_CallMethodDef CallEntries[] = { {"RAdbcStatementExecutePartitions", (DL_FUNC)&RAdbcStatementExecutePartitions, 4}, {"RAdbcStatementCancel", (DL_FUNC)&RAdbcStatementCancel, 2}, {"RAdbcXptrEnv", (DL_FUNC)&RAdbcXptrEnv, 1}, + {"RAdbcXptrSetProtected", (DL_FUNC)&RAdbcXptrSetProtected, 2}, {NULL, NULL, 0}}; /* end generated by tools/make-callentries.R */ diff --git a/r/adbcdrivermanager/src/radbc.cc b/r/adbcdrivermanager/src/radbc.cc index fe09de13d0..da27e4d04b 100644 --- a/r/adbcdrivermanager/src/radbc.cc +++ b/r/adbcdrivermanager/src/radbc.cc @@ -41,6 +41,24 @@ static void adbc_error_warn(int code, AdbcError* error, const char* context) { } } +static int adbc_update_parent_child_count(SEXP xptr, int delta) { + SEXP parent_xptr = R_ExternalPtrProtected(xptr); + if (parent_xptr == R_NilValue) { + return NA_INTEGER; + } + + SEXP parent_env = R_ExternalPtrTag(parent_xptr); + if (parent_env == R_NilValue) { + return NA_INTEGER; + } + + SEXP child_count_sexp = Rf_findVarInFrame(parent_env, Rf_install(".child_count")); + int* child_count = INTEGER(child_count_sexp); + int old_value = child_count[0]; + child_count[0] = child_count[0] + delta; + return old_value; +} + static void finalize_driver_xptr(SEXP driver_xptr) { auto driver = reinterpret_cast(R_ExternalPtrAddr(driver_xptr)); if (driver == nullptr) { @@ -186,6 +204,9 @@ static void finalize_connection_xptr(SEXP connection_xptr) { AdbcError error = ADBC_ERROR_INIT; int status = AdbcConnectionRelease(connection, &error); adbc_error_warn(status, &error, "finalize_connection_xptr()"); + if (status == ADBC_STATUS_OK) { + adbc_update_parent_child_count(connection_xptr, -1); + } } adbc_xptr_default_finalize(connection_xptr); @@ -236,6 +257,7 @@ extern "C" SEXP RAdbcConnectionInit(SEXP connection_xptr, SEXP database_xptr, // Keep the database pointer alive for as long as the connection pointer // is alive R_SetExternalPtrProtected(connection_xptr, database_xptr); + adbc_update_parent_child_count(connection_xptr, 1); } return adbc_wrap_status(result); @@ -245,6 +267,10 @@ extern "C" SEXP RAdbcConnectionRelease(SEXP connection_xptr, SEXP error_xptr) { auto connection = adbc_from_xptr(connection_xptr); auto error = adbc_from_xptr(error_xptr); int status = AdbcConnectionRelease(connection, error); + if (status == ADBC_STATUS_OK) { + adbc_update_parent_child_count(connection_xptr, -1); + } + return adbc_wrap_status(status); } @@ -384,6 +410,9 @@ static void finalize_statement_xptr(SEXP statement_xptr) { AdbcError error = ADBC_ERROR_INIT; int status = AdbcStatementRelease(statement, &error); adbc_error_warn(status, &error, "finalize_statement_xptr()"); + if (status == ADBC_STATUS_OK) { + adbc_update_parent_child_count(statement_xptr, -1); + } } adbc_xptr_default_finalize(statement_xptr); @@ -401,6 +430,7 @@ extern "C" SEXP RAdbcStatementNew(SEXP connection_xptr) { adbc_error_stop(status, &error); R_SetExternalPtrProtected(statement_xptr, connection_xptr); + adbc_update_parent_child_count(statement_xptr, 1); UNPROTECT(1); return statement_xptr; @@ -430,6 +460,10 @@ extern "C" SEXP RAdbcStatementRelease(SEXP statement_xptr, SEXP error_xptr) { auto statement = adbc_from_xptr(statement_xptr); auto error = adbc_from_xptr(error_xptr); int status = AdbcStatementRelease(statement, error); + if (status == ADBC_STATUS_OK) { + adbc_update_parent_child_count(statement_xptr, -1); + } + return adbc_wrap_status(status); } diff --git a/r/adbcdrivermanager/src/radbc.h b/r/adbcdrivermanager/src/radbc.h index 27772802ad..4f1ec28317 100644 --- a/r/adbcdrivermanager/src/radbc.h +++ b/r/adbcdrivermanager/src/radbc.h @@ -22,6 +22,8 @@ #include +#include + template static inline const char* adbc_xptr_class(); diff --git a/r/adbcdrivermanager/src/utils.c b/r/adbcdrivermanager/src/utils.c index 71258fc06a..74e5c90c7e 100644 --- a/r/adbcdrivermanager/src/utils.c +++ b/r/adbcdrivermanager/src/utils.c @@ -26,3 +26,14 @@ SEXP RAdbcXptrEnv(SEXP xptr) { return R_ExternalPtrTag(xptr); } + +SEXP RAdbcXptrSetProtected(SEXP xptr, SEXP prot) { + if (TYPEOF(xptr) != EXTPTRSXP) { + Rf_error("object is not an external pointer"); + } + + SEXP old_prot = PROTECT(R_ExternalPtrProtected(xptr)); + R_SetExternalPtrProtected(xptr, prot); + UNPROTECT(1); + return old_prot; +} diff --git a/r/adbcdrivermanager/tests/testthat/test-driver_monkey.R b/r/adbcdrivermanager/tests/testthat/test-driver_monkey.R index d3ed532a89..5275b408f8 100644 --- a/r/adbcdrivermanager/tests/testthat/test-driver_monkey.R +++ b/r/adbcdrivermanager/tests/testthat/test-driver_monkey.R @@ -27,6 +27,11 @@ test_that("the monkey driver sees, and the monkey driver does", { stream <- nanoarrow::nanoarrow_allocate_array_stream() expect_identical(adbc_statement_execute_query(stmt, stream), -1) expect_identical(as.data.frame(stream$get_next()), input) + expect_error( + adbc_statement_release(stmt), + class = "adbc_error_child_count_not_zero" + ) + stream$release() adbc_statement_release(stmt) stmt <- adbc_statement_init(con, input) diff --git a/r/adbcdrivermanager/tests/testthat/test-helpers.R b/r/adbcdrivermanager/tests/testthat/test-helpers.R index 3e65053e54..4444835962 100644 --- a/r/adbcdrivermanager/tests/testthat/test-helpers.R +++ b/r/adbcdrivermanager/tests/testthat/test-helpers.R @@ -129,8 +129,6 @@ test_that("joiners work for databases, connections, and statements", { }) test_that("joiners work with streams", { - skip_if_not(packageVersion("nanoarrow") >= "0.1.0.9000") - stream <- local({ db <- local_adbc(adbc_database_init(adbc_driver_monkey())) @@ -143,8 +141,7 @@ test_that("joiners work with streams", { expect_false(adbc_xptr_is_valid(con)) stream <- local_adbc(nanoarrow::nanoarrow_allocate_array_stream()) - adbc_statement_execute_query(stmt, stream) - adbc_stream_join(stream, stmt) + adbc_statement_execute_query(stmt, stream, stream_join_parent = TRUE) expect_false(adbc_xptr_is_valid(stmt)) adbc_xptr_move(stream) diff --git a/r/adbcdrivermanager/tests/testthat/test-utils.R b/r/adbcdrivermanager/tests/testthat/test-utils.R index 0a9c7a49d1..e99209c929 100644 --- a/r/adbcdrivermanager/tests/testthat/test-utils.R +++ b/r/adbcdrivermanager/tests/testthat/test-utils.R @@ -17,8 +17,8 @@ test_that("external pointer embedded environment works", { db <- adbc_database_init(adbc_driver_void()) - expect_identical(names(db), "driver") - expect_identical(length(db), 1L) + expect_setequal(names(db), c("driver", ".child_count")) + expect_identical(db$.child_count, 0L) db$key <- "value" expect_identical(db$key, "value") From 5ed0f5b07d193b443107a6f42feb3b8a467a981e Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 19 Dec 2023 15:25:42 -0500 Subject: [PATCH 19/51] docs: update driver implementation status (#1383) Fixes #1382. --- docs/source/driver/postgresql.rst | 131 +++++++++++++++++- docs/source/driver/status.rst | 12 +- docs/source/python/recipe/postgresql.rst | 12 ++ .../recipe/postgresql_create_append_table.py | 27 +++- .../recipe/postgresql_create_temp_table.py | 89 ++++++++++++ .../recipe/postgresql_get_query_schema.py | 55 ++++++++ .../adbc_driver_manager/dbapi.py | 2 +- 7 files changed, 315 insertions(+), 13 deletions(-) create mode 100644 docs/source/python/recipe/postgresql_create_temp_table.py create mode 100644 docs/source/python/recipe/postgresql_get_query_schema.py diff --git a/docs/source/driver/postgresql.rst b/docs/source/driver/postgresql.rst index 7ec5b7f589..36349644ba 100644 --- a/docs/source/driver/postgresql.rst +++ b/docs/source/driver/postgresql.rst @@ -31,7 +31,7 @@ overall approach. .. _libpq: https://www.postgresql.org/docs/current/libpq.html .. _pgeon: https://github.com/0x0L/pgeon -.. note:: The PostgreSQL driver is experimental. +.. note:: The PostgreSQL driver is in beta. Performance/optimization and support for complex types and different ADBC features is still ongoing. @@ -188,6 +188,129 @@ PostgreSQL allows defining new types at runtime, so the driver must build a mapping of available types. This is currently done once at startup. -Type support is currently limited. Parameter binding and bulk -ingestion support int16, int32, int64, and string. Reading result -sets is limited to int32, int64, float, double, and string. +Type support is currently limited depending on the type and whether it is +being read or written. + +.. list-table:: Arrow type to PostgreSQL type mapping + :header-rows: 1 + + * - Arrow Type + - As Bind Parameter + - In Bulk Ingestion + + * - binary + - Y + - BYTEA + + * - bool + - Y + - BOOLEAN + + * - date32 + - Y + - DATE + + * - date64 + - N + - N + + * - dictionary + - (as unpacked type) + - (as unpacked type, only for binary/string) + + * - duration + - Y + - INTERVAL + + * - float32 + - Y + - REAL + + * - float64 + - Y + - DOUBLE PRECISION + + * - int8 + - Y + - SMALLINT + + * - int16 + - Y + - SMALLINT + + * - int32 + - Y + - INTEGER + + * - int64 + - Y + - BIGINT + + * - large_binary + - N + - N + + * - large_string + - Y + - TEXT + + * - month_day_nano_interval + - Y + - INTERVAL + + * - NA + - N + - N + + * - string + - Y + - TEXT + + * - timestamp + - Y + - TIMESTAMP/TIMESTAMP WITH TIMEZONE + +.. list-table:: PostgreSQL type to Arrow type mapping + :header-rows: 1 + + * - PostgreSQL Type + - In Result Set + + * - ARRAY + - list + * - BIGINT + - int64 + * - BINARY + - binary + * - BOOLEAN + - bool + * - CHAR + - utf8 + * - DATE + - date32 + * - DOUBLE PRECISION + - float64 + * - INTEGER + - int32 + * - INTERVAL + - month_day_nano_interval + * - NUMERIC + - utf8 [#numeric-utf8]_ + * - REAL + - float32 + * - SMALLINT + - int16 + * - TEXT + - utf8 + * - TIME + - time64 + * - TIMESTAMP WITH TIME ZONE + - timestamp[unit, UTC] + * - TIMESTAMP WITHOUT TIME ZONE + - timestamp[unit] + * - VARCHAR + - utf8 + +.. [#numeric-utf8] NUMERIC types are read as the string representation of the + value, because the PostgreSQL NUMERIC type cannot be + losslessly converted to the Arrow decimal types. diff --git a/docs/source/driver/status.rst b/docs/source/driver/status.rst index 64c50532b3..d295bc3f7c 100644 --- a/docs/source/driver/status.rst +++ b/docs/source/driver/status.rst @@ -34,6 +34,11 @@ Implementation Status - Implementation Language - Status + * - BigQuery + - C# + - C# + - Experimental + * - Flight SQL (Go) - C, Go - Go @@ -64,11 +69,6 @@ Implementation Status - Go - Experimental - * - BigQuery - - C# - - C# - - Experimental - .. [#supported-languages] C drivers are usable from Go, Python, and Ruby as well. Feature Support @@ -177,10 +177,10 @@ Update Queries - Y * - PostgreSQL + - N/A - Y - Y - Y - - N/A - Y * - SQLite diff --git a/docs/source/python/recipe/postgresql.rst b/docs/source/python/recipe/postgresql.rst index dbf28adba8..7e93a47912 100644 --- a/docs/source/python/recipe/postgresql.rst +++ b/docs/source/python/recipe/postgresql.rst @@ -24,11 +24,18 @@ Authenticate with a username and password .. recipe:: postgresql_authenticate.py +.. _recipe-postgresql-create-append: + Create/append to a table from an Arrow table ============================================ .. recipe:: postgresql_create_append_table.py +Create/append to a temporary table +================================== + +.. recipe:: postgresql_create_temp_table.py + Execute a statement with bind parameters ======================================== @@ -39,6 +46,11 @@ Get the Arrow schema of a table .. recipe:: postgresql_get_table_schema.py +Get the Arrow schema of a query +=============================== + +.. recipe:: postgresql_get_query_schema.py + List catalogs, schemas, and tables ================================== diff --git a/docs/source/python/recipe/postgresql_create_append_table.py b/docs/source/python/recipe/postgresql_create_append_table.py index 54331ba097..36e29b9386 100644 --- a/docs/source/python/recipe/postgresql_create_append_table.py +++ b/docs/source/python/recipe/postgresql_create_append_table.py @@ -28,10 +28,11 @@ uri = os.environ["ADBC_POSTGRESQL_TEST_URI"] conn = adbc_driver_postgresql.dbapi.connect(uri) -#: For the purposes of testing, we'll first make sure the table -#: doesn't exist. +#: For the purposes of testing, we'll first make sure the tables we're about +#: to use don't exist. with conn.cursor() as cur: cur.execute("DROP TABLE IF EXISTS example") + cur.execute("DROP TABLE IF EXISTS example2") #: Now we can create the table. with conn.cursor() as cur: @@ -77,4 +78,26 @@ cur.execute("SELECT COUNT(*) FROM example") assert cur.fetchone() == (8,) +#: We can also choose to create the table if it doesn't exist, and otherwise +#: append. + +with conn.cursor() as cur: + cur.adbc_ingest("example2", data, mode="create_append") + + cur.execute("SELECT COUNT(*) FROM example2") + assert cur.fetchone() == (4,) + + cur.adbc_ingest("example2", data, mode="create_append") + + cur.execute("SELECT COUNT(*) FROM example2") + assert cur.fetchone() == (8,) + +#: Finally, we can replace the table. + +with conn.cursor() as cur: + cur.adbc_ingest("example", data.slice(0, 2), mode="replace") + + cur.execute("SELECT COUNT(*) FROM example") + assert cur.fetchone() == (2,) + conn.close() diff --git a/docs/source/python/recipe/postgresql_create_temp_table.py b/docs/source/python/recipe/postgresql_create_temp_table.py new file mode 100644 index 0000000000..2d762b9a49 --- /dev/null +++ b/docs/source/python/recipe/postgresql_create_temp_table.py @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# RECIPE STARTS HERE +#: ADBC allows creating and appending to temporary tables as well. + +import os + +import pyarrow + +import adbc_driver_postgresql.dbapi + +uri = os.environ["ADBC_POSTGRESQL_TEST_URI"] +conn = adbc_driver_postgresql.dbapi.connect(uri) + +#: For the purposes of testing, we'll first make sure the tables we're about +#: to use don't exist. +with conn.cursor() as cur: + cur.execute("DROP TABLE IF EXISTS example") + +#: To create a temporary table, just specify the option "temporary". +data = pyarrow.table( + [ + [1, 2, None, 4], + ], + schema=pyarrow.schema( + [ + ("ints", "int32"), + ] + ), +) + +with conn.cursor() as cur: + cur.adbc_ingest("example", data, mode="create", temporary=True) + +conn.commit() + +#: After ingestion, we can fetch the result. +with conn.cursor() as cur: + cur.execute("SELECT * FROM example") + assert cur.fetchone() == (1,) + assert cur.fetchone() == (2,) + + cur.execute("SELECT COUNT(*) FROM example") + assert cur.fetchone() == (4,) + +#: Temporary tables are separate from regular tables, even if they have the +#: same name. + +with conn.cursor() as cur: + cur.adbc_ingest("example", data.slice(0, 2), mode="create", temporary=False) + +conn.commit() + +with conn.cursor() as cur: + #: Because we have two tables with the same name, we have to explicitly + #: reference the normal temporary table here. + cur.execute("SELECT COUNT(*) FROM public.example") + assert cur.fetchone() == (2,) + + cur.execute("SELECT COUNT(*) FROM example") + assert cur.fetchone() == (4,) + +conn.close() + +#: After closing the connection, the temporary table is implicitly dropped. +#: If we reconnect, the table won't exist; we'll see only the 'normal' table. + +with adbc_driver_postgresql.dbapi.connect(uri) as conn: + with conn.cursor() as cur: + cur.execute("SELECT COUNT(*) FROM example") + assert cur.fetchone() == (2,) + +#: All the regular ingestion options apply to temporary tables, too. See +#: :ref:`recipe-postgresql-create-append` for more examples. diff --git a/docs/source/python/recipe/postgresql_get_query_schema.py b/docs/source/python/recipe/postgresql_get_query_schema.py new file mode 100644 index 0000000000..2568453924 --- /dev/null +++ b/docs/source/python/recipe/postgresql_get_query_schema.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# RECIPE STARTS HERE + +#: ADBC lets you get the schema of a result set, without executing the query. + +import os + +import pyarrow + +import adbc_driver_postgresql.dbapi + +uri = os.environ["ADBC_POSTGRESQL_TEST_URI"] +conn = adbc_driver_postgresql.dbapi.connect(uri) + +#: We'll create an example table to test. +with conn.cursor() as cur: + cur.execute("DROP TABLE IF EXISTS example") + cur.execute("CREATE TABLE example (ints INT, bigints BIGINT)") + +conn.commit() + +expected = pyarrow.schema( + [ + ("ints", "int32"), + ("bigints", "int64"), + ] +) + +with conn.cursor() as cur: + assert cur.adbc_execute_schema("SELECT * FROM example") == expected + + #: PostgreSQL doesn't know the type here, so it just returns a guess. + assert cur.adbc_execute_schema("SELECT $1 AS res") == pyarrow.schema( + [ + ("res", "string"), + ] + ) + +conn.close() diff --git a/python/adbc_driver_manager/adbc_driver_manager/dbapi.py b/python/adbc_driver_manager/adbc_driver_manager/dbapi.py index 4c36ad5cbd..8f06ed0396 100644 --- a/python/adbc_driver_manager/adbc_driver_manager/dbapi.py +++ b/python/adbc_driver_manager/adbc_driver_manager/dbapi.py @@ -807,7 +807,7 @@ def adbc_ingest( The Arrow data to insert. This can be a pyarrow RecordBatch, Table or RecordBatchReader, or any Arrow-compatible data that implements the Arrow PyCapsule Protocol (i.e. has an ``__arrow_c_array__`` - or ``__arrow_c_stream__ ``method). + or ``__arrow_c_stream__`` method). mode How to deal with existing data: From 95897540f7582699e99da26f99cde8f825ca828f Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 20 Dec 2023 09:58:17 -0500 Subject: [PATCH 20/51] feat(c/driver/postgresql): set rows_affected appropriately (#1384) Fixes #1226. --- c/driver/postgresql/postgresql_test.cc | 6 ------ c/driver/postgresql/statement.cc | 13 ++++++++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/c/driver/postgresql/postgresql_test.cc b/c/driver/postgresql/postgresql_test.cc index e1f95a4901..5e04b45548 100644 --- a/c/driver/postgresql/postgresql_test.cc +++ b/c/driver/postgresql/postgresql_test.cc @@ -824,12 +824,6 @@ class PostgresStatementTest : public ::testing::Test, void TestSqlPrepareErrorParamCountMismatch() { GTEST_SKIP() << "Not yet implemented"; } void TestSqlPrepareGetParameterSchema() { GTEST_SKIP() << "Not yet implemented"; } void TestSqlPrepareSelectParams() { GTEST_SKIP() << "Not yet implemented"; } - void TestSqlQueryRowsAffectedDelete() { - GTEST_SKIP() << "Cannot query rows affected in delete (not implemented)"; - } - void TestSqlQueryRowsAffectedDeleteStream() { - GTEST_SKIP() << "Cannot query rows affected in delete stream (not implemented)"; - } void TestConcurrentStatements() { // TODO: refactor driver so that we read all the data as soon as diff --git a/c/driver/postgresql/statement.cc b/c/driver/postgresql/statement.cc index 6f1bb9bd54..656e187627 100644 --- a/c/driver/postgresql/statement.cc +++ b/c/driver/postgresql/statement.cc @@ -1306,7 +1306,18 @@ AdbcStatusCode PostgresStatement::ExecuteUpdateQuery(int64_t* rows_affected, PQclear(result); return code; } - if (rows_affected) *rows_affected = PQntuples(reader_.result_); + if (rows_affected) { + if (status == PGRES_TUPLES_OK) { + *rows_affected = PQntuples(reader_.result_); + } else { + // In theory, PQcmdTuples would work here, but experimentally it gives + // an empty string even for a DELETE. (Also, why does it return a + // string...) Possibly, it doesn't work because we use PQexecPrepared + // but the docstring is careful to specify it works on an EXECUTE of a + // prepared statement. + *rows_affected = -1; + } + } PQclear(result); return ADBC_STATUS_OK; } From 5ca9c29e684c364ab38184a9ae3ba4090cd8b9ed Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 20 Dec 2023 12:49:56 -0500 Subject: [PATCH 21/51] fix(c/driver/postgresql): support catalog arg of GetTableSchema (#1387) Fixes #1339. --- c/driver/postgresql/connection.cc | 37 ++++---------- c/driver/postgresql/postgresql_test.cc | 34 +++++++++++-- c/validation/adbc_validation.cc | 51 +++++++++++++++++-- c/validation/adbc_validation.h | 33 +++++++++++- ci/conda_env_cpp_lint.txt | 4 +- .../recipe/postgresql_get_table_schema.py | 26 +++++++++- 6 files changed, 147 insertions(+), 38 deletions(-) diff --git a/c/driver/postgresql/connection.cc b/c/driver/postgresql/connection.cc index d389a66c2b..deae3171c5 100644 --- a/c/driver/postgresql/connection.cc +++ b/c/driver/postgresql/connection.cc @@ -1147,38 +1147,23 @@ AdbcStatusCode PostgresConnection::GetTableSchema(const char* catalog, struct ArrowSchema* schema, struct AdbcError* error) { AdbcStatusCode final_status = ADBC_STATUS_OK; - struct StringBuilder query; - std::memset(&query, 0, sizeof(query)); - std::vector params; - if (StringBuilderInit(&query, /*initial_size=*/256) != 0) return ADBC_STATUS_INTERNAL; - if (StringBuilderAppend( - &query, "%s", - "SELECT attname, atttypid " - "FROM pg_catalog.pg_class AS cls " - "INNER JOIN pg_catalog.pg_attribute AS attr ON cls.oid = attr.attrelid " - "INNER JOIN pg_catalog.pg_type AS typ ON attr.atttypid = typ.oid " - "WHERE attr.attnum >= 0 AND cls.oid = ") != 0) - return ADBC_STATUS_INTERNAL; + std::string query = + "SELECT attname, atttypid " + "FROM pg_catalog.pg_class AS cls " + "INNER JOIN pg_catalog.pg_attribute AS attr ON cls.oid = attr.attrelid " + "INNER JOIN pg_catalog.pg_type AS typ ON attr.atttypid = typ.oid " + "WHERE attr.attnum >= 0 AND cls.oid = $1::regclass::oid"; + std::vector params; if (db_schema != nullptr) { - if (StringBuilderAppend(&query, "%s", "$1.")) { - StringBuilderReset(&query); - return ADBC_STATUS_INTERNAL; - } - params.push_back(db_schema); - } - - if (StringBuilderAppend(&query, "%s%" PRIu64 "%s", "$", - static_cast(params.size() + 1), "::regclass::oid")) { - StringBuilderReset(&query); - return ADBC_STATUS_INTERNAL; + params.push_back(std::string(db_schema) + "." + table_name); + } else { + params.push_back(table_name); } - params.push_back(table_name); PqResultHelper result_helper = - PqResultHelper{conn_, std::string(query.buffer), params, error}; - StringBuilderReset(&query); + PqResultHelper{conn_, std::string(query.c_str()), params, error}; RAISE_ADBC(result_helper.Prepare()); auto result = result_helper.Execute(); diff --git a/c/driver/postgresql/postgresql_test.cc b/c/driver/postgresql/postgresql_test.cc index 5e04b45548..2327767a4e 100644 --- a/c/driver/postgresql/postgresql_test.cc +++ b/c/driver/postgresql/postgresql_test.cc @@ -61,6 +61,18 @@ class PostgresQuirks : public adbc_validation::DriverQuirks { return AdbcStatementRelease(&statement.value, error); } + AdbcStatusCode DropTable(struct AdbcConnection* connection, const std::string& name, + const std::string& db_schema, + struct AdbcError* error) const override { + Handle statement; + RAISE_ADBC(AdbcStatementNew(connection, &statement.value, error)); + + std::string query = "DROP TABLE IF EXISTS \"" + db_schema + "\".\"" + name + "\""; + RAISE_ADBC(AdbcStatementSetSqlQuery(&statement.value, query.c_str(), error)); + RAISE_ADBC(AdbcStatementExecuteQuery(&statement.value, nullptr, nullptr, error)); + return AdbcStatementRelease(&statement.value, error); + } + AdbcStatusCode DropTempTable(struct AdbcConnection* connection, const std::string& name, struct AdbcError* error) const override { Handle statement; @@ -83,6 +95,18 @@ class PostgresQuirks : public adbc_validation::DriverQuirks { return AdbcStatementRelease(&statement.value, error); } + AdbcStatusCode EnsureDbSchema(struct AdbcConnection* connection, + const std::string& name, + struct AdbcError* error) const override { + Handle statement; + RAISE_ADBC(AdbcStatementNew(connection, &statement.value, error)); + + std::string query = "CREATE SCHEMA IF NOT EXISTS \"" + name + "\""; + RAISE_ADBC(AdbcStatementSetSqlQuery(&statement.value, query.c_str(), error)); + RAISE_ADBC(AdbcStatementExecuteQuery(&statement.value, nullptr, nullptr, error)); + return AdbcStatementRelease(&statement.value, error); + } + std::string BindParameter(int index) const override { return "$" + std::to_string(index + 1); } @@ -343,7 +367,7 @@ TEST_F(PostgresConnectionTest, GetObjectsGetAllFindsPrimaryKey) { ASSERT_THAT(AdbcStatementExecuteQuery(&statement, &reader.stream.value, &reader.rows_affected, &error), IsOkStatus(&error)); - ASSERT_EQ(reader.rows_affected, 0); + ASSERT_EQ(reader.rows_affected, -1); ASSERT_NO_FATAL_FAILURE(reader.GetSchema()); ASSERT_NO_FATAL_FAILURE(reader.Next()); ASSERT_EQ(reader.array->release, nullptr); @@ -416,7 +440,7 @@ TEST_F(PostgresConnectionTest, GetObjectsGetAllFindsForeignKey) { ASSERT_THAT(AdbcStatementExecuteQuery(&statement, &reader.stream.value, &reader.rows_affected, &error), IsOkStatus(&error)); - ASSERT_EQ(reader.rows_affected, 0); + ASSERT_EQ(reader.rows_affected, -1); ASSERT_NO_FATAL_FAILURE(reader.GetSchema()); ASSERT_NO_FATAL_FAILURE(reader.Next()); ASSERT_EQ(reader.array->release, nullptr); @@ -435,7 +459,7 @@ TEST_F(PostgresConnectionTest, GetObjectsGetAllFindsForeignKey) { ASSERT_THAT(AdbcStatementExecuteQuery(&statement, &reader.stream.value, &reader.rows_affected, &error), IsOkStatus(&error)); - ASSERT_EQ(reader.rows_affected, 0); + ASSERT_EQ(reader.rows_affected, -1); ASSERT_NO_FATAL_FAILURE(reader.GetSchema()); ASSERT_NO_FATAL_FAILURE(reader.Next()); ASSERT_EQ(reader.array->release, nullptr); @@ -1162,7 +1186,7 @@ TEST_F(PostgresStatementTest, UpdateInExecuteQuery) { ASSERT_THAT(AdbcStatementExecuteQuery(&statement, &reader.stream.value, &reader.rows_affected, &error), IsOkStatus(&error)); - ASSERT_EQ(reader.rows_affected, 0); + ASSERT_EQ(reader.rows_affected, -1); ASSERT_NO_FATAL_FAILURE(reader.GetSchema()); ASSERT_NO_FATAL_FAILURE(reader.Next()); ASSERT_EQ(reader.array->release, nullptr); @@ -1177,7 +1201,7 @@ TEST_F(PostgresStatementTest, UpdateInExecuteQuery) { ASSERT_THAT(AdbcStatementExecuteQuery(&statement, &reader.stream.value, &reader.rows_affected, &error), IsOkStatus(&error)); - ASSERT_EQ(reader.rows_affected, 0); + ASSERT_EQ(reader.rows_affected, -1); ASSERT_NO_FATAL_FAILURE(reader.GetSchema()); ASSERT_NO_FATAL_FAILURE(reader.Next()); ASSERT_EQ(reader.array->release, nullptr); diff --git a/c/validation/adbc_validation.cc b/c/validation/adbc_validation.cc index d30aa0a979..97d12be169 100644 --- a/c/validation/adbc_validation.cc +++ b/c/validation/adbc_validation.cc @@ -70,7 +70,9 @@ bool iequals(std::string_view s1, std::string_view s2) { // DriverQuirks AdbcStatusCode DoIngestSampleTable(struct AdbcConnection* connection, - const std::string& name, struct AdbcError* error) { + const std::string& name, + std::optional db_schema, + struct AdbcError* error) { Handle schema; Handle array; struct ArrowError na_error; @@ -84,6 +86,10 @@ AdbcStatusCode DoIngestSampleTable(struct AdbcConnection* connection, CHECK_OK(AdbcStatementNew(connection, &statement.value, error)); CHECK_OK(AdbcStatementSetOption(&statement.value, ADBC_INGEST_OPTION_TARGET_TABLE, name.c_str(), error)); + if (db_schema.has_value()) { + CHECK_OK(AdbcStatementSetOption(&statement.value, ADBC_INGEST_OPTION_TARGET_DB_SCHEMA, + db_schema->c_str(), error)); + } CHECK_OK(AdbcStatementBind(&statement.value, &array.value, &schema.value, error)); CHECK_OK(AdbcStatementExecuteQuery(&statement.value, nullptr, nullptr, error)); CHECK_OK(AdbcStatementRelease(&statement.value, error)); @@ -91,7 +97,8 @@ AdbcStatusCode DoIngestSampleTable(struct AdbcConnection* connection, } void IngestSampleTable(struct AdbcConnection* connection, struct AdbcError* error) { - ASSERT_THAT(DoIngestSampleTable(connection, "bulk_ingest", error), IsOkStatus(error)); + ASSERT_THAT(DoIngestSampleTable(connection, "bulk_ingest", std::nullopt, error), + IsOkStatus(error)); } AdbcStatusCode DriverQuirks::EnsureSampleTable(struct AdbcConnection* connection, @@ -107,7 +114,17 @@ AdbcStatusCode DriverQuirks::CreateSampleTable(struct AdbcConnection* connection if (!supports_bulk_ingest(ADBC_INGEST_OPTION_MODE_CREATE)) { return ADBC_STATUS_NOT_IMPLEMENTED; } - return DoIngestSampleTable(connection, name, error); + return DoIngestSampleTable(connection, name, std::nullopt, error); +} + +AdbcStatusCode DriverQuirks::CreateSampleTable(struct AdbcConnection* connection, + const std::string& name, + const std::string& schema, + struct AdbcError* error) const { + if (!supports_bulk_ingest(ADBC_INGEST_OPTION_MODE_CREATE)) { + return ADBC_STATUS_NOT_IMPLEMENTED; + } + return DoIngestSampleTable(connection, name, schema, error); } //------------------------------------------------------------ @@ -431,6 +448,34 @@ void ConnectionTest::TestMetadataGetTableSchema() { {"strings", NANOARROW_TYPE_STRING, NULLABLE}})); } +void ConnectionTest::TestMetadataGetTableSchemaDbSchema() { + ASSERT_THAT(AdbcConnectionNew(&connection, &error), IsOkStatus(&error)); + ASSERT_THAT(AdbcConnectionInit(&connection, &database, &error), IsOkStatus(&error)); + + auto status = quirks()->EnsureDbSchema(&connection, "otherschema", &error); + if (status == ADBC_STATUS_NOT_IMPLEMENTED) { + GTEST_SKIP() << "Schema not supported"; + return; + } + ASSERT_THAT(status, IsOkStatus(&error)); + + ASSERT_THAT(quirks()->DropTable(&connection, "bulk_ingest", "otherschema", &error), + IsOkStatus(&error)); + ASSERT_THAT( + quirks()->CreateSampleTable(&connection, "bulk_ingest", "otherschema", &error), + IsOkStatus(&error)); + + Handle schema; + ASSERT_THAT(AdbcConnectionGetTableSchema(&connection, /*catalog=*/nullptr, + /*db_schema=*/"otherschema", "bulk_ingest", + &schema.value, &error), + IsOkStatus(&error)); + + ASSERT_NO_FATAL_FAILURE( + CompareSchema(&schema.value, {{"int64s", NANOARROW_TYPE_INT64, NULLABLE}, + {"strings", NANOARROW_TYPE_STRING, NULLABLE}})); +} + void ConnectionTest::TestMetadataGetTableSchemaEscaping() { if (!quirks()->supports_bulk_ingest(ADBC_INGEST_OPTION_MODE_CREATE)) { GTEST_SKIP(); diff --git a/c/validation/adbc_validation.h b/c/validation/adbc_validation.h index 874d9a0584..30a20491eb 100644 --- a/c/validation/adbc_validation.h +++ b/c/validation/adbc_validation.h @@ -50,6 +50,13 @@ class DriverQuirks { return ADBC_STATUS_OK; } + virtual AdbcStatusCode DropTable(struct AdbcConnection* connection, + const std::string& name, + const std::string& db_schema, + struct AdbcError* error) const { + return ADBC_STATUS_NOT_IMPLEMENTED; + } + /// \brief Drop the given temporary table. Used by tests to reset state. virtual AdbcStatusCode DropTempTable(struct AdbcConnection* connection, const std::string& name, @@ -68,13 +75,33 @@ class DriverQuirks { const std::string& name, struct AdbcError* error) const; + /// \brief Create a schema for testing. + virtual AdbcStatusCode EnsureDbSchema(struct AdbcConnection* connection, + const std::string& name, + struct AdbcError* error) const { + return ADBC_STATUS_NOT_IMPLEMENTED; + } + + /// \brief Create a table of sample data with a fixed schema for testing. + /// + /// The table should have two columns: + /// - "int64s" with Arrow type int64. + /// - "strings" with Arrow type utf8. + virtual AdbcStatusCode CreateSampleTable(struct AdbcConnection* connection, + const std::string& name, + struct AdbcError* error) const; + /// \brief Create a table of sample data with a fixed schema for testing. /// + /// Create it in the given schema. Specify "" for the default schema. + /// Return NOT_IMPLEMENTED if not supported by this backend. + /// /// The table should have two columns: /// - "int64s" with Arrow type int64. /// - "strings" with Arrow type utf8. virtual AdbcStatusCode CreateSampleTable(struct AdbcConnection* connection, const std::string& name, + const std::string& schema, struct AdbcError* error) const; /// \brief Get the statement to create a table with a primary key, or nullopt if not @@ -197,7 +224,7 @@ class DriverQuirks { /// \brief Default catalog to use for tests virtual std::string catalog() const { return ""; } - /// \brief Default Schema to use for tests + /// \brief Default database schema to use for tests virtual std::string db_schema() const { return ""; } }; @@ -243,6 +270,7 @@ class ConnectionTest { void TestMetadataGetInfo(); void TestMetadataGetTableSchema(); + void TestMetadataGetTableSchemaDbSchema(); void TestMetadataGetTableSchemaEscaping(); void TestMetadataGetTableSchemaNotFound(); void TestMetadataGetTableTypes(); @@ -277,6 +305,9 @@ class ConnectionTest { TEST_F(FIXTURE, MetadataCurrentDbSchema) { TestMetadataCurrentDbSchema(); } \ TEST_F(FIXTURE, MetadataGetInfo) { TestMetadataGetInfo(); } \ TEST_F(FIXTURE, MetadataGetTableSchema) { TestMetadataGetTableSchema(); } \ + TEST_F(FIXTURE, MetadataGetTableSchemaDbSchema) { \ + TestMetadataGetTableSchemaDbSchema(); \ + } \ TEST_F(FIXTURE, MetadataGetTableSchemaEscaping) { \ TestMetadataGetTableSchemaEscaping(); \ } \ diff --git a/ci/conda_env_cpp_lint.txt b/ci/conda_env_cpp_lint.txt index 471ef0de75..7cc81c1e1a 100644 --- a/ci/conda_env_cpp_lint.txt +++ b/ci/conda_env_cpp_lint.txt @@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. -clang=14 -clang-tools=14 +clang=14.* +clang-tools=14.* diff --git a/docs/source/python/recipe/postgresql_get_table_schema.py b/docs/source/python/recipe/postgresql_get_table_schema.py index 3f1bae7264..aacbc1c254 100644 --- a/docs/source/python/recipe/postgresql_get_table_schema.py +++ b/docs/source/python/recipe/postgresql_get_table_schema.py @@ -28,13 +28,18 @@ uri = os.environ["ADBC_POSTGRESQL_TEST_URI"] conn = adbc_driver_postgresql.dbapi.connect(uri) -#: We'll create an example table to test. +#: We'll create some example tables to test. with conn.cursor() as cur: cur.execute("DROP TABLE IF EXISTS example") cur.execute("CREATE TABLE example (ints INT, bigints BIGINT)") + cur.execute("CREATE SCHEMA IF NOT EXISTS other_schema") + cur.execute("DROP TABLE IF EXISTS other_schema.example") + cur.execute("CREATE TABLE other_schema.example (strings TEXT, values NUMERIC)") + conn.commit() +#: By default the "active" catalog/schema are assumed. assert conn.adbc_get_table_schema("example") == pyarrow.schema( [ ("ints", "int32"), @@ -42,4 +47,23 @@ ] ) +#: We can explicitly specify the PostgreSQL schema to get the Arrow schema of +#: a table in a different namespace. +#: +#: .. note:: In PostgreSQL, you can only query the database (catalog) that you +#: are connected to. So we cannot specify the catalog here (or +#: rather, there is no point in doing so). +#: +#: Note that the NUMERIC column is read as a string, because PostgreSQL +#: decimals do not map onto Arrow decimals. +assert conn.adbc_get_table_schema( + "example", + db_schema_filter="other_schema", +) == pyarrow.schema( + [ + ("strings", "string"), + ("values", "string"), + ] +) + conn.close() From 97765976b65830159b9a9b52c92b160cb3b7987b Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 20 Dec 2023 12:50:12 -0500 Subject: [PATCH 22/51] docs: enable linking to Javadoc via Intersphinx (#1381) - Generate a Sphinx Intersphinx inventory by scraping generated Javadocs. - Inject the location of this inventory in CI. - Inject a fake URL to use for the location of the Javadocs since we don't know this at documentation build time. - At website build time, replace the fake URL with the real URL. The approach here is extensible to other languages. Fixes #1357. --- .github/workflows/native-unix.yml | 7 ++ ci/scripts/docs_build.sh | 23 +++- ci/scripts/website_build.sh | 7 ++ docs/source/conf.py | 25 ++++ docs/source/ext/adbc_java_domain.py | 47 ++++++++ docs/source/ext/javadoc_inventory.py | 173 +++++++++++++++++++++++++++ docs/source/format/specification.rst | 10 +- 7 files changed, 282 insertions(+), 10 deletions(-) create mode 100644 docs/source/ext/adbc_java_domain.py create mode 100644 docs/source/ext/javadoc_inventory.py diff --git a/.github/workflows/native-unix.yml b/.github/workflows/native-unix.yml index 4d6c50c50e..d8084da790 100644 --- a/.github/workflows/native-unix.yml +++ b/.github/workflows/native-unix.yml @@ -571,6 +571,13 @@ jobs: shell: bash -l {0} run: | ./ci/scripts/docs_build.sh "$(pwd)" + - name: Archive docs + uses: actions/upload-artifact@v3 + with: + name: docs + retention-days: 2 + path: | + docs/build/html - name: Test Recipes (C++) shell: bash -l {0} run: | diff --git a/ci/scripts/docs_build.sh b/ci/scripts/docs_build.sh index 7294e6d68b..271dec459f 100755 --- a/ci/scripts/docs_build.sh +++ b/ci/scripts/docs_build.sh @@ -25,15 +25,26 @@ main() { doxygen popd - pushd "$source_dir/docs" - make html - make doctest - popd - pushd "$source_dir/java" mvn site + popd + + pushd "$source_dir/docs" + # The project name/version don't really matter here. + python "$source_dir/docs/source/ext/javadoc_inventory.py" \ + "ADBC" \ + "version" \ + "$source_dir/java/target/site/apidocs" \ + "java/api" + + # We need to determine the base URL without knowing it... + # Inject a dummy URL here, and fix it up in website_build.sh + export ADBC_INTERSPHINX_MAPPING_java_adbc="http://javadocs.home.arpa/;$source_dir/java/target/site/apidocs/objects.inv" + + make html rm -rf "$source_dir/docs/build/html/java/api" - cp -r target/site/apidocs "$source_dir/docs/build/html/java/api" + cp -r "$source_dir/java/target/site/apidocs" "$source_dir/docs/build/html/java/api" + make doctest popd for desc_file in $(find "${source_dir}/r" -name DESCRIPTION); do diff --git a/ci/scripts/website_build.sh b/ci/scripts/website_build.sh index 480c57c908..31db174a4a 100755 --- a/ci/scripts/website_build.sh +++ b/ci/scripts/website_build.sh @@ -47,16 +47,23 @@ main() { fi local -r regex='^([0-9]+\.[0-9]+\.[0-9]+)$' + local directory="main" if [[ "${new_version}" =~ $regex ]]; then cp -r "${docs}" "${site}/${new_version}" git -C "${site}" add --force "${new_version}" + directory="${new_version}" else # Assume this is dev docs rm -rf "${site}/main" cp -r "${docs}" "${site}/main" git -C "${site}" add --force "main" + directory="main" fi + # Fix up lazy Intersphinx links (see docs_build.sh) + # Assumes GNU sed + sed -i "s|http://javadocs.home.arpa/|https://arrow.apache.org/adbc/${directory}/|g" $(grep -Rl javadocs.home.arpa "${site}/${directory}/") + # Copy the version script and regenerate the version list # The versions get embedded into the JavaScript file to save a roundtrip rm -f "${site}/version.js" diff --git a/docs/source/conf.py b/docs/source/conf.py index 6e4858a179..128dfc736d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import os import sys from pathlib import Path @@ -35,7 +36,10 @@ exclude_patterns = [] extensions = [ + # recipe directive "adbc_cookbook", + # generic directives to enable intersphinx for java + "adbc_java_domain", "breathe", "numpydoc", "sphinx.ext.autodoc", @@ -111,6 +115,27 @@ "arrow": ("https://arrow.apache.org/docs/", None), } +# Add env vars like ADBC_INTERSPHINX_MAPPING_adbc_java = url;path +# to inject more mappings + + +def _find_intersphinx_mappings(): + prefix = "ADBC_INTERSPHINX_MAPPING_" + for key, val in os.environ.items(): + if key.startswith(prefix): + name = key[len(prefix) :] + url, _, path = val.partition(";") + print("[ADBC] Found Intersphinx mapping", name) + intersphinx_mapping[name] = (url, path) + # "adbc_java": ( + # "http://localhost:8000/", + # "/home/lidavidm/Code/arrow-adbc/java/target/site/apidocs/objects.inv", + # ), + + +_find_intersphinx_mappings() + + # -- Options for numpydoc ---------------------------------------------------- numpydoc_class_members_toctree = False diff --git a/docs/source/ext/adbc_java_domain.py b/docs/source/ext/adbc_java_domain.py new file mode 100644 index 0000000000..186141fc84 --- /dev/null +++ b/docs/source/ext/adbc_java_domain.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""A basic Java domain for Sphinx.""" + +import typing + +from sphinx.application import Sphinx + + +def setup(app: Sphinx) -> dict[str, typing.Any]: + # XXX: despite documentation, this is added to 'std' domain not 'rst' + # domain (look at the source) + app.add_object_type( + "javatype", + "jtype", + objname="Java Type", + ) + app.add_object_type( + "javamember", + "jmember", + objname="Java Member", + ) + app.add_object_type( + "javapackage", + "jpackage", + objname="Java Package", + ) + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/docs/source/ext/javadoc_inventory.py b/docs/source/ext/javadoc_inventory.py new file mode 100644 index 0000000000..7c2fbcf255 --- /dev/null +++ b/docs/source/ext/javadoc_inventory.py @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Generate a Sphinx inventory for a Javadoc site.""" + +from __future__ import annotations + +import argparse +import json +import typing +import urllib.parse +from pathlib import Path + +import sphinx.util.inventory + +# XXX: we're taking advantage of duck typing to do stupid things here. + + +class FakeEnv(typing.NamedTuple): + project: str + version: str + + +class FakeObject(typing.NamedTuple): + # Looks like this + # name domainname:typ prio uri dispname + name: str + # written as '-' if equal to name + dispname: str + # member, doc, etc + typ: str + # passed through builder.get_target_uri + docname: str + # not including the # + anchor: str + # written, but never used + prio: str + + +class FakeDomain(typing.NamedTuple): + objects: list[FakeObject] + + def get_objects(self): + return self.objects + + +class FakeBuildEnvironment(typing.NamedTuple): + config: FakeEnv + domains: dict[str, FakeDomain] + + +class FakeBuilder: + def get_target_uri(self, docname: str) -> str: + return docname + + +def extract_index(data: str, prelude: str) -> list: + epilogue = ";updateSearchResults();" + if not data.startswith(prelude): + raise ValueError( + f"Cannot parse search index; expected {prelude!r} but found {data[:50]!r}" + ) + if data.endswith(epilogue): + data = data[len(prelude) : -len(epilogue)] + else: + # Some JDK versions appear to generate without the epilogue + data = data[len(prelude) :] + return json.loads(data) + + +def make_fake_domains(root: Path, base_url: str) -> dict[str, FakeDomain]: + if not base_url.endswith("/"): + base_url += "/" + + # Scrape the search index generated by Javadoc + # https://github.com/openjdk/jdk17/blob/4afbcaf55383ec2f5da53282a1547bac3d099e9d/src/jdk.javadoc/share/classes/jdk/javadoc/internal/doclets/toolkit/util/IndexItem.java#L515 + # "p" is containing package + # "m" is containing module + # "c" is containing class + # "l" is label + # "u" is the URL anchor + + with open(root / "type-search-index.js") as f: + data = extract_index(f.read(), "typeSearchIndex = ") + with open(root / "member-search-index.js") as f: + data.extend(extract_index(f.read(), "memberSearchIndex = ")) + with open(root / "package-search-index.js") as f: + data.extend(extract_index(f.read(), "packageSearchIndex = ")) + + domains = { + "std": FakeDomain(objects=[]), + } + + for item in data: + if "p" not in item: + # Non-code item (package, or index) + if "All " in item["l"]: + # Ignore indices ("All Packages") + continue + # This is a package + name = item["l"] + url = f"{item['l'].replace('.', '/')}/package-summary.html" + anchor = "" + typ = "javapackage" + domain = "std" + elif "c" in item: + # This is a class member + name = f"{item['p']}.{item['c']}#{item['l']}" + url = f"{item['p'].replace('.', '/')}/{item['c']}.html" + anchor = item["u"] if "u" in item else item["l"] + typ = "javamember" + domain = "std" + else: + # This is a class/interface + name = f"{item['p']}.{item['l']}" + url = f"{item['p'].replace('.', '/')}/{item['l']}.html" + anchor = "" + typ = "javatype" + domain = "std" + + url = urllib.parse.urljoin(base_url, url) + domains[domain].objects.append( + FakeObject( + name=name, + dispname=name, + typ=typ, + docname=url, + anchor=anchor, + prio=1, + ) + ) + + return domains + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("project", help="Project name.") + parser.add_argument("version", help="Project version.") + parser.add_argument("path", type=Path, help="Path to the generated Javadocs.") + parser.add_argument("url", help="Eventual base URL of the Javadocs.") + + args = parser.parse_args() + + domains = make_fake_domains(args.path, args.url) + config = FakeEnv(project=args.project, version=args.version) + env = FakeBuildEnvironment(config=config, domains=domains) + + output = args.path / "objects.inv" + sphinx.util.inventory.InventoryFile.dump( + str(output), + env, + FakeBuilder(), + ) + print("Wrote", output) + + +if __name__ == "__main__": + main() diff --git a/docs/source/format/specification.rst b/docs/source/format/specification.rst index 19b73010cb..d1c91ea120 100644 --- a/docs/source/format/specification.rst +++ b/docs/source/format/specification.rst @@ -23,7 +23,9 @@ This document summarizes the general featureset. - For C/C++ details, see :doc:`adbc.h <../../cpp/api/adbc>`. - For Go details, see the `source `__. -- For Java details, see the `source `__. +- For Java details, see the `source + `__, particularly + the package :jpackage:`org.apache.arrow.adbc.core`. Databases ========= @@ -34,7 +36,7 @@ provides a place to hold ownership of the in-memory database. - C/C++: :cpp:class:`AdbcDatabase` - Go: ``Driver`` -- Java: ``org.apache.arrow.adbc.core.AdbcDatabase`` +- Java: :jtype:`org.apache.arrow.adbc.core.AdbcDatabase` Connections =========== @@ -43,7 +45,7 @@ A connection is a single, logical connection to a database. - C/C++: :cpp:class:`AdbcConnection` - Go: ``Connection`` -- Java: ``org.apache.arrow.adbc.core.AdbcConnection`` +- Java: :jtype:`org.apache.arrow.adbc.core.AdbcConnection` Autocommit ---------- @@ -55,7 +57,7 @@ implementations will support this. - C/C++: :c:macro:`ADBC_CONNECTION_OPTION_AUTOCOMMIT` - Go: ``OptionKeyAutoCommit`` -- Java: ``org.apache.arrow.adbc.core.AdbcConnection#setAutoCommit(boolean)`` +- Java: :jmember:`org.apache.arrow.adbc.core.AdbcConnection#setAutoCommit(boolean)` Metadata -------- From 5c1808617d9e8c18887a22cd1766ca4d360f0d02 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 20 Dec 2023 13:10:05 -0500 Subject: [PATCH 23/51] chore: use LF not CRLF for csproj (#1386) --- .pre-commit-config.yaml | 4 +- .../Apache.Arrow.Adbc.csproj | 42 ++++----- .../Client/Apache.Arrow.Adbc.Client.csproj | 32 +++---- .../Apache.Arrow.Adbc.Drivers.BigQuery.csproj | 44 +++++----- ...Apache.Arrow.Adbc.Drivers.FlightSql.csproj | 24 ++--- ...rrow.Adbc.Drivers.Interop.Snowflake.csproj | 78 ++++++++-------- .../Apache.Arrow.Adbc.Tests.csproj | 46 +++++----- ...e.Arrow.Adbc.Tests.Drivers.BigQuery.csproj | 58 ++++++------ ....Arrow.Adbc.Tests.Drivers.FlightSql.csproj | 56 ++++++------ ...dbc.Tests.Drivers.Interop.Snowflake.csproj | 54 ++++++------ .../Apache.Arrow.Adbc.SmokeTests.csproj | 88 +++++++++---------- ...ow.Adbc.SmokeTests.Drivers.BigQuery.csproj | 66 +++++++------- ...w.Adbc.SmokeTests.Drivers.FlightSql.csproj | 56 ++++++------ ...mokeTests.Drivers.Interop.Snowflake.csproj | 66 +++++++------- 14 files changed, 357 insertions(+), 357 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 01e624caa0..194924ae0f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,11 +32,11 @@ repos: - id: mixed-line-ending name: "Mixed line endings (LF)" args: [--fix=lf] - exclude: '\.(bat|sln|csproj)$' + exclude: '\.(bat|sln)$' - id: mixed-line-ending name: "Mixed line endings (CRLF)" args: [--fix=crlf] - files: '\.(bat|sln|csproj)$' + files: '\.(bat|sln)$' - id: trailing-whitespace exclude: "^r/.*?/_snaps/.*?.md$" - repo: https://github.com/pocc/pre-commit-hooks diff --git a/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj b/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj index b64b32d778..358eca01ae 100644 --- a/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj +++ b/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj @@ -1,21 +1,21 @@ - - - - netstandard2.0;net6.0 - true - readme.md - - - - - - - - - - true - \ - PreserveNewest - - - + + + + netstandard2.0;net6.0 + true + readme.md + + + + + + + + + + true + \ + PreserveNewest + + + diff --git a/csharp/src/Client/Apache.Arrow.Adbc.Client.csproj b/csharp/src/Client/Apache.Arrow.Adbc.Client.csproj index f44be6f229..ced800c665 100644 --- a/csharp/src/Client/Apache.Arrow.Adbc.Client.csproj +++ b/csharp/src/Client/Apache.Arrow.Adbc.Client.csproj @@ -1,16 +1,16 @@ - - - netstandard2.0;net6.0 - readme.md - - - - - - - true - \ - PreserveNewest - - - + + + netstandard2.0;net6.0 + readme.md + + + + + + + true + \ + PreserveNewest + + + diff --git a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj index a7451c7bd1..c078245835 100644 --- a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj +++ b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj @@ -1,22 +1,22 @@ - - - netstandard2.0;net6.0 - readme.md - - - - - - - - - - - - - true - \ - PreserveNewest - - - + + + netstandard2.0;net6.0 + readme.md + + + + + + + + + + + + + true + \ + PreserveNewest + + + diff --git a/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj b/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj index b6a7fb4c68..61a38ffb2b 100644 --- a/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj +++ b/csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj @@ -1,12 +1,12 @@ - - - netstandard2.0;net6.0 - - - - - - - - - + + + netstandard2.0;net6.0 + + + + + + + + + diff --git a/csharp/src/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Drivers.Interop.Snowflake.csproj b/csharp/src/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Drivers.Interop.Snowflake.csproj index ece287a7c9..815418f5a3 100644 --- a/csharp/src/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Drivers.Interop.Snowflake.csproj +++ b/csharp/src/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Drivers.Interop.Snowflake.csproj @@ -1,39 +1,39 @@ - - - netstandard2.0;net472;net6.0 - readme.md - - - - - - - - - - true - lib\netstandard2.0 - PreserveNewest - - - true - lib\net472 - PreserveNewest - - - true - lib\net6.0 - PreserveNewest - - - - - true - \ - PreserveNewest - - - - - - + + + netstandard2.0;net472;net6.0 + readme.md + + + + + + + + + + true + lib\netstandard2.0 + PreserveNewest + + + true + lib\net472 + PreserveNewest + + + true + lib\net6.0 + PreserveNewest + + + + + true + \ + PreserveNewest + + + + + + diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj index eb53f350ac..15354c2d1f 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj @@ -1,23 +1,23 @@ - - - - net472;net6.0 - disable - false - true - True - - - - - - - - - - - - - - - + + + + net472;net6.0 + disable + false + true + True + + + + + + + + + + + + + + + diff --git a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj index d4c2fb541b..f8992c780d 100644 --- a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj +++ b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj @@ -1,29 +1,29 @@ - - - net472;net6.0 - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - PreserveNewest - - - Never - - - + + + net472;net6.0 + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + PreserveNewest + + + Never + + + diff --git a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj index 7647af2d76..e8dfd14445 100644 --- a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj +++ b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj @@ -1,28 +1,28 @@ - - - net472;net6.0 - disable - False - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - Never - - - + + + net472;net6.0 + disable + False + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + Never + + + diff --git a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj index a7dba73d0e..11a969e2c7 100644 --- a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj +++ b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj @@ -1,27 +1,27 @@ - - - net472;net6.0 - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - PreserveNewest - - - Never - - - + + + net472;net6.0 + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + PreserveNewest + + + Never + + + diff --git a/csharp/test/SmokeTests/Apache.Arrow.Adbc.SmokeTests/Apache.Arrow.Adbc.SmokeTests.csproj b/csharp/test/SmokeTests/Apache.Arrow.Adbc.SmokeTests/Apache.Arrow.Adbc.SmokeTests.csproj index 08ad0ea09d..a7a7a21303 100644 --- a/csharp/test/SmokeTests/Apache.Arrow.Adbc.SmokeTests/Apache.Arrow.Adbc.SmokeTests.csproj +++ b/csharp/test/SmokeTests/Apache.Arrow.Adbc.SmokeTests/Apache.Arrow.Adbc.SmokeTests.csproj @@ -1,44 +1,44 @@ - - - - - net472;net6.0 - disable - false - true - True - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + net472;net6.0 + disable + false + true + True + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/csharp/test/SmokeTests/BigQuery/Apache.Arrow.Adbc.SmokeTests.Drivers.BigQuery.csproj b/csharp/test/SmokeTests/BigQuery/Apache.Arrow.Adbc.SmokeTests.Drivers.BigQuery.csproj index d032501e43..6a323fd590 100644 --- a/csharp/test/SmokeTests/BigQuery/Apache.Arrow.Adbc.SmokeTests.Drivers.BigQuery.csproj +++ b/csharp/test/SmokeTests/BigQuery/Apache.Arrow.Adbc.SmokeTests.Drivers.BigQuery.csproj @@ -1,33 +1,33 @@ - - - - net472;net6.0 - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - PreserveNewest - - - - - - + + + + net472;net6.0 + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + PreserveNewest + + + + + + diff --git a/csharp/test/SmokeTests/FlightSql/Apache.Arrow.Adbc.SmokeTests.Drivers.FlightSql.csproj b/csharp/test/SmokeTests/FlightSql/Apache.Arrow.Adbc.SmokeTests.Drivers.FlightSql.csproj index 7647af2d76..e8dfd14445 100644 --- a/csharp/test/SmokeTests/FlightSql/Apache.Arrow.Adbc.SmokeTests.Drivers.FlightSql.csproj +++ b/csharp/test/SmokeTests/FlightSql/Apache.Arrow.Adbc.SmokeTests.Drivers.FlightSql.csproj @@ -1,28 +1,28 @@ - - - net472;net6.0 - disable - False - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - Never - - - + + + net472;net6.0 + disable + False + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + Never + + + diff --git a/csharp/test/SmokeTests/Interop/Snowflake/Apache.Arrow.Adbc.SmokeTests.Drivers.Interop.Snowflake.csproj b/csharp/test/SmokeTests/Interop/Snowflake/Apache.Arrow.Adbc.SmokeTests.Drivers.Interop.Snowflake.csproj index fa722ea644..ced24e6426 100644 --- a/csharp/test/SmokeTests/Interop/Snowflake/Apache.Arrow.Adbc.SmokeTests.Drivers.Interop.Snowflake.csproj +++ b/csharp/test/SmokeTests/Interop/Snowflake/Apache.Arrow.Adbc.SmokeTests.Drivers.Interop.Snowflake.csproj @@ -1,33 +1,33 @@ - - - - net472;net6.0 - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - PreserveNewest - - - - - - + + + + net472;net6.0 + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + PreserveNewest + + + + + + From 1d2b53b11d4cdc93daf1c8fcfd7c252b676a5d93 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 20 Dec 2023 13:13:50 -0500 Subject: [PATCH 24/51] chore: stage website changes after rewrite (#1388) --- ci/scripts/website_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/website_build.sh b/ci/scripts/website_build.sh index 31db174a4a..1f7f146e1c 100755 --- a/ci/scripts/website_build.sh +++ b/ci/scripts/website_build.sh @@ -63,6 +63,7 @@ main() { # Fix up lazy Intersphinx links (see docs_build.sh) # Assumes GNU sed sed -i "s|http://javadocs.home.arpa/|https://arrow.apache.org/adbc/${directory}/|g" $(grep -Rl javadocs.home.arpa "${site}/${directory}/") + git -C "${site}" add --force "${directory}" # Copy the version script and regenerate the version list # The versions get embedded into the JavaScript file to save a roundtrip From eec32c482862683424bad1fda2615eea216cbb9f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:47:14 -0500 Subject: [PATCH 25/51] chore(csharp): bump Google.Cloud.BigQuery.Storage.V1 from 3.6.0 to 3.10.0 in /csharp (#1374) Bumps [Google.Cloud.BigQuery.Storage.V1](https://github.com/googleapis/google-cloud-dotnet) from 3.6.0 to 3.10.0.
Commits
  • af07588 Release Google.Cloud.BigQuery.Storage.V1 version 3.10.0 (#10872)
  • f8e8169 Release Google.Cloud.Bigtable.Admin.V2 version 3.6.0 (#10873)
  • 4541395 Release Google.Cloud.BinaryAuthorization.V1Beta1 version 2.0.0-beta04 (#10874)
  • e86ed41 Release Google.Cloud.BigQuery.DataPolicies.V1 version 1.1.0 (#10871)
  • b5d8650 Release Google.Cloud.Container.V1 version 3.17.0 (#10876)
  • fe66a31 Release Google.Cloud.Dialogflow.Cx.V3 version 2.10.0 (#10877)
  • 0a21a90 Release Google.Cloud.ConfidentialComputing.V1 version 1.0.0-beta04 (#10875)
  • 9c80b90 Release Google.Cloud.Dialogflow.V2 version 4.12.0 (#10878)
  • 6e97b0a Release Google.Cloud.OsLogin.V1Beta version 3.0.0-beta03 (#10882)
  • f4f6c51 Release Google.Cloud.NetworkManagement.V1 version 2.5.0 (#10880)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Google.Cloud.BigQuery.Storage.V1&package-manager=nuget&previous-version=3.6.0&new-version=3.10.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj index c078245835..0411f1e369 100644 --- a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj +++ b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj @@ -5,7 +5,7 @@ - + From 71a746355d01c26ffdf508ac7ee1c0ca15e7d725 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:47:41 -0500 Subject: [PATCH 26/51] chore(csharp): bump xunit from 2.5.3 to 2.6.3 in /csharp (#1378) Bumps [xunit](https://github.com/xunit/xunit) from 2.5.3 to 2.6.3.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=xunit&package-manager=nuget&previous-version=2.5.3&new-version=2.6.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj | 2 +- .../BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj | 2 +- .../FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj | 2 +- .../Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj index 15354c2d1f..174136409c 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj index f8992c780d..fac1f346cd 100644 --- a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj +++ b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj @@ -4,7 +4,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj index e8dfd14445..4ffcffaa6f 100644 --- a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj +++ b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj @@ -6,7 +6,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj index 11a969e2c7..5c4a928cdd 100644 --- a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj +++ b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj @@ -4,7 +4,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive From 3cf72aef391bff9106b458c16a1b03c4366cc532 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:48:28 -0500 Subject: [PATCH 27/51] chore(csharp): bump Moq from 4.20.69 to 4.20.70 in /csharp (#1375) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [//]: # (dependabot-start) ⚠️ **Dependabot is rebasing this PR** ⚠️ Rebasing might not happen immediately, so don't worry if this takes some time. Note: if you make any changes to this PR yourself, they will take precedence over the rebase. --- [//]: # (dependabot-end) Bumps [Moq](https://github.com/moq/moq) from 4.20.69 to 4.20.70.
Release notes

Sourced from Moq's releases.

v4.20.70

What's Changed

🔨 Other

New Contributors

Full Changelog: https://github.com/devlooped/moq/compare/v4.20.69...v4.20.70

Changelog

Sourced from Moq's changelog.

Moq Changelog

All notable changes to this project will be documented in this file.

The format is loosely based on Keep a Changelog.

Commits
  • 2b55659 ⬆️ Bump files with dotnet-file sync
  • cb605f4 Bump NuGetizer from 1.1.1 to 1.2.0
  • 6816a49 ⬆️ Bump files with dotnet-file sync
  • 8e46ab8 A minor negation in GetDelay to make it more readable #1419
  • d702a5e Bump Microsoft.NET.Test.Sdk from 17.7.1 to 17.7.2
  • f4022d9 Bump NuGetizer from 1.1.0 to 1.1.1
  • 1d33044 ⬆️ Bump files with dotnet-file sync
  • 06d88ec +Mᐁ includes
  • 6146125 Bump Microsoft.NET.Test.Sdk from 17.7.0 to 17.7.1
  • 633214f Manually update CHANGELOG.md for now
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Moq&package-manager=nuget&previous-version=4.20.69&new-version=4.20.70)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj index 174136409c..3573fd680b 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj @@ -10,7 +10,7 @@ - + From e3ca270ea35559a31c1067f4796961acd9d1b80d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 14:08:16 -0500 Subject: [PATCH 28/51] chore(csharp): bump xunit.runner.visualstudio from 2.5.3 to 2.5.5 in /csharp (#1376) Bumps [xunit.runner.visualstudio](https://github.com/xunit/visualstudio.xunit) from 2.5.3 to 2.5.5.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=xunit.runner.visualstudio&package-manager=nuget&previous-version=2.5.3&new-version=2.5.5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj | 2 +- .../FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj | 2 +- .../Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj index fac1f346cd..7c758785e6 100644 --- a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj +++ b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj @@ -5,7 +5,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj index 4ffcffaa6f..744cb4e66b 100644 --- a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj +++ b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj @@ -7,7 +7,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj index 5c4a928cdd..3297525bad 100644 --- a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj +++ b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj @@ -5,7 +5,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive From 21a84bdbc705aea5ac5bc84f88cf4170d12a234e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 14:08:23 -0500 Subject: [PATCH 29/51] chore(csharp): bump System.Text.Json from 7.0.2 to 7.0.4 in /csharp (#1377) Bumps [System.Text.Json](https://github.com/dotnet/runtime) from 7.0.2 to 7.0.4.
Release notes

Sourced from System.Text.Json's releases.

.NET 7.0.4

Release

.NET 7.0.3

Release

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=System.Text.Json&package-manager=nuget&previous-version=7.0.2&new-version=7.0.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj index 3573fd680b..efbd47d5c9 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj @@ -11,7 +11,7 @@ - + From e7fa7c6cdf8abaf0d8fbcd5d678d03f1080add43 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 20 Dec 2023 15:16:59 -0500 Subject: [PATCH 30/51] fix(c/driver/postgresql): check for underflow (#1389) Fixes #1190. --- c/driver/postgresql/postgres_copy_reader.h | 19 ++++++++++++++++--- c/driver/postgresql/statement.cc | 15 +++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/c/driver/postgresql/postgres_copy_reader.h b/c/driver/postgresql/postgres_copy_reader.h index 66a3141898..7ba29ea3a3 100644 --- a/c/driver/postgresql/postgres_copy_reader.h +++ b/c/driver/postgresql/postgres_copy_reader.h @@ -17,10 +17,14 @@ #pragma once +// Windows +#define NOMINMAX + #include #include #include #include +#include #include #include #include @@ -68,6 +72,9 @@ constexpr int64_t kMaxSafeMicrosToNanos = 9223372036854775L; // without overflow constexpr int64_t kMinSafeMicrosToNanos = -9223372036854775L; +// 2000-01-01 00:00:00.000000 in microseconds +constexpr int64_t kPostgresTimestampEpoch = 946684800000000L; + // Read a value from the buffer without checking the buffer size. Advances // the cursor of data and reduces its size by sizeof(T). template @@ -1333,14 +1340,20 @@ class PostgresCopyTimestampFieldWriter : public PostgresCopyFieldWriter { if (!overflow_safe) { ArrowErrorSet(error, - "Row %" PRId64 " timestamp value %" PRId64 + "[libpq] Row %" PRId64 " timestamp value %" PRId64 " with unit %d would overflow", index, raw_value, TU); return ADBC_STATUS_INVALID_ARGUMENT; } - // 2000-01-01 00:00:00.000000 in microseconds - constexpr int64_t kPostgresTimestampEpoch = 946684800000000; + if (value < std::numeric_limits::min() + kPostgresTimestampEpoch) { + ArrowErrorSet(error, + "[libpq] Row %" PRId64 " timestamp value %" PRId64 + " with unit %d would underflow", + index, raw_value, TU); + return ADBC_STATUS_INVALID_ARGUMENT; + } + const int64_t scaled = value - kPostgresTimestampEpoch; NANOARROW_RETURN_NOT_OK(WriteChecked(buffer, scaled, error)); diff --git a/c/driver/postgresql/statement.cc b/c/driver/postgresql/statement.cc index 656e187627..6c0541a6c5 100644 --- a/c/driver/postgresql/statement.cc +++ b/c/driver/postgresql/statement.cc @@ -15,6 +15,9 @@ // specific language governing permissions and limitations // under the License. +// Windows +#define NOMINMAX + #include "statement.h" #include @@ -23,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -432,8 +436,6 @@ struct BindStream { case ArrowType::NANOARROW_TYPE_TIMESTAMP: { int64_t val = array_view->children[col]->buffer_views[1].data.as_int64[row]; - // 2000-01-01 00:00:00.000000 in microseconds - constexpr int64_t kPostgresTimestampEpoch = 946684800000000; bool overflow_safe = true; auto unit = bind_schema_fields[col].time_unit; @@ -469,6 +471,15 @@ struct BindStream { return ADBC_STATUS_INVALID_ARGUMENT; } + if (val < std::numeric_limits::min() + kPostgresTimestampEpoch) { + SetError(error, + "[libpq] Field #%" PRId64 " ('%s') Row #%" PRId64 + " has value '%" PRIi64 "' which would underflow", + col + 1, bind_schema->children[col]->name, row + 1, + array_view->children[col]->buffer_views[1].data.as_int64[row]); + return ADBC_STATUS_INVALID_ARGUMENT; + } + if (bind_schema_fields[col].type == ArrowType::NANOARROW_TYPE_TIMESTAMP) { const uint64_t value = ToNetworkInt64(val - kPostgresTimestampEpoch); std::memcpy(param_values[col], &value, sizeof(int64_t)); From 0260dc88039cb9ae3e481544dd00374760f90f46 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 21 Dec 2023 09:43:47 -0500 Subject: [PATCH 31/51] docs: improve PostgreSQL type support table (#1390) Fixes #1385. --- docs/source/driver/postgresql.rst | 51 +++++++++++++++++-------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/docs/source/driver/postgresql.rst b/docs/source/driver/postgresql.rst index 36349644ba..282a86a1a6 100644 --- a/docs/source/driver/postgresql.rst +++ b/docs/source/driver/postgresql.rst @@ -196,78 +196,74 @@ being read or written. * - Arrow Type - As Bind Parameter - - In Bulk Ingestion + - In Bulk Ingestion [#bulk-ingestion]_ * - binary - - Y + - BYTEA - BYTEA * - bool - - Y + - BOOLEAN - BOOLEAN * - date32 - - Y + - DATE - DATE * - date64 - - N - - N + - ❌ + - ❌ * - dictionary - (as unpacked type) - (as unpacked type, only for binary/string) * - duration - - Y + - INTERVAL - INTERVAL * - float32 - - Y + - REAL - REAL * - float64 - - Y + - DOUBLE PRECISION - DOUBLE PRECISION * - int8 - - Y + - SMALLINT - SMALLINT * - int16 - - Y + - SMALLINT - SMALLINT * - int32 - - Y + - INTEGER - INTEGER * - int64 - - Y + - BIGINT - BIGINT * - large_binary - - N - - N + - ❌ + - ❌ * - large_string - - Y + - TEXT - TEXT * - month_day_nano_interval - - Y - INTERVAL - - * - NA - - N - - N + - INTERVAL * - string - - Y + - TEXT - TEXT * - timestamp - - Y + - TIMESTAMP [#timestamp]_ - TIMESTAMP/TIMESTAMP WITH TIMEZONE .. list-table:: PostgreSQL type to Arrow type mapping @@ -311,6 +307,15 @@ being read or written. * - VARCHAR - utf8 +.. [#bulk-ingestion] This is the data type used when creating/appending to a + table from Arrow data via the bulk ingestion feature. + .. [#numeric-utf8] NUMERIC types are read as the string representation of the value, because the PostgreSQL NUMERIC type cannot be losslessly converted to the Arrow decimal types. + +.. [#timestamp] When binding a timestamp value, the time zone (if present) is + ignored. The value will be converted to microseconds and + adjusted to the PostgreSQL epoch (2000-01-01) and so may + overflow/underflow; an error will be returned if this would be + the case. From 7531ac1e97252cc84c10f3622339c2763852a113 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 21 Dec 2023 10:19:04 -0500 Subject: [PATCH 32/51] docs: improve landing page (#1392) Fixes #1385. --- docs/source/csharp/index.rst | 26 ++++++++++++++++++++++++++ docs/source/faq.rst | 2 +- docs/source/index.rst | 9 ++++++++- 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 docs/source/csharp/index.rst diff --git a/docs/source/csharp/index.rst b/docs/source/csharp/index.rst new file mode 100644 index 0000000000..05681563c4 --- /dev/null +++ b/docs/source/csharp/index.rst @@ -0,0 +1,26 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +======= +C#/.NET +======= + +The ADBC C# libraries are under development, supporting: + +- A native driver to Google BigQuery +- A way to import/export native (C/C++) drivers +- Bindings to the Snowflake driver diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 803b54f56b..43bc889b96 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -30,7 +30,7 @@ ADBC is: For example, result sets of queries in ADBC are all returned as streams of Arrow data, not row-by-row. - A set of implementations of that API in different languages (C/C++, - Go, Java, Python, and Ruby) that target different databases + C#/.NET, Go, Java, Python, and Ruby) that target different databases (e.g. PostgreSQL, SQLite, any database supporting Flight SQL). Why not just use JDBC/ODBC? diff --git a/docs/source/index.rst b/docs/source/index.rst index 96608fe33b..5537a94365 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -40,10 +40,12 @@ Apache Arrow ADBC **Cross-language**, **Arrow-native** database access. ADBC is a set of APIs and libraries for Arrow-native access to databases. -Execute SQL and Substrait queries, query database catalogs, and more, all +Execute SQL and Substrait_ queries, query database catalogs, and more, all using Arrow data to eliminate unnecessary data copies, speed up access, and make it more convenient to build analytical applications. +.. _Substrait: https://substrait.io/ + .. div:: .. grid:: @@ -160,10 +162,14 @@ Why ADBC? :gutter: 1 .. grid-item-card:: Arrow-native + :link: https://arrow.apache.org/ Execute queries and get back results in Arrow format, eliminating extra data copies for Arrow-native backends. + +++ + Learn about Apache Arrow + .. grid-item-card:: Backend-agnostic :link: driver/status :link-type: doc @@ -202,6 +208,7 @@ Why ADBC? :hidden: C/C++ + C#/.NET Go Java Python From 0f06843e498d5f7afed45c5d50ab523b05933f67 Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 22 Dec 2023 08:53:27 -0500 Subject: [PATCH 33/51] fix(c/driver/postgresql): fix ingest with multiple batches (#1393) The COPY writer was ending the COPY command after each batch, so any dataset with more than one batch would fail. Instead, write the header once and don't end the command until we've written all batches. Fixes #1310. --- c/driver/postgresql/postgres_copy_reader.h | 13 +- .../postgresql/postgres_copy_reader_test.cc | 55 +++++- c/driver/postgresql/postgres_util.h | 4 +- c/driver/postgresql/statement.cc | 51 +++-- docs/source/python/recipe/postgresql.rst | 5 + .../recipe/postgresql_create_dataset_table.py | 184 ++++++++++++++++++ .../adbc_driver_manager/dbapi.py | 16 ++ .../tests/test_dbapi.py | 56 ++++++ 8 files changed, 350 insertions(+), 34 deletions(-) create mode 100644 docs/source/python/recipe/postgresql_create_dataset_table.py diff --git a/c/driver/postgresql/postgres_copy_reader.h b/c/driver/postgresql/postgres_copy_reader.h index 7ba29ea3a3..686d54b81b 100644 --- a/c/driver/postgresql/postgres_copy_reader.h +++ b/c/driver/postgresql/postgres_copy_reader.h @@ -1460,16 +1460,20 @@ static inline ArrowErrorCode MakeCopyFieldWriter(struct ArrowSchema* schema, class PostgresCopyStreamWriter { public: - ArrowErrorCode Init(struct ArrowSchema* schema, struct ArrowArray* array) { + ArrowErrorCode Init(struct ArrowSchema* schema) { schema_ = schema; NANOARROW_RETURN_NOT_OK( ArrowArrayViewInitFromSchema(&array_view_.value, schema, nullptr)); - NANOARROW_RETURN_NOT_OK(ArrowArrayViewSetArray(&array_view_.value, array, nullptr)); root_writer_.Init(&array_view_.value); ArrowBufferInit(&buffer_.value); return NANOARROW_OK; } + ArrowErrorCode SetArray(struct ArrowArray* array) { + NANOARROW_RETURN_NOT_OK(ArrowArrayViewSetArray(&array_view_.value, array, nullptr)); + return NANOARROW_OK; + } + ArrowErrorCode WriteHeader(ArrowError* error) { NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(&buffer_.value, kPgCopyBinarySignature, sizeof(kPgCopyBinarySignature))); @@ -1508,6 +1512,11 @@ class PostgresCopyStreamWriter { const struct ArrowBuffer& WriteBuffer() const { return buffer_.value; } + void Rewind() { + records_written_ = 0; + buffer_->size_bytes = 0; + } + private: PostgresCopyFieldTupleWriter root_writer_; struct ArrowSchema* schema_; diff --git a/c/driver/postgresql/postgres_copy_reader_test.cc b/c/driver/postgresql/postgres_copy_reader_test.cc index 00ba120480..7882d602af 100644 --- a/c/driver/postgresql/postgres_copy_reader_test.cc +++ b/c/driver/postgresql/postgres_copy_reader_test.cc @@ -60,13 +60,14 @@ class PostgresCopyStreamTester { class PostgresCopyStreamWriteTester { public: ArrowErrorCode Init(struct ArrowSchema* schema, struct ArrowArray* array, - ArrowError* error = nullptr) { - NANOARROW_RETURN_NOT_OK(writer_.Init(schema, array)); + struct ArrowError* error = nullptr) { + NANOARROW_RETURN_NOT_OK(writer_.Init(schema)); NANOARROW_RETURN_NOT_OK(writer_.InitFieldWriters(error)); + NANOARROW_RETURN_NOT_OK(writer_.SetArray(array)); return NANOARROW_OK; } - ArrowErrorCode WriteAll(ArrowError* error = nullptr) { + ArrowErrorCode WriteAll(struct ArrowError* error) { NANOARROW_RETURN_NOT_OK(writer_.WriteHeader(error)); int result; @@ -77,8 +78,20 @@ class PostgresCopyStreamWriteTester { return result; } + ArrowErrorCode WriteArray(struct ArrowArray* array, struct ArrowError* error) { + writer_.SetArray(array); + int result; + do { + result = writer_.WriteRecord(error); + } while (result == NANOARROW_OK); + + return result; + } + const struct ArrowBuffer& WriteBuffer() const { return writer_.WriteBuffer(); } + void Rewind() { writer_.Rewind(); } + private: PostgresCopyStreamWriter writer_; }; @@ -1261,4 +1274,40 @@ TEST(PostgresCopyUtilsTest, PostgresCopyReadCustomRecord) { ASSERT_DOUBLE_EQ(data_buffer2[2], 0); } +TEST(PostgresCopyUtilsTest, PostgresCopyWriteMultiBatch) { + // Regression test for https://github.com/apache/arrow-adbc/issues/1310 + adbc_validation::Handle schema; + adbc_validation::Handle array; + struct ArrowError na_error; + ASSERT_EQ(adbc_validation::MakeSchema(&schema.value, {{"col", NANOARROW_TYPE_INT32}}), + NANOARROW_OK); + ASSERT_EQ(adbc_validation::MakeBatch(&schema.value, &array.value, &na_error, + {-123, -1, 1, 123, std::nullopt}), + NANOARROW_OK); + + PostgresCopyStreamWriteTester tester; + ASSERT_EQ(tester.Init(&schema.value, &array.value), NANOARROW_OK); + ASSERT_EQ(tester.WriteAll(nullptr), ENODATA); + + struct ArrowBuffer buf = tester.WriteBuffer(); + // The last 2 bytes of a message can be transmitted via PQputCopyData + // so no need to test those bytes from the Writer + size_t buf_size = sizeof(kTestPgCopyInteger) - 2; + ASSERT_EQ(buf.size_bytes, buf_size); + for (size_t i = 0; i < buf_size; i++) { + ASSERT_EQ(buf.data[i], kTestPgCopyInteger[i]); + } + + tester.Rewind(); + ASSERT_EQ(tester.WriteArray(&array.value, nullptr), ENODATA); + + buf = tester.WriteBuffer(); + // Ignore the header and footer + buf_size = sizeof(kTestPgCopyInteger) - 21; + ASSERT_EQ(buf.size_bytes, buf_size); + for (size_t i = 0; i < buf_size; i++) { + ASSERT_EQ(buf.data[i], kTestPgCopyInteger[i + 19]); + } +} + } // namespace adbcpq diff --git a/c/driver/postgresql/postgres_util.h b/c/driver/postgresql/postgres_util.h index 1009d70b55..95e2619f10 100644 --- a/c/driver/postgresql/postgres_util.h +++ b/c/driver/postgresql/postgres_util.h @@ -166,9 +166,11 @@ struct Handle { Handle() { std::memset(&value, 0, sizeof(value)); } - ~Handle() { Releaser::Release(&value); } + ~Handle() { reset(); } Resource* operator->() { return &value; } + + void reset() { Releaser::Release(&value); } }; } // namespace adbcpq diff --git a/c/driver/postgresql/statement.cc b/c/driver/postgresql/statement.cc index 6c0541a6c5..811a086a4f 100644 --- a/c/driver/postgresql/statement.cc +++ b/c/driver/postgresql/statement.cc @@ -565,7 +565,12 @@ struct BindStream { AdbcStatusCode ExecuteCopy(PGconn* conn, int64_t* rows_affected, struct AdbcError* error) { if (rows_affected) *rows_affected = 0; - PGresult* result = nullptr; + + PostgresCopyStreamWriter writer; + CHECK_NA(INTERNAL, writer.Init(&bind_schema.value), error); + CHECK_NA(INTERNAL, writer.InitFieldWriters(nullptr), error); + + CHECK_NA(INTERNAL, writer.WriteHeader(nullptr), error); while (true) { Handle array; @@ -579,20 +584,9 @@ struct BindStream { } if (!array->release) break; - Handle array_view; - CHECK_NA( - INTERNAL, - ArrowArrayViewInitFromSchema(&array_view.value, &bind_schema.value, nullptr), - error); - CHECK_NA(INTERNAL, ArrowArrayViewSetArray(&array_view.value, &array.value, nullptr), - error); - - PostgresCopyStreamWriter writer; - CHECK_NA(INTERNAL, writer.Init(&bind_schema.value, &array.value), error); - CHECK_NA(INTERNAL, writer.InitFieldWriters(nullptr), error); + CHECK_NA(INTERNAL, writer.SetArray(&array.value), error); // build writer buffer - CHECK_NA(INTERNAL, writer.WriteHeader(nullptr), error); int write_result; do { write_result = writer.WriteRecord(nullptr); @@ -611,25 +605,26 @@ struct BindStream { return ADBC_STATUS_IO; } - if (PQputCopyEnd(conn, NULL) <= 0) { - SetError(error, "Error message returned by PQputCopyEnd: %s", - PQerrorMessage(conn)); - return ADBC_STATUS_IO; - } + if (rows_affected) *rows_affected += array->length; + writer.Rewind(); + } - result = PQgetResult(conn); - ExecStatusType pg_status = PQresultStatus(result); - if (pg_status != PGRES_COMMAND_OK) { - AdbcStatusCode code = - SetError(error, result, "[libpq] Failed to execute COPY statement: %s %s", - PQresStatus(pg_status), PQerrorMessage(conn)); - PQclear(result); - return code; - } + if (PQputCopyEnd(conn, NULL) <= 0) { + SetError(error, "Error message returned by PQputCopyEnd: %s", PQerrorMessage(conn)); + return ADBC_STATUS_IO; + } + PGresult* result = PQgetResult(conn); + ExecStatusType pg_status = PQresultStatus(result); + if (pg_status != PGRES_COMMAND_OK) { + AdbcStatusCode code = + SetError(error, result, "[libpq] Failed to execute COPY statement: %s %s", + PQresStatus(pg_status), PQerrorMessage(conn)); PQclear(result); - if (rows_affected) *rows_affected += array->length; + return code; } + + PQclear(result); return ADBC_STATUS_OK; } }; diff --git a/docs/source/python/recipe/postgresql.rst b/docs/source/python/recipe/postgresql.rst index 7e93a47912..7d578b3633 100644 --- a/docs/source/python/recipe/postgresql.rst +++ b/docs/source/python/recipe/postgresql.rst @@ -26,6 +26,11 @@ Authenticate with a username and password .. _recipe-postgresql-create-append: +Create/append to a table from an Arrow dataset +============================================== + +.. recipe:: postgresql_create_dataset_table.py + Create/append to a table from an Arrow table ============================================ diff --git a/docs/source/python/recipe/postgresql_create_dataset_table.py b/docs/source/python/recipe/postgresql_create_dataset_table.py new file mode 100644 index 0000000000..e26093a308 --- /dev/null +++ b/docs/source/python/recipe/postgresql_create_dataset_table.py @@ -0,0 +1,184 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# RECIPE STARTS HERE + +#: ADBC makes it easy to load PyArrow datasets into your datastore. + +import os +import tempfile +from pathlib import Path + +import pyarrow +import pyarrow.csv +import pyarrow.dataset +import pyarrow.feather +import pyarrow.parquet + +import adbc_driver_postgresql.dbapi + +uri = os.environ["ADBC_POSTGRESQL_TEST_URI"] +conn = adbc_driver_postgresql.dbapi.connect(uri) + +#: For the purposes of testing, we'll first make sure the tables we're about +#: to use don't exist. +with conn.cursor() as cur: + cur.execute("DROP TABLE IF EXISTS csvtable") + cur.execute("DROP TABLE IF EXISTS ipctable") + cur.execute("DROP TABLE IF EXISTS pqtable") + cur.execute("DROP TABLE IF EXISTS csvdataset") + cur.execute("DROP TABLE IF EXISTS ipcdataset") + cur.execute("DROP TABLE IF EXISTS pqdataset") + +conn.commit() + +#: Generating sample data +#: ~~~~~~~~~~~~~~~~~~~~~~ + +tempdir = tempfile.TemporaryDirectory( + prefix="adbc-docs-", + ignore_cleanup_errors=True, +) +root = Path(tempdir.name) +table = pyarrow.table( + [ + [1, 1, 2], + ["foo", "bar", "baz"], + ], + names=["ints", "strs"], +) + +#: First we'll write single files. + +csv_file = root / "example.csv" +pyarrow.csv.write_csv(table, csv_file) + +ipc_file = root / "example.arrow" +pyarrow.feather.write_feather(table, ipc_file) + +parquet_file = root / "example.parquet" +pyarrow.parquet.write_table(table, parquet_file) + +#: We'll also generate some partitioned datasets. + +csv_dataset = root / "csv_dataset" +pyarrow.dataset.write_dataset( + table, + csv_dataset, + format="csv", + partitioning=["ints"], +) + +ipc_dataset = root / "ipc_dataset" +pyarrow.dataset.write_dataset( + table, + ipc_dataset, + format="feather", + partitioning=["ints"], +) + +parquet_dataset = root / "parquet_dataset" +pyarrow.dataset.write_dataset( + table, + parquet_dataset, + format="parquet", + partitioning=["ints"], +) + +#: Loading CSV Files into PostgreSQL +#: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +#: We can directly pass a :py:class:`pyarrow.RecordBatchReader` (from +#: ``open_csv``) to ``adbc_ingest``. We can also pass a +#: :py:class:`pyarrow.dataset.Dataset`, or a +#: :py:class:`pyarrow.dataset.Scanner`. + +with conn.cursor() as cur: + reader = pyarrow.csv.open_csv(csv_file) + cur.adbc_ingest("csvtable", reader, mode="create") + + reader = pyarrow.dataset.dataset( + csv_dataset, + format="csv", + partitioning=["ints"], + ) + cur.adbc_ingest("csvdataset", reader, mode="create") + +conn.commit() + +with conn.cursor() as cur: + cur.execute("SELECT ints, strs FROM csvtable ORDER BY ints, strs ASC") + assert cur.fetchall() == [(1, "bar"), (1, "foo"), (2, "baz")] + + cur.execute("SELECT ints, strs FROM csvdataset ORDER BY ints, strs ASC") + assert cur.fetchall() == [(1, "bar"), (1, "foo"), (2, "baz")] + +#: Loading Arrow IPC (Feather) Files into PostgreSQL +#: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +with conn.cursor() as cur: + reader = pyarrow.ipc.RecordBatchFileReader(ipc_file) + #: Because of quirks in the PyArrow API, we have to read the file into + #: memory. + cur.adbc_ingest("ipctable", reader.read_all(), mode="create") + + #: The Dataset API will stream the data into memory and then into + #: PostgreSQL, though. + reader = pyarrow.dataset.dataset( + ipc_dataset, + format="feather", + partitioning=["ints"], + ) + cur.adbc_ingest("ipcdataset", reader, mode="create") + +conn.commit() + +with conn.cursor() as cur: + cur.execute("SELECT ints, strs FROM ipctable ORDER BY ints, strs ASC") + assert cur.fetchall() == [(1, "bar"), (1, "foo"), (2, "baz")] + + cur.execute("SELECT ints, strs FROM ipcdataset ORDER BY ints, strs ASC") + assert cur.fetchall() == [(1, "bar"), (1, "foo"), (2, "baz")] + +#: Loading Parquet Files into PostgreSQL +#: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +with conn.cursor() as cur: + reader = pyarrow.parquet.ParquetFile(parquet_file) + cur.adbc_ingest("pqtable", reader.iter_batches(), mode="create") + + reader = pyarrow.dataset.dataset( + parquet_dataset, + format="parquet", + partitioning=["ints"], + ) + cur.adbc_ingest("pqdataset", reader, mode="create") + +conn.commit() + +with conn.cursor() as cur: + cur.execute("SELECT ints, strs FROM pqtable ORDER BY ints, strs ASC") + assert cur.fetchall() == [(1, "bar"), (1, "foo"), (2, "baz")] + + cur.execute("SELECT ints, strs FROM pqdataset ORDER BY ints, strs ASC") + assert cur.fetchall() == [(1, "bar"), (1, "foo"), (2, "baz")] + +#: Cleanup +#: ~~~~~~~ + +conn.close() +tempdir.cleanup() diff --git a/python/adbc_driver_manager/adbc_driver_manager/dbapi.py b/python/adbc_driver_manager/adbc_driver_manager/dbapi.py index 8f06ed0396..1e86144c12 100644 --- a/python/adbc_driver_manager/adbc_driver_manager/dbapi.py +++ b/python/adbc_driver_manager/adbc_driver_manager/dbapi.py @@ -43,6 +43,15 @@ except ImportError as e: raise ImportError("PyArrow is required for the DBAPI-compatible interface") from e +try: + import pyarrow.dataset +except ImportError: + _pya_dataset = () + _pya_scanner = () +else: + _pya_dataset = (pyarrow.dataset.Dataset,) + _pya_scanner = (pyarrow.dataset.Scanner,) + import adbc_driver_manager from . import _lib, _reader @@ -891,6 +900,13 @@ def adbc_ingest( else: if isinstance(data, pyarrow.Table): data = data.to_reader() + elif isinstance(data, pyarrow.dataset.Dataset): + data = data.scanner().to_reader() + elif isinstance(data, pyarrow.dataset.Scanner): + data = data.to_reader() + elif not hasattr(data, "_export_to_c"): + data = pyarrow.Table.from_batches(data) + data = data.to_reader() handle = _lib.ArrowArrayStreamHandle() data._export_to_c(handle.address) self._stmt.bind_stream(handle) diff --git a/python/adbc_driver_postgresql/tests/test_dbapi.py b/python/adbc_driver_postgresql/tests/test_dbapi.py index 2a132bd4a7..283e3fe687 100644 --- a/python/adbc_driver_postgresql/tests/test_dbapi.py +++ b/python/adbc_driver_postgresql/tests/test_dbapi.py @@ -15,9 +15,11 @@ # specific language governing permissions and limitations # under the License. +from pathlib import Path from typing import Generator import pyarrow +import pyarrow.dataset import pytest from adbc_driver_postgresql import StatementOptions, dbapi @@ -213,6 +215,60 @@ def test_stmt_ingest(postgres: dbapi.Connection) -> None: assert cur.fetch_arrow_table() == table +def test_stmt_ingest_dataset(postgres: dbapi.Connection, tmp_path: Path) -> None: + # Regression test for https://github.com/apache/arrow-adbc/issues/1310 + table = pyarrow.table( + [ + [1, 1, 2, 2, 3, 3], + ["a", "a", None, None, "b", "b"], + ], + schema=pyarrow.schema([("ints", "int32"), ("strs", "string")]), + ) + pyarrow.dataset.write_dataset( + table, tmp_path, format="parquet", partitioning=["ints"] + ) + ds = pyarrow.dataset.dataset(tmp_path, format="parquet", partitioning=["ints"]) + + with postgres.cursor() as cur: + for item in ( + lambda: ds, + lambda: ds.scanner(), + lambda: ds.scanner().to_reader(), + lambda: ds.scanner().to_table(), + ): + cur.execute("DROP TABLE IF EXISTS test_ingest") + + cur.adbc_ingest( + "test_ingest", + item(), + mode="create_append", + ) + cur.execute("SELECT ints, strs FROM test_ingest ORDER BY ints") + assert cur.fetch_arrow_table() == table + + +def test_stmt_ingest_multi(postgres: dbapi.Connection) -> None: + # Regression test for https://github.com/apache/arrow-adbc/issues/1310 + table = pyarrow.table( + [ + [1, 1, 2, 2, 3, 3], + ["a", "a", None, None, "b", "b"], + ], + names=["ints", "strs"], + ) + + with postgres.cursor() as cur: + cur.execute("DROP TABLE IF EXISTS test_ingest") + + cur.adbc_ingest( + "test_ingest", + table.to_batches(max_chunksize=2), + mode="create_append", + ) + cur.execute("SELECT * FROM test_ingest ORDER BY ints") + assert cur.fetch_arrow_table() == table + + def test_ddl(postgres: dbapi.Connection): with postgres.cursor() as cur: cur.execute("DROP TABLE IF EXISTS test_ddl") From 36ac8b8262d0495884f12d4890125cd22c79d72a Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 22 Dec 2023 13:48:30 -0400 Subject: [PATCH 34/51] chore(r): Housekeeping for R packages (#1399) A few housekeeping updates: - Updated documentation to reflect implementation/installation status in the documentation - Updated NEWS entries up to the current version number - Upstreamed updates requested by CRAN regarding the Snowflake driver (syncing most with the flightsql driver as well). Some of this could be factored out into a common 'how to link a go library from R' package, but it's probably too early for that since there isn't quite consensus on what is actually going to be required yet. --- docs/source/driver/duckdb.rst | 14 +++- docs/source/driver/flight_sql.rst | 2 +- docs/source/driver/installation.rst | 15 ++-- docs/source/driver/postgresql.rst | 3 +- docs/source/driver/sqlite.rst | 5 +- r/adbcdrivermanager/NEWS.md | 8 ++ r/adbcflightsql/DESCRIPTION | 1 + r/adbcflightsql/cleanup | 2 + r/adbcflightsql/configure | 74 +++++++++++++------ r/adbcflightsql/src/.gitignore | 2 + r/adbcflightsql/src/Makevars.in | 3 +- r/adbcpostgresql/NEWS.md | 8 ++ r/adbcpostgresql/cran-comments.md | 4 +- r/adbcsnowflake/DESCRIPTION | 3 +- r/adbcsnowflake/cleanup | 2 + r/adbcsnowflake/configure | 28 +++++-- r/adbcsnowflake/src/.gitignore | 2 + r/adbcsnowflake/src/Makevars.in | 3 +- r/adbcsnowflake/tools/.gitignore | 2 +- .../tools/create-go-vendor-archive.R | 8 +- .../tools/download-go-vendor-archive.R | 6 +- .../tools/verify-go-vendor-archive.R | 64 ++++++++++++++++ r/adbcsqlite/NEWS.md | 8 ++ r/adbcsqlite/cran-comments.md | 4 +- 24 files changed, 214 insertions(+), 57 deletions(-) create mode 100644 r/adbcsnowflake/tools/verify-go-vendor-archive.R diff --git a/docs/source/driver/duckdb.rst b/docs/source/driver/duckdb.rst index ab109b81de..410331c39f 100644 --- a/docs/source/driver/duckdb.rst +++ b/docs/source/driver/duckdb.rst @@ -19,7 +19,7 @@ DuckDB Support ============== -**Available for:** C/C++, GLib/Ruby, Go, Python +**Available for:** C/C++, GLib/Ruby, Go, Python, R `DuckDB`_ provides ADBC support since `version 0.8.0 `_. @@ -87,6 +87,18 @@ ADBC support in DuckDB requires the driver manager. See the `DuckDB Python documentation`_. + .. tab-item:: R + :sync: r + + You must have DuckDB 0.9.1 or higher. + + .. code-block:: r + + # install.packages("duckdb") + library(adbcdrivermanager) + db <- adbc_database_init(duckdb::duckdb_adbc(), ...) + + .. _DuckDB C++ documentation: https://duckdb.org/docs/api/adbc.html#c .. _DuckDB Python documentation: https://duckdb.org/docs/api/adbc.html#python diff --git a/docs/source/driver/flight_sql.rst b/docs/source/driver/flight_sql.rst index db650620a2..aca95d86cd 100644 --- a/docs/source/driver/flight_sql.rst +++ b/docs/source/driver/flight_sql.rst @@ -19,7 +19,7 @@ Flight SQL Driver ================= -**Available for:** C/C++, GLib/Ruby, Go, Java, Python +**Available for:** C/C++, GLib/Ruby, Go, Java, Python, R The Flight SQL Driver provides access to any database implementing a :doc:`arrow:format/FlightSql` compatible endpoint. diff --git a/docs/source/driver/installation.rst b/docs/source/driver/installation.rst index 44f3235a18..bdd1f13a03 100644 --- a/docs/source/driver/installation.rst +++ b/docs/source/driver/installation.rst @@ -145,18 +145,21 @@ From conda-forge_: R = -Install the appropriate driver package from GitHub: +Install the appropriate driver package from CRAN: + +.. code-block:: r + + install.packages("adbcsqlite") + install.packages("adbcpostgresql") + install.packages("duckdb") + +Drivers not yet available on CRAN can be installed from GitHub: .. code-block:: r # install.packages("pak") pak::pak("apache/arrow-adbc/r/adbcflightsql") - pak::pak("apache/arrow-adbc/r/adbcpostgresql") pak::pak("apache/arrow-adbc/r/adbcsnowflake") - pak::pak("apache/arrow-adbc/r/adbcsqlite") - -Installation of stable releases from CRAN is anticipated following the -release of ADBC Libraries 0.6.0. Ruby ==== diff --git a/docs/source/driver/postgresql.rst b/docs/source/driver/postgresql.rst index 282a86a1a6..ddf9115d76 100644 --- a/docs/source/driver/postgresql.rst +++ b/docs/source/driver/postgresql.rst @@ -75,8 +75,7 @@ Installation .. code-block:: r - # install.packages("pak") - pak::pak("apache/arrow-adbc/r/adbcpostgresql") + install.packages("adbcpostgresql") Usage ===== diff --git a/docs/source/driver/sqlite.rst b/docs/source/driver/sqlite.rst index 3f78bc919c..30e7d32b67 100644 --- a/docs/source/driver/sqlite.rst +++ b/docs/source/driver/sqlite.rst @@ -65,10 +65,9 @@ Installation .. tab-item:: R :sync: r - .. code-block:: shell + .. code-block:: r - # install.packages("pak") - pak::pak("apache/arrow-adbc/r/adbcsqlite") + install.packages("adbcsqlite") Usage ===== diff --git a/r/adbcdrivermanager/NEWS.md b/r/adbcdrivermanager/NEWS.md index 0f69dca35f..dadc9aa48a 100644 --- a/r/adbcdrivermanager/NEWS.md +++ b/r/adbcdrivermanager/NEWS.md @@ -1,3 +1,11 @@ +# adbcdrivermanager 0.8.0 + +- Update upstream ADBC libraries to version 0.8.0. + +# adbcdrivermanager 0.7.0 + +- Update upstream ADBC libraries to version 0.7.0. + # adbcdrivermanager 0.6.0 - **r**: Ensure that info_codes are coerced to integer (#986) diff --git a/r/adbcflightsql/DESCRIPTION b/r/adbcflightsql/DESCRIPTION index c9ea48995c..43707deb0a 100644 --- a/r/adbcflightsql/DESCRIPTION +++ b/r/adbcflightsql/DESCRIPTION @@ -19,6 +19,7 @@ RoxygenNote: 7.2.3 Suggests: nanoarrow, testthat (>= 3.0.0) +SystemRequirements: GNU make, Go (>= 1.19.13) Config/testthat/edition: 3 Config/build/bootstrap: TRUE URL: https://github.com/apache/arrow-adbc diff --git a/r/adbcflightsql/cleanup b/r/adbcflightsql/cleanup index 01e970b324..2261ddb19a 100755 --- a/r/adbcflightsql/cleanup +++ b/r/adbcflightsql/cleanup @@ -16,3 +16,5 @@ # under the License. rm src/*.o src/go/*.a +rm -rf src/.go-cache +rm -rf src/.go-path diff --git a/r/adbcflightsql/configure b/r/adbcflightsql/configure index ef44e54df5..3ba0a59462 100755 --- a/r/adbcflightsql/configure +++ b/r/adbcflightsql/configure @@ -18,6 +18,22 @@ R_BIN="$R_HOME/bin/R" RSCRIPT_BIN="$R_HOME/bin/Rscript" +# Set to non-false for CRAN releases, which require this approach to comply with +# guidance regarding large dependency sources. When non-false, this script +# will fail if the download fails or if checksum verification fails on +# the downloaded file. +DOWNLOAD_DEPENDENCY_ARCHIVE="false" + +# CRAN checks CPU time to enforce ~2 cores for building. Set GOMAXPROCS to 1 +# when building on CRAN to be safe. +if [ ! "$DOWNLOAD_DEPENDENCY_ARCHIVE" = "false" ] && [ -z "$GOMAXPROCS" ]; then + echo "Using GOMAXPROCS=1 for go build" + echo "Set GOMAXPROCS to a higher value in ~/.Renviron for a faster build" + GOMAXPROCS=1 +elif [ ! -z "$GOMAXPROCS" ]; then + echo "Using GOMAXPROCS=$GOMAXPROCS for go build" +fi + # Run bootstrap.R. This will have already run if we are installing a source # package built with pkgbuild::build() with pkgbuild >1.4.0; however, we # run it again in case this is R CMD INSTALL on a directory or @@ -29,26 +45,45 @@ if [ -f bootstrap.R ]; then fi # Find the go binary so that we can go build! +# If we've downloaded a specific version of go to src/go/tmp, use that +# one (helpful for testing different version of go locally) +PREVIOUSLY_DOWNLOADED_GO="`pwd`/src/go/tmp/go/bin/go" +if [ -z "$GO_BIN" ] && [ -f "$PREVIOUSLY_DOWNLOADED_GO" ]; then + GO_BIN="$PREVIOUSLY_DOWNLOADED_GO" +fi + +# Check go on PATH if [ -z "$GO_BIN" ]; then GO_BIN=`which go` fi +# Try some default install locations that might not be on PATH +DEFAULT_GO_WIN="C:/Program Files/Go/bin/go.exe" +if [ -z "$GO_BIN" ] && [ -f "$DEFAULT_GO_WIN" ]; then + GO_BIN="$DEFAULT_GO_WIN" +fi + +DEFAULT_GO_MACOS="/usr/local/go/bin/go" +if [ -z "$GO_BIN" ] && [ -f "$DEFAULT_GO_MACOS" ]; then + GO_BIN="$DEFAULT_GO_MACOS" +fi + +DEFAULT_GO_HOMEBREW_M1="/opt/homebrew/bin/go" +if [ -z "$GO_BIN" ] && [ -f "$DEFAULT_GO_HOMEBREW_M1" ]; then + GO_BIN="$DEFAULT_GO_HOMEBREW_M1" +fi + if [ -z "$GO_BIN" ]; then - if [ ! -f src/go/tmp/go/bin/go ]; then - echo "" - echo "Downloading and extracting Go into the package source directory:" - echo "This may take a few minutes. To eliminate this step, install Go" - echo "from your faviourite package manager or set the GO_BIN environment variable:" - echo "- apt-get install golang" - echo "- brew install golang" - echo "- dnf install golang" - echo "- apk add go" - echo "- pacman -S go" - - "$RSCRIPT_BIN" tools/download-go.R - fi - - GO_BIN="`pwd`/src/go/tmp/go/bin/go" + echo "" + echo "The Go compiler is required to install this package. You can install go" + echo "from your faviourite package manager or set the GO_BIN environment variable:" + echo "- apt-get install golang" + echo "- brew install golang" + echo "- dnf install golang" + echo "- apk add go" + echo "- pacman -S go" + echo "...or from the official installers available at https://go.dev/dl/" + exit 1 fi echo "Trying 'go version' with GO_BIN at '$GO_BIN'" @@ -62,12 +97,8 @@ fi CC=`"$R_BIN" CMD config CC` CXX=`"$R_BIN" CMD config CXX` -# clang and gcc use different symbol-hiding syntax and we need to -# make sure to hide any Adbc* symbols that might conflict with another -# driver. -if "$R_BIN" CMD config CC | grep -e "clang" ; then - SYMBOL_ARGS="-Wl,-exported_symbol,_adbcflightsql_c_flightsql -Wl,-exported_symbol,_R_init_adbcflightsql" -elif "$R_BIN" CMD config CC | grep -e "gcc" ; then +# Attempt to hide symbols where possible +if "$R_BIN" CMD config CC | grep -e "gcc" >/dev/null ; then SYMBOL_ARGS="-Wl,--version-script=go/symbols.map" fi @@ -83,6 +114,7 @@ sed \ -e "s|@libs@|$PKG_LIBS|" \ -e "s|@cc@|$CC|" \ -e "s|@cxx@|$CXX|" \ + -e "s|@nproc@|$GOMAXPROCS|" \ src/Makevars.in > src/Makevars if [ -f "src/go/adbc/pkg/flightsql/driver.go" ]; then diff --git a/r/adbcflightsql/src/.gitignore b/r/adbcflightsql/src/.gitignore index fb883613f9..e040acd74e 100644 --- a/r/adbcflightsql/src/.gitignore +++ b/r/adbcflightsql/src/.gitignore @@ -17,3 +17,5 @@ adbc.h Makevars +.go-cache +.go-path diff --git a/r/adbcflightsql/src/Makevars.in b/r/adbcflightsql/src/Makevars.in index eb74ef3356..86101d5054 100644 --- a/r/adbcflightsql/src/Makevars.in +++ b/r/adbcflightsql/src/Makevars.in @@ -21,10 +21,11 @@ PKG_LIBS=-L$(CURDIR)/go -ladbc_driver_flightsql -lresolv @libs@ CGO_CC = @cc@ CGO_CXX = @cxx@ CGO_CFLAGS = $(ALL_CPPFLAGS) +GOMAXPROCS = @nproc@ .PHONY: all gostatic all: $(SHLIB) $(SHLIB): gostatic gostatic: - (cd "$(CURDIR)/go/adbc"; CC="$(CGO_CC)" CXX="$(CGO_CXX)" CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(PKG_LIBS)" "@gobin@" build -v -tags driverlib -o $(CURDIR)/go/libadbc_driver_flightsql.a -buildmode=c-archive "./pkg/flightsql") + (cd "$(CURDIR)/go/adbc"; GOMAXPROCS=$(GOMAXPROCS) GOPATH="$(CURDIR)/.go-path" GOCACHE="$(CURDIR)/.go-cache" CC="$(CGO_CC)" CXX="$(CGO_CXX)" CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(PKG_LIBS)" "@gobin@" build -v -tags driverlib -o $(CURDIR)/go/libadbc_driver_flightsql.a -buildmode=c-archive "./pkg/flightsql") diff --git a/r/adbcpostgresql/NEWS.md b/r/adbcpostgresql/NEWS.md index 842387e43e..700150c221 100644 --- a/r/adbcpostgresql/NEWS.md +++ b/r/adbcpostgresql/NEWS.md @@ -1,3 +1,11 @@ +# adbcdrivermanager 0.8.0 + +- Update upstream ADBC libraries to version 0.8.0. + +# adbcdrivermanager 0.7.0 + +- Update upstream ADBC libraries to version 0.7.0. + # adbcpostgresql 0.6.0 * Initial CRAN submission. diff --git a/r/adbcpostgresql/cran-comments.md b/r/adbcpostgresql/cran-comments.md index 2d2bbbdc12..906efef731 100644 --- a/r/adbcpostgresql/cran-comments.md +++ b/r/adbcpostgresql/cran-comments.md @@ -4,6 +4,4 @@ Apache Arrow ADBC libraries version. ## R CMD check results -0 errors | 0 warnings | 1 note - -* This is a new release. +0 errors | 0 warnings | 0 notes diff --git a/r/adbcsnowflake/DESCRIPTION b/r/adbcsnowflake/DESCRIPTION index 5d7af67367..f22f18ba4e 100644 --- a/r/adbcsnowflake/DESCRIPTION +++ b/r/adbcsnowflake/DESCRIPTION @@ -18,8 +18,9 @@ Roxygen: list(markdown = TRUE) RoxygenNote: 7.2.3 Suggests: nanoarrow, + openssl, testthat (>= 3.0.0) -SystemRequirements: GNU make +SystemRequirements: GNU make, Go (>= 1.19.13) Config/testthat/edition: 3 Config/build/bootstrap: TRUE URL: https://github.com/apache/arrow-adbc diff --git a/r/adbcsnowflake/cleanup b/r/adbcsnowflake/cleanup index 01e970b324..2261ddb19a 100755 --- a/r/adbcsnowflake/cleanup +++ b/r/adbcsnowflake/cleanup @@ -16,3 +16,5 @@ # under the License. rm src/*.o src/go/*.a +rm -rf src/.go-cache +rm -rf src/.go-path diff --git a/r/adbcsnowflake/configure b/r/adbcsnowflake/configure index 9157426196..e5a101435a 100755 --- a/r/adbcsnowflake/configure +++ b/r/adbcsnowflake/configure @@ -24,6 +24,16 @@ RSCRIPT_BIN="$R_HOME/bin/Rscript" # the downloaded file. DOWNLOAD_DEPENDENCY_ARCHIVE="false" +# CRAN checks CPU time to enforce ~2 cores for building. Set GOMAXPROCS to 1 +# when building on CRAN to be safe. +if [ ! "$DOWNLOAD_DEPENDENCY_ARCHIVE" = "false" ] && [ -z "$GOMAXPROCS" ]; then + echo "Using GOMAXPROCS=1 for go build" + echo "Set GOMAXPROCS to a higher value in ~/.Renviron for a faster build" + GOMAXPROCS=1 +elif [ ! -z "$GOMAXPROCS" ]; then + echo "Using GOMAXPROCS=$GOMAXPROCS for go build" +fi + # Run bootstrap.R. This will have already run if we are installing a source # package built with pkgbuild::build() with pkgbuild >1.4.0; however, we # run it again in case this is R CMD INSTALL on a directory or @@ -62,8 +72,13 @@ if [ -f tools/src-go-adbc-vendor.zip ]; then cd .. "$RSCRIPT_BIN" tools/extract-go-vendor-archive.R else - echo "Checksum verification failed for vendored dependency archive" - exit 1 + cd .. + if "$RSCRIPT_BIN" tools/verify-go-vendor-archive.R ; then + "$RSCRIPT_BIN" tools/extract-go-vendor-archive.R + else + echo "Checksum verification failed for vendored dependency archive" + exit 1 + fi fi fi @@ -120,12 +135,8 @@ fi CC=`"$R_BIN" CMD config CC` CXX=`"$R_BIN" CMD config CXX` -# clang and gcc use different symbol-hiding syntax and we need to -# make sure to hide any Adbc* symbols that might conflict with another -# driver. -if "$R_BIN" CMD config CC | grep -e "clang" >/dev/null ; then - SYMBOL_ARGS="-Wl,-exported_symbol,_adbcsnowflake_c_snowflake -Wl,-exported_symbol,_R_init_adbcsnowflake" -elif "$R_BIN" CMD config CC | grep -e "gcc" >/dev/null ; then +# Attempt to hide symbols where possible +if "$R_BIN" CMD config CC | grep -e "gcc" >/dev/null ; then SYMBOL_ARGS="-Wl,--version-script=go/symbols.map" fi @@ -148,6 +159,7 @@ sed \ -e "s|@cflags@|$PKG_CPPFLAGS|" \ -e "s|@cc@|$CC|" \ -e "s|@cxx@|$CXX|" \ + -e "s|@nproc@|$GOMAXPROCS|" \ src/Makevars.in > src/Makevars if [ -f "src/go/adbc/pkg/snowflake/driver.go" ]; then diff --git a/r/adbcsnowflake/src/.gitignore b/r/adbcsnowflake/src/.gitignore index fb883613f9..e040acd74e 100644 --- a/r/adbcsnowflake/src/.gitignore +++ b/r/adbcsnowflake/src/.gitignore @@ -17,3 +17,5 @@ adbc.h Makevars +.go-cache +.go-path diff --git a/r/adbcsnowflake/src/Makevars.in b/r/adbcsnowflake/src/Makevars.in index 58dc4a36c9..fcc1c0ee13 100644 --- a/r/adbcsnowflake/src/Makevars.in +++ b/r/adbcsnowflake/src/Makevars.in @@ -21,10 +21,11 @@ PKG_LIBS=-L$(CURDIR)/go -ladbc_driver_snowflake @libs@ CGO_CC = @cc@ CGO_CXX = @cxx@ CGO_CFLAGS = $(ALL_CPPFLAGS) +GOMAXPROCS = @nproc@ .PHONY: all gostatic all: $(SHLIB) $(SHLIB): gostatic gostatic: - (cd "$(CURDIR)/go/adbc"; CC="$(CGO_CC)" CXX="$(CGO_CXX)" CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(PKG_LIBS)" "@gobin@" build -v -tags driverlib -o $(CURDIR)/go/libadbc_driver_snowflake.a -buildmode=c-archive "./pkg/snowflake") + (cd "$(CURDIR)/go/adbc"; GOMAXPROCS=$(GOMAXPROCS) GOPATH="$(CURDIR)/.go-path" GOCACHE="$(CURDIR)/.go-cache" CC="$(CGO_CC)" CXX="$(CGO_CXX)" CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(PKG_LIBS)" "@gobin@" build -v -tags driverlib -o $(CURDIR)/go/libadbc_driver_snowflake.a -buildmode=c-archive "./pkg/snowflake") diff --git a/r/adbcsnowflake/tools/.gitignore b/r/adbcsnowflake/tools/.gitignore index eb6f76fe0f..c8439b9b68 100644 --- a/r/adbcsnowflake/tools/.gitignore +++ b/r/adbcsnowflake/tools/.gitignore @@ -15,4 +15,4 @@ # specific language governing permissions and limitations # under the License. -src-go-adbc-vendor.zip* +src-go-adbc-vendor.zip diff --git a/r/adbcsnowflake/tools/create-go-vendor-archive.R b/r/adbcsnowflake/tools/create-go-vendor-archive.R index 04c7ed33d6..a9045b0598 100644 --- a/r/adbcsnowflake/tools/create-go-vendor-archive.R +++ b/r/adbcsnowflake/tools/create-go-vendor-archive.R @@ -20,7 +20,7 @@ go_bin <- Sys.getenv("GO_BIN", unname(Sys.which("go"))) withr::with_dir("src/go/adbc", { system(paste(shQuote(go_bin), "mod vendor -v")) - # go mod vendor for arrow/v13 doesn't include some files needed for go build + # go mod vendor for arrow/v14 doesn't include some files needed for go build tmp_zip <- tempfile() tmp_extract <- tempfile() local({ @@ -30,7 +30,7 @@ withr::with_dir("src/go/adbc", { }) curl::curl_download( - "https://github.com/apache/arrow/archive/refs/tags/go/v13.0.0.zip", + "https://github.com/apache/arrow/archive/refs/tags/go/v14.0.2.zip", tmp_zip ) @@ -38,10 +38,10 @@ withr::with_dir("src/go/adbc", { src_go_arrow_cdata_arrow_dir <- file.path( tmp_extract, - "arrow-go-v13.0.0/go/arrow/cdata/arrow" + "arrow-go-v14.0.2/go/arrow/cdata/arrow" ) - dst_go_arrow_cdata_dir <- "vendor/github.com/apache/arrow/go/v13/arrow/cdata/" + dst_go_arrow_cdata_dir <- "vendor/github.com/apache/arrow/go/v14/arrow/cdata/" stopifnot(file.copy(src_go_arrow_cdata_arrow_dir, dst_go_arrow_cdata_dir, recursive = TRUE)) }) diff --git a/r/adbcsnowflake/tools/download-go-vendor-archive.R b/r/adbcsnowflake/tools/download-go-vendor-archive.R index 5c992c40d7..89902efd79 100644 --- a/r/adbcsnowflake/tools/download-go-vendor-archive.R +++ b/r/adbcsnowflake/tools/download-go-vendor-archive.R @@ -25,4 +25,8 @@ if (identical(uri, "")) { cat(sprintf("Downloading vendored dependency archive from %s\n", uri)) unlink("tools/src-go-adbc-vendor.zip") -download.file(uri, "tools/src-go-adbc-vendor.zip") +local({ + opts <- options(timeout = max(300, getOption("timeout"))) + on.exit(options(opts)) + download.file(uri, "tools/src-go-adbc-vendor.zip", mode = "wb") +}) diff --git a/r/adbcsnowflake/tools/verify-go-vendor-archive.R b/r/adbcsnowflake/tools/verify-go-vendor-archive.R new file mode 100644 index 0000000000..6e1a01008e --- /dev/null +++ b/r/adbcsnowflake/tools/verify-go-vendor-archive.R @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +digest_openssl <- function(f) { + con <- file(f, "rb") + on.exit(close(con)) + as.character(as.character(openssl::sha512(con))) +} + +digest_digest <- function(f) { + digest::digest(f, algo = "sha512", file = TRUE) +} + +read_check <- function(f) { + con <- file(f, "rb") + on.exit(close(con)) + scan(con, character(1), n = 1, quiet = TRUE) +} + +verify <- function() { + if (requireNamespace("digest", quietly = TRUE)) { + cat("Using digest::digest() to verify digest\n") + digest <- digest_digest("tools/src-go-adbc-vendor.zip") + } else if (requireNamespace("openssl", quietly = TRUE)) { + cat("Using openssl::sha512() to verify digest\n") + digest <- digest_openssl("tools/src-go-adbc-vendor.zip") + } else { + cat("openssl nor digest package was installed to verify digest\n") + return(FALSE) + } + + digest_check <- read_check("tools/src-go-adbc-vendor.zip.sha512") + result <- identical(digest_check, digest) + + if (isTRUE(result)) { + result + } else { + cat(sprintf("Digest: %s\n", digest)) + cat(sprintf("Check : %s\n", digest_check)) + FALSE + } +} + +result <- try(verify()) + +if (!isTRUE(result) && !interactive()) { + q(status = 1) +} else if (!interactive()) { + q(status = 0) +} diff --git a/r/adbcsqlite/NEWS.md b/r/adbcsqlite/NEWS.md index f77c65de3c..59b9be569b 100644 --- a/r/adbcsqlite/NEWS.md +++ b/r/adbcsqlite/NEWS.md @@ -1,3 +1,11 @@ +# adbcdrivermanager 0.8.0 + +- Update upstream ADBC libraries to version 0.8.0. + +# adbcdrivermanager 0.7.0 + +- Update upstream ADBC libraries to version 0.7.0. + # adbcsqlite 0.6.0 * Initial CRAN submission. diff --git a/r/adbcsqlite/cran-comments.md b/r/adbcsqlite/cran-comments.md index 2d2bbbdc12..906efef731 100644 --- a/r/adbcsqlite/cran-comments.md +++ b/r/adbcsqlite/cran-comments.md @@ -4,6 +4,4 @@ Apache Arrow ADBC libraries version. ## R CMD check results -0 errors | 0 warnings | 1 note - -* This is a new release. +0 errors | 0 warnings | 0 notes From 106a5ce16160aebdfe67b397edaa84bf15976d50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 14:50:12 -0800 Subject: [PATCH 35/51] chore(csharp): bump xunit from 2.6.3 to 2.6.4 in /csharp (#1412) Bumps [xunit](https://github.com/xunit/xunit) from 2.6.3 to 2.6.4.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=xunit&package-manager=nuget&previous-version=2.6.3&new-version=2.6.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj | 2 +- .../BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj | 2 +- .../FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj | 2 +- .../Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj index efbd47d5c9..c54373d4d5 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj index 7c758785e6..3c787c4a73 100644 --- a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj +++ b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj @@ -4,7 +4,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj index 744cb4e66b..cab38edfb5 100644 --- a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj +++ b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj @@ -6,7 +6,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj index 3297525bad..623047882a 100644 --- a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj +++ b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj @@ -4,7 +4,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive From 607e7f44703bcec4e1f8b60bf807068ec37ef6e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 14:50:26 -0800 Subject: [PATCH 36/51] chore(csharp): bump Google.Cloud.BigQuery.V2 from 3.2.0 to 3.5.0 in /csharp (#1411) Bumps [Google.Cloud.BigQuery.V2](https://github.com/googleapis/google-cloud-dotnet) from 3.2.0 to 3.5.0.
Commits
  • a9f6fb6 Release Google.Cloud.BigQuery.V2 version 3.5.0 (#11177)
  • 7f578e5 chore: Generate Google.Cloud.SecureSourceManager.V1
  • 8e6d1e3 feat: Add Google.Cloud.SecureSourceManager.V1
  • 5f52f01 Release Google.Shopping.Type version 1.0.0-beta02 (#11174)
  • 3e2da96 chore(deps): update dependency octokit to v8.1.1
  • 6223a25 docs: Update ConfigurationModifier property documentation
  • 7491a30 feat: Channel enum is added
  • a7431da feat: added the route modifiers
  • 3fbf033 feat: ReportingContext enum is added
  • 17b1777 chore: Stop automatic major version updates for .NET
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Google.Cloud.BigQuery.V2&package-manager=nuget&previous-version=3.2.0&new-version=3.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj index 0411f1e369..b6c7a00c69 100644 --- a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj +++ b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj @@ -6,7 +6,7 @@ - + From 4482eab3946e8ea29e43b42efd004267874b0e1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 14:51:37 -0800 Subject: [PATCH 37/51] chore: bump actions/setup-dotnet from 3 to 4 (#1405) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-dotnet](https://github.com/actions/setup-dotnet) from 3 to 4.
Release notes

Sourced from actions/setup-dotnet's releases.

v4.0.0

What's Changed

In the scope of this release, the version of the Node.js runtime was updated to 20. The majority of dependencies were updated to the latest versions. From now on, the code for the setup-dotnet will run on Node.js 20 instead of Node.js 16.

Breaking changes

Update Node.js runtime to version 20 by @​harithavattikuti in #484

Non-breaking changes

New Contributors

Full Changelog: https://github.com/actions/setup-dotnet/compare/v3...v4.0.0

v3.2.0

What's Changed

In scope of this minor release, the ability to cache the NuGet global-packages folder was added in actions/setup-dotnet#303 by @​nogic1008

For caching, the action uses the @​toolkit/cache library under the hood, which in turn allows getting rid of configuring the @​actions/cache action separately.

Such input parameters as cache and cache-dependency-path were added. The cache input is optional, and caching is turned off by default, cache-dependency-path is used to specify the path to a dependency file - packages.lock.json.

Example of use-case:

- uses: actions/setup-dotnet@v3
  with:
    dotnet-version: 6.x
    cache: true
    cache-dependency-path: subdir/packages.lock.json

More details can be found in the action's documentation.

Full Changelog: https://github.com/actions/setup-dotnet/compare/v3...v3.2.0

v3.1.0

What's Changed

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-dotnet&package-manager=github_actions&previous-version=3&new-version=4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/csharp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index 9e701df55e..2cd9ad1db2 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -49,7 +49,7 @@ jobs: os: [ubuntu-latest, windows-2019, macos-latest] steps: - name: Install C# - uses: actions/setup-dotnet@v3 + uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ matrix.dotnet }} - name: Checkout ADBC From 3cff206d66110f50e95c71941d76f63d3d8d7198 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 15:45:38 -0800 Subject: [PATCH 38/51] chore(csharp): bump System.Text.Json from 7.0.3 to 7.0.4 in /csharp (#1408) Bumps [System.Text.Json](https://github.com/dotnet/runtime) from 7.0.3 to 7.0.4.
Release notes

Sourced from System.Text.Json's releases.

.NET 7.0.4

Release

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=System.Text.Json&package-manager=nuget&previous-version=7.0.3&new-version=7.0.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj index b6c7a00c69..6d50c3efc8 100644 --- a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj +++ b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj @@ -7,7 +7,7 @@ - + From 9b2b8e08a580b453644cf59eb423c8e8d2aa7eea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 19:19:39 -0800 Subject: [PATCH 39/51] chore(csharp): bump xunit.runner.visualstudio from 2.5.5 to 2.5.6 in /csharp (#1409) Bumps [xunit.runner.visualstudio](https://github.com/xunit/visualstudio.xunit) from 2.5.5 to 2.5.6.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=xunit.runner.visualstudio&package-manager=nuget&previous-version=2.5.5&new-version=2.5.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj | 2 +- .../FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj | 2 +- .../Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj index 3c787c4a73..209c7d46cd 100644 --- a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj +++ b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj @@ -5,7 +5,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj index cab38edfb5..18f8f67b29 100644 --- a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj +++ b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj @@ -7,7 +7,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj index 623047882a..3e90defff5 100644 --- a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj +++ b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj @@ -5,7 +5,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive From 3c0ca4b590c39e459aa77e5c30620260ffd52a05 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Mon, 25 Dec 2023 19:44:39 -0800 Subject: [PATCH 40/51] chore(csharp): bump Microsoft.NET.Test.Sdk from 17.6.0 to 17.8.0 in /csharp (#1414) (The dependabot upgrade isn't working probably because the source has a mix of versions.) --- .../test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj | 2 +- .../BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj | 2 +- .../FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj | 2 +- .../Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj index c54373d4d5..7deb5dc0b5 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Tests.csproj @@ -9,7 +9,7 @@ - + diff --git a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj index 209c7d46cd..d4aba6faf6 100644 --- a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj +++ b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj @@ -3,7 +3,7 @@ net472;net6.0 - + all diff --git a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj index 18f8f67b29..4f72495c4b 100644 --- a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj +++ b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj @@ -5,7 +5,7 @@ False - + all diff --git a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj index 3e90defff5..99f13c3d85 100644 --- a/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj +++ b/csharp/test/Drivers/Interop/Snowflake/Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake.csproj @@ -3,7 +3,7 @@ net472;net6.0 - + all From b8cb2720652db092857cc798fa840b81eb84a548 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 08:02:03 -0500 Subject: [PATCH 41/51] chore: bump actions/setup-go from 3 to 5 (#1403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3 to 5.
Release notes

Sourced from actions/setup-go's releases.

v5.0.0

What's Changed

In scope of this release, we change Nodejs runtime from node16 to node20 (actions/setup-go#421). Moreover, we update some dependencies to the latest versions (actions/setup-go#445).

Besides, this release contains such changes as:

New Contributors

Full Changelog: https://github.com/actions/setup-go/compare/v4...v5.0.0

v4.1.0

What's Changed

In scope of this release, slow installation on Windows was fixed by @​dsame in actions/setup-go#393 and OS version was added to primaryKey for Ubuntu runners to avoid conflicts (actions/setup-go#383)

This release also includes the following changes:

New Contributors

Full Changelog: https://github.com/actions/setup-go/compare/v4...v4.1.0

v4.0.1

What's Changed

New Contributors

Full Changelog: https://github.com/actions/setup-go/compare/v4...v4.0.1

v4.0.0

In scope of release we enable cache by default. The action won’t throw an error if the cache can’t be restored or saved. The action will throw a warning message but it won’t stop a build process. The cache can be disabled by specifying cache: false.

</tr></table>

... (truncated)

Commits
  • 0c52d54 Update dependencies for node20 (#445)
  • bfd2fb3 Merge pull request #421 from chenrui333/node20-runtime
  • 3d65fa5 feat: bump to use actions/checkout@v4
  • 8a505c9 feat: bump to use node20 runtime
  • 883490d Merge pull request #417 from artemgavrilov/main
  • d45ebba Rephrase sentence
  • 317c661 Replace wildcards term with globs.
  • f90673a Merge pull request #1 from artemgavrilov/caching-docs-improvement
  • 8018234 Improve documentation regarding dependencies cachin
  • d085b4f Merge pull request #411 from galargh/fix/windows-hostedtoolcache
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-go&package-manager=github_actions&previous-version=3&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/integration.yml | 4 ++-- .github/workflows/native-unix.yml | 8 ++++---- .github/workflows/native-windows.yml | 2 +- .github/workflows/packaging.yml | 4 ++-- .github/workflows/r-check.yml | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index cdbc1296ab..dbd5589239 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -37,7 +37,7 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version-file: 'go/adbc/go.mod' check-latest: true diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ba1ed14448..bd6d79d16f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -129,7 +129,7 @@ jobs: --file ci/conda_env_cpp.txt \ --file ci/conda_env_python.txt pip install pytest-error-for-skips - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version: 1.19.13 check-latest: true @@ -348,7 +348,7 @@ jobs: mamba install -c conda-forge \ --file ci/conda_env_cpp.txt \ --file ci/conda_env_python.txt - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version: 1.19.13 check-latest: true diff --git a/.github/workflows/native-unix.yml b/.github/workflows/native-unix.yml index d8084da790..c498cb96c7 100644 --- a/.github/workflows/native-unix.yml +++ b/.github/workflows/native-unix.yml @@ -94,7 +94,7 @@ jobs: run: | mamba install -c conda-forge \ --file ci/conda_env_cpp.txt - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version: 1.19.13 check-latest: true @@ -298,7 +298,7 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version-file: 'go/adbc/go.mod' check-latest: true @@ -357,7 +357,7 @@ jobs: run: | mamba install -c conda-forge \ --file ci/conda_env_cpp.txt - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version: 1.19.13 check-latest: true @@ -437,7 +437,7 @@ jobs: python=${{ matrix.python }} \ --file ci/conda_env_cpp.txt \ --file ci/conda_env_python.txt - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version: 1.19.13 check-latest: true diff --git a/.github/workflows/native-windows.yml b/.github/workflows/native-windows.yml index 747495a635..acee122c6b 100644 --- a/.github/workflows/native-windows.yml +++ b/.github/workflows/native-windows.yml @@ -221,7 +221,7 @@ jobs: run: | mamba install -c conda-forge \ --file ci/conda_env_cpp.txt - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version: 1.20.8 check-latest: true diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index d6dd0b01ff..82b39bc163 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -605,7 +605,7 @@ jobs: ci/scripts/install_vcpkg.sh $VCPKG_ROOT ${{ steps.vcpkg_version.outputs.VCPKG_VERSION }} popd - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version: 1.19.13 check-latest: true @@ -729,7 +729,7 @@ jobs: # Windows needs newer Go than 1.19 # https://github.com/golang/go/issues/51007 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version: 1.20.8 check-latest: true diff --git a/.github/workflows/r-check.yml b/.github/workflows/r-check.yml index a6e6983097..e29f7f3ed7 100644 --- a/.github/workflows/r-check.yml +++ b/.github/workflows/r-check.yml @@ -41,7 +41,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version: '1.20' - uses: r-lib/actions/setup-r@v2 From c77fdde263b04ad78fcc13111af8ac10b4eaee45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 08:08:15 -0500 Subject: [PATCH 42/51] chore: bump actions/setup-python from 4 to 5 (#1417) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5.
Release notes

Sourced from actions/setup-python's releases.

v5.0.0

What's Changed

In scope of this release, we update node version runtime from node16 to node20 (actions/setup-python#772). Besides, we update dependencies to the latest versions.

Full Changelog: https://github.com/actions/setup-python/compare/v4.8.0...v5.0.0

v4.8.0

What's Changed

In scope of this release we added support for GraalPy (actions/setup-python#694). You can use this snippet to set up GraalPy:

steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
  with:
    python-version: 'graalpy-22.3'
- run: python my_script.py

Besides, the release contains such changes as:

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v4...v4.8.0

v4.7.1

What's Changed

Full Changelog: https://github.com/actions/setup-python/compare/v4...v4.7.1

v4.7.0

In scope of this release, the support for reading python version from pyproject.toml was added (actions/setup-python#669).

      - name: Setup Python
        uses: actions/setup-python@v4
</tr></table>

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=4&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/packaging.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index dbd5589239..e4d57943f9 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -41,7 +41,7 @@ jobs: with: go-version-file: 'go/adbc/go.mod' check-latest: true - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 - name: install golangci-lint run: | go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.49.0 diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 82b39bc163..df965da310 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -737,7 +737,7 @@ jobs: cache-dependency-path: adbc/go/adbc/go.sum - name: Install Python ${{ matrix.python_version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} From 6166fa6287e5eddde34d0b1ab163fa322a92fead Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 08:11:09 -0500 Subject: [PATCH 43/51] chore(go): bump github.com/dvsekhvalnov/jose2go from 1.5.0 to 1.6.0 in /go/adbc (#1413) Bumps [github.com/dvsekhvalnov/jose2go](https://github.com/dvsekhvalnov/jose2go) from 1.5.0 to 1.6.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/dvsekhvalnov/jose2go&package-manager=go_modules&previous-version=1.5.0&new-version=1.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/apache/arrow-adbc/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go/adbc/go.mod | 2 +- go/adbc/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/adbc/go.mod b/go/adbc/go.mod index 7368231f3d..69e9eaf591 100644 --- a/go/adbc/go.mod +++ b/go/adbc/go.mod @@ -60,7 +60,7 @@ require ( github.com/danieljoos/wincred v1.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/dvsekhvalnov/jose2go v1.5.0 // indirect + github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/goccy/go-json v0.10.2 // indirect diff --git a/go/adbc/go.sum b/go/adbc/go.sum index edb94077b9..d8e09e3ffb 100644 --- a/go/adbc/go.sum +++ b/go/adbc/go.sum @@ -69,8 +69,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM= -github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= +github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= +github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= From d7e06cb9736df9a43503f128ccbb021a5bfd5958 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 08:11:39 -0500 Subject: [PATCH 44/51] chore: bump docker/login-action from 2 to 3 (#1407) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3.
Release notes

Sourced from docker/login-action's releases.

v3.0.0

Full Changelog: https://github.com/docker/login-action/compare/v2.2.0...v3.0.0

v2.2.0

Full Changelog: https://github.com/docker/login-action/compare/v2.1.0...v2.2.0

v2.1.0

  • Ensure AWS temp credentials are redacted in workflow logs by @​crazy-max (#275)
  • Bump @​actions/core from 1.6.0 to 1.10.0 (#252 #292)
  • Bump @​aws-sdk/client-ecr from 3.53.0 to 3.186.0 (#298)
  • Bump @​aws-sdk/client-ecr-public from 3.53.0 to 3.186.0 (#299)

Full Changelog: https://github.com/docker/login-action/compare/v2.0.0...v2.1.0

Commits
  • 343f7c4 Merge pull request #599 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • aad0f97 chore: update generated content
  • 2e0cd39 build(deps): bump the aws-sdk-dependencies group with 2 updates
  • 203bc9c Merge pull request #588 from docker/dependabot/npm_and_yarn/proxy-agent-depen...
  • 2199648 chore: update generated content
  • b489376 build(deps): bump the proxy-agent-dependencies group with 1 update
  • 7c309e7 Merge pull request #598 from docker/dependabot/npm_and_yarn/actions/core-1.10.1
  • 0ccf222 chore: update generated content
  • 56d703e Merge pull request #597 from docker/dependabot/github_actions/aws-actions/con...
  • 24d3b35 build(deps): bump @​actions/core from 1.10.0 to 1.10.1
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=2&new-version=3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/packaging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index df965da310..7d5d2ccffb 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -275,7 +275,7 @@ jobs: restore-keys: linux-${{ env.TASK_NAMESPACE }}-ccache-${{ matrix.target }}- - name: Login to GitHub Container registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} From e8224bfc8f6748e3a14baf3c9ebb70e72f273829 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:42:12 -0500 Subject: [PATCH 45/51] chore: bump actions/download-artifact from 3 to 4 (#1404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4.
Release notes

Sourced from actions/download-artifact's releases.

v4.0.0

What's Changed

The release of upload-artifact@v4 and download-artifact@v4 are major changes to the backend architecture of Artifacts. They have numerous performance and behavioral improvements.

For more information, see the @​actions/artifact documentation.

New Contributors

Full Changelog: https://github.com/actions/download-artifact/compare/v3...v4.0.0

v3.0.2

  • Bump @actions/artifact to v1.1.1 - actions/download-artifact#195
  • Fixed a bug in Node16 where if an HTTP download finished too quickly (<1ms, e.g. when it's mocked) we attempt to delete a temp file that has not been created yet actions/toolkit#1278

v3.0.1

Commits
  • f44cd7b Merge pull request #259 from actions/robherley/glob-downloads
  • 3181fe8 add some migration docs
  • aaaac7b licensed cache
  • 7c9182f update readme
  • b94e701 licensed cache
  • 0b55470 add test case for globbed downloads to same directory
  • 0b51c2e update prettier/eslint versions
  • c4c6db7 support globbing artifact list & merging download directory
  • 1bd0606 Merge pull request #252 from stchr/patch-1
  • eff4d42 fix default for run-id
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=3&new-version=4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Li --- .github/workflows/native-unix.yml | 12 ++++---- .github/workflows/native-windows.yml | 6 ++-- .github/workflows/nightly-verify.yml | 4 +-- .github/workflows/nightly-website.yml | 4 +-- .github/workflows/packaging.yml | 44 +++++++++++++-------------- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/native-unix.yml b/.github/workflows/native-unix.yml index c498cb96c7..b1e87b0811 100644 --- a/.github/workflows/native-unix.yml +++ b/.github/workflows/native-unix.yml @@ -120,7 +120,7 @@ jobs: export PATH=$RUNNER_TOOL_CACHE/go/1.19.13/x64/bin:$PATH ./ci/scripts/go_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local" - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: driver-manager-${{ matrix.os }} retention-days: 3 @@ -268,7 +268,7 @@ jobs: --file ci/conda_env_cpp.txt \ --file ci/conda_env_glib.txt - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: driver-manager-${{ matrix.os }} path: ~/local @@ -368,7 +368,7 @@ jobs: if: ${{ !contains('macos-latest', matrix.os) }} run: go install honnef.co/go/tools/cmd/staticcheck@v0.3.3 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: driver-manager-${{ matrix.os }} path: ~/local @@ -444,7 +444,7 @@ jobs: cache: true cache-dependency-path: go/adbc/go.sum - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: driver-manager-${{ matrix.os }} path: ~/local @@ -557,7 +557,7 @@ jobs: python=${{ matrix.python }} \ --file ci/conda_env_docs.txt \ --file ci/conda_env_python.txt - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: driver-manager-${{ matrix.os }} path: ~/local @@ -572,7 +572,7 @@ jobs: run: | ./ci/scripts/docs_build.sh "$(pwd)" - name: Archive docs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: docs retention-days: 2 diff --git a/.github/workflows/native-windows.yml b/.github/workflows/native-windows.yml index acee122c6b..fc9760edc6 100644 --- a/.github/workflows/native-windows.yml +++ b/.github/workflows/native-windows.yml @@ -99,7 +99,7 @@ jobs: run: | .\ci\scripts\cpp_build.ps1 $pwd ${{ github.workspace }}\build - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: driver-manager-${{ matrix.os }} retention-days: 3 @@ -228,7 +228,7 @@ jobs: cache: true cache-dependency-path: go/adbc/go.sum - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: driver-manager-${{ matrix.os }} path: ${{ github.workspace }}/build @@ -292,7 +292,7 @@ jobs: --file ci/conda_env_cpp.txt \ --file ci/conda_env_python.txt - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: driver-manager-${{ matrix.os }} path: ${{ github.workspace }}/build diff --git a/.github/workflows/nightly-verify.yml b/.github/workflows/nightly-verify.yml index c4a3fdf63c..658b2efb0c 100644 --- a/.github/workflows/nightly-verify.yml +++ b/.github/workflows/nightly-verify.yml @@ -82,7 +82,7 @@ jobs: shasum --algorithm 512 \ apache-arrow-adbc-${VERSION}.tar.gz > apache-arrow-adbc-${VERSION}.tar.gz.sha512 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: source retention-days: 7 @@ -115,7 +115,7 @@ jobs: grep -E -o '[0-9]+\.[0-9]+\.[0-9]+') echo "VERSION=${VERSION}" >> $GITHUB_ENV - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: source path: ${{ github.workspace }}/apache-arrow-adbc-${{ env.VERSION }}-rc0/ diff --git a/.github/workflows/nightly-website.yml b/.github/workflows/nightly-website.yml index 428cfb73a5..192a02275d 100644 --- a/.github/workflows/nightly-website.yml +++ b/.github/workflows/nightly-website.yml @@ -45,7 +45,7 @@ jobs: run: | docker-compose run docs - name: Archive docs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: docs retention-days: 2 @@ -70,7 +70,7 @@ jobs: path: scripts persist-credentials: false - name: Download docs - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: docs path: temp diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 7d5d2ccffb..ca7ae623ff 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -110,7 +110,7 @@ jobs: apache-arrow-adbc-${{ steps.version.outputs.VERSION }} \ $(git log -n 1 --format=%h) - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: source retention-days: 7 @@ -123,7 +123,7 @@ jobs: needs: - source steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: source @@ -160,7 +160,7 @@ jobs: popd - name: Archive docs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: docs retention-days: 2 @@ -173,7 +173,7 @@ jobs: needs: - source steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: source @@ -201,7 +201,7 @@ jobs: cp -a adbc/dist/ ./ - name: Archive JARs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: java retention-days: 7 @@ -224,7 +224,7 @@ jobs: - debian-bullseye - ubuntu-jammy steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: source @@ -299,7 +299,7 @@ jobs: tar czf ${{ matrix.target }}.tar.gz ${DISTRIBUTION} - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.target }} retention-days: 7 @@ -353,7 +353,7 @@ jobs: # TODO: "linux_aarch64_" arch: ["linux_64_"] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: source @@ -386,7 +386,7 @@ jobs: popd - name: Archive Conda packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: python-${{ matrix.arch }}-conda retention-days: 7 @@ -420,7 +420,7 @@ jobs: run: shell: bash -l {0} steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: source @@ -462,7 +462,7 @@ jobs: ./adbc/ci/scripts/python_conda_build.sh $(pwd)/adbc ${ARCH_CONDA_FORGE}.yaml $(pwd)/adbc/build - name: Archive Conda packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: python-${{ matrix.arch }}-conda retention-days: 7 @@ -493,7 +493,7 @@ jobs: - arch: arm64v8 is_pr: true steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: source @@ -530,7 +530,7 @@ jobs: popd - name: Archive wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: python-${{ matrix.arch }}-manylinux${{ matrix.manylinux_version }} retention-days: 7 @@ -566,7 +566,7 @@ jobs: # Where to install vcpkg VCPKG_ROOT: "${{ github.workspace }}/vcpkg" steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: source @@ -631,7 +631,7 @@ jobs: popd - name: Archive wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: python-${{ matrix.arch }}-macos retention-days: 7 @@ -681,7 +681,7 @@ jobs: # Where to install vcpkg VCPKG_ROOT: "${{ github.workspace }}\\vcpkg" steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: source @@ -752,7 +752,7 @@ jobs: popd - name: Archive wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: python${{ matrix.python_version }}-windows retention-days: 7 @@ -779,7 +779,7 @@ jobs: needs: - source steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: source @@ -810,7 +810,7 @@ jobs: popd - name: Archive sdist - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: python-sdist retention-days: 7 @@ -842,7 +842,7 @@ jobs: - python-sdist steps: - name: Get All Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: release-artifacts - name: Release @@ -895,7 +895,7 @@ jobs: fetch-depth: 0 persist-credentials: true - name: Get All Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: conda-packages - uses: conda-incubator/setup-miniconda@v3 @@ -938,7 +938,7 @@ jobs: fetch-depth: 0 persist-credentials: true - name: Get All Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: nightly-artifacts - name: Upload From 877ff75f3d3b56b6e005856f32c8b663c3d1a8a8 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 2 Jan 2024 13:49:32 -0500 Subject: [PATCH 46/51] test(go/adbc/driver/snowflake): skip failing test (#1421) Fixes #1364. --- go/adbc/driver/snowflake/driver_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/adbc/driver/snowflake/driver_test.go b/go/adbc/driver/snowflake/driver_test.go index 6f8707a15c..3230fed540 100644 --- a/go/adbc/driver/snowflake/driver_test.go +++ b/go/adbc/driver/snowflake/driver_test.go @@ -871,6 +871,8 @@ func ConnectWithJwt(uri, keyValue, passcode string) { } func (suite *SnowflakeTests) TestJwtPrivateKey() { + suite.T().Skipf("apache/arrow-adbc#1364") + // grab the username from the DSN cfg, err := gosnowflake.ParseDSN(suite.Quirks.dsn) suite.NoError(err) From fa31305228fd872a104a23c6f11f696f90f6ab0b Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 2 Jan 2024 13:50:08 -0500 Subject: [PATCH 47/51] docs: add simple glossary (#1420) --- docs/source/glossary.rst | 55 ++++++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 8 ++++++ 2 files changed, 63 insertions(+) create mode 100644 docs/source/glossary.rst diff --git a/docs/source/glossary.rst b/docs/source/glossary.rst new file mode 100644 index 0000000000..f54c63b290 --- /dev/null +++ b/docs/source/glossary.rst @@ -0,0 +1,55 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +======== +Glossary +======== + +.. glossary:: + + client API + The API that an application uses to interact with a database. May + abstract over the underlying :term:`wire protocol` and other details. + For example, ADBC, JDBC, ODBC. + + driver + A library that implements a :term:`client API` using a :term:`wire + protocol`. For example, the ADBC PostgreSQL driver exposes the ADBC + client API and interacts with a PostgreSQL database via the PostgreSQL + wire protocol. The JDBC PostgreSQL driver uses the same wire protocol, + but exposes the JDBC client API instead. + + driver manager + A library that helps manage multiple drivers for a given client API. + For example, the JDBC driver manager can find a appropriate driver + implementation for a database URI. + + The ADBC driver manager in each language is similar. In C/C++, it can + dynamically load drivers so that applications do not have to directly + link to them. (Since all drivers expose the same API, their symbols + would collide otherwise.) In Python, it loads drivers and provides + Python bindings on top. + + wire protocol + The protocol that a database driver uses to interact with a database. + For example, :external:doc:`format/FlightSql`, the `PostgreSQL wire + protocol`_, or `Tabular Data Stream`_ (Microsoft SQL Server). Generally + not directly used by an application. + +.. _Arrow Flight SQL: https://arrow.apache.org/docs/format/FlightSql.html +.. _PostgreSQL wire protocol: https://www.postgresql.org/docs/current/protocol.html +.. _Tabular Data Stream: https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-tds/b46a581a-39de-4745-b076-ec4dbb7d13ec diff --git a/docs/source/index.rst b/docs/source/index.rst index 5537a94365..494ccfc711 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -126,6 +126,13 @@ make it more convenient to build analytical applications. FAQ + .. button-ref:: glossary + :ref-type: doc + :color: secondary + :expand: + + Glossary + .. grid-item-card:: :columns: 12 4 4 4 @@ -201,6 +208,7 @@ Why ADBC? :hidden: faq + glossary .. toctree:: :maxdepth: 1 From 650994d59e2c44971ff292d7bc880ea8aabd1125 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 2 Jan 2024 16:03:43 -0500 Subject: [PATCH 48/51] docs: update footer to match arrow.apache.org (#1423) --- docs/source/conf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 128dfc736d..c1b715e2d0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -25,7 +25,10 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information project = "ADBC" -copyright = "2022, Apache Arrow Developers" +copyright = """2022–2024 The Apache Software Foundation. Apache Arrow, Arrow, +Apache, the Apache feather logo, and the Apache Arrow project logo are either +registered trademarks or trademarks of The Apache Software Foundation in the +United States and other countries.""" author = "the Apache Arrow Developers" release = "0.9.0 (dev)" # Needed to generate version switcher From 2116cffffe264ccdbaecffeee9fd06c88f22bce5 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Wed, 3 Jan 2024 08:28:08 -0500 Subject: [PATCH 49/51] feat(c/driver/postgresql): Support for writing DECIMAL types (#1288) --- c/driver/postgresql/postgres_copy_reader.h | 160 ++++++++++++ .../postgresql/postgres_copy_reader_test.cc | 66 +++++ c/driver/postgresql/postgresql_benchmark.cc | 152 +++++++++++ c/driver/postgresql/postgresql_test.cc | 238 ++++++++++++++++++ c/driver/postgresql/statement.cc | 9 + c/validation/adbc_validation_util.h | 4 + 6 files changed, 629 insertions(+) diff --git a/c/driver/postgresql/postgres_copy_reader.h b/c/driver/postgresql/postgres_copy_reader.h index 686d54b81b..8a9192c329 100644 --- a/c/driver/postgresql/postgres_copy_reader.h +++ b/c/driver/postgresql/postgres_copy_reader.h @@ -1224,6 +1224,152 @@ class PostgresCopyIntervalFieldWriter : public PostgresCopyFieldWriter { } }; +// Inspiration for this taken from get_str_from_var in the pg source +// src/backend/utils/adt/numeric.c +template +class PostgresCopyNumericFieldWriter : public PostgresCopyFieldWriter { +public: + PostgresCopyNumericFieldWriter(int32_t precision, int32_t scale) : + precision_{precision}, scale_{scale} {} + + ArrowErrorCode Write(ArrowBuffer* buffer, int64_t index, ArrowError* error) override { + struct ArrowDecimal decimal; + ArrowDecimalInit(&decimal, bitwidth_, precision_, scale_); + ArrowArrayViewGetDecimalUnsafe(array_view_, index, &decimal); + + const int16_t sign = ArrowDecimalSign(&decimal) > 0 ? kNumericPos : kNumericNeg; + + // Number of decimal digits per Postgres digit + constexpr int kDecDigits = 4; + std::vector pg_digits; + int16_t weight = -(scale_ / kDecDigits); + int16_t dscale = scale_; + bool seen_decimal = scale_ == 0; + bool truncating_trailing_zeros = true; + + char decimal_string[max_decimal_digits_ + 1]; + int digits_remaining = DecimalToString(&decimal, decimal_string); + do { + const int start_pos = digits_remaining < kDecDigits ? + 0 : digits_remaining - kDecDigits; + const size_t len = digits_remaining < 4 ? digits_remaining : kDecDigits; + char substr[kDecDigits + 1]; + std::memcpy(substr, decimal_string + start_pos, len); + substr[len] = '\0'; + int16_t val = static_cast(std::atoi(substr)); + + if (val == 0) { + if (!seen_decimal && truncating_trailing_zeros) { + dscale -= kDecDigits; + } + } else { + pg_digits.insert(pg_digits.begin(), val); + if (!seen_decimal && truncating_trailing_zeros) { + if (val % 1000 == 0) { + dscale -= 3; + } else if (val % 100 == 0) { + dscale -= 2; + } else if (val % 10 == 0) { + dscale -= 1; + } + } + truncating_trailing_zeros = false; + } + digits_remaining -= kDecDigits; + if (digits_remaining <= 0) { + break; + } + weight++; + + if (start_pos <= static_cast(std::strlen(decimal_string)) - scale_) { + seen_decimal = true; + } + } while (true); + + int16_t ndigits = pg_digits.size(); + int32_t field_size_bytes = sizeof(ndigits) + + sizeof(weight) + + sizeof(sign) + + sizeof(dscale) + + ndigits * sizeof(int16_t); + + NANOARROW_RETURN_NOT_OK(WriteChecked(buffer, field_size_bytes, error)); + NANOARROW_RETURN_NOT_OK(WriteChecked(buffer, ndigits, error)); + NANOARROW_RETURN_NOT_OK(WriteChecked(buffer, weight, error)); + NANOARROW_RETURN_NOT_OK(WriteChecked(buffer, sign, error)); + NANOARROW_RETURN_NOT_OK(WriteChecked(buffer, dscale, error)); + + const size_t pg_digit_bytes = sizeof(int16_t) * pg_digits.size(); + NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, pg_digit_bytes)); + for (auto pg_digit : pg_digits) { + WriteUnsafe(buffer, pg_digit); + } + + return ADBC_STATUS_OK; + } + +private: + // returns the length of the string + template + int DecimalToString(struct ArrowDecimal* decimal, char* out) { + constexpr size_t nwords = (DEC_WIDTH == 128) ? 2 : 4; + uint8_t tmp[DEC_WIDTH / 8]; + ArrowDecimalGetBytes(decimal, tmp); + uint64_t buf[DEC_WIDTH / 64]; + std::memcpy(buf, tmp, sizeof(buf)); + const int16_t sign = ArrowDecimalSign(decimal) > 0 ? kNumericPos : kNumericNeg; + const bool is_negative = sign == kNumericNeg ? true : false; + if (is_negative) { + buf[0] = ~buf[0] + 1; + for (size_t i = 1; i < nwords; i++) { + buf[i] = ~buf[i]; + } + } + + // Basic approach adopted from https://stackoverflow.com/a/8023862/621736 + char s[max_decimal_digits_ + 1]; + std::memset(s, '0', sizeof(s) - 1); + s[sizeof(s) - 1] = '\0'; + + for (size_t i = 0; i < DEC_WIDTH; i++) { + int carry; + + carry = (buf[nwords - 1] >= 0x7FFFFFFFFFFFFFFF); + for (size_t j = nwords - 1; j > 0; j--) { + buf[j] = ((buf[j] << 1) & 0xFFFFFFFFFFFFFFFF) + (buf[j-1] >= 0x7FFFFFFFFFFFFFFF); + } + buf[0] = ((buf[0] << 1) & 0xFFFFFFFFFFFFFFFF); + + for (int j = sizeof(s) - 2; j>= 0; j--) { + s[j] += s[j] - '0' + carry; + carry = (s[j] > '9'); + if (carry) { + s[j] -= 10; + } + } + } + + char* p = s; + while ((p[0] == '0') && (p < &s[sizeof(s) - 2])) { + p++; + } + + const size_t ndigits = sizeof(s) - 1 - (p - s); + std::memcpy(out, p, ndigits); + out[ndigits] = '\0'; + + return ndigits; + } + + static constexpr uint16_t kNumericPos = 0x0000; + static constexpr uint16_t kNumericNeg = 0x4000; + static constexpr int32_t bitwidth_ = (T == NANOARROW_TYPE_DECIMAL128) ? 128 : 256; + static constexpr size_t max_decimal_digits_ = + (T == NANOARROW_TYPE_DECIMAL128) ? 39 : 78; + const int32_t precision_; + const int32_t scale_; +}; + template class PostgresCopyDurationFieldWriter : public PostgresCopyFieldWriter { public: @@ -1392,6 +1538,20 @@ static inline ArrowErrorCode MakeCopyFieldWriter(struct ArrowSchema* schema, case NANOARROW_TYPE_DOUBLE: *out = new PostgresCopyDoubleFieldWriter(); return NANOARROW_OK; + case NANOARROW_TYPE_DECIMAL128: { + const auto precision = schema_view.decimal_precision; + const auto scale = schema_view.decimal_scale; + *out = new PostgresCopyNumericFieldWriter< + NANOARROW_TYPE_DECIMAL128>(precision, scale); + return NANOARROW_OK; + } + case NANOARROW_TYPE_DECIMAL256: { + const auto precision = schema_view.decimal_precision; + const auto scale = schema_view.decimal_scale; + *out = new PostgresCopyNumericFieldWriter< + NANOARROW_TYPE_DECIMAL256>(precision, scale); + return NANOARROW_OK; + } case NANOARROW_TYPE_BINARY: case NANOARROW_TYPE_STRING: case NANOARROW_TYPE_LARGE_STRING: diff --git a/c/driver/postgresql/postgres_copy_reader_test.cc b/c/driver/postgresql/postgres_copy_reader_test.cc index 7882d602af..201aa223a2 100644 --- a/c/driver/postgresql/postgres_copy_reader_test.cc +++ b/c/driver/postgresql/postgres_copy_reader_test.cc @@ -693,6 +693,72 @@ TEST(PostgresCopyUtilsTest, PostgresCopyReadNumeric) { EXPECT_EQ(std::string(item.data, item.size_bytes), "inf"); } +// This buffer is similar to the read variant above but removes special values +// nan, ±inf as they are not supported via the Arrow Decimal types +// COPY (SELECT CAST(col AS NUMERIC) AS col FROM ( VALUES (NULL), (-123.456), +// ('0.00001234'), (1.0000), (123.456), (1000000)) AS drvd(col)) +// TO STDOUT WITH (FORMAT binary); +static uint8_t kTestPgCopyNumericWrite[] = { + 0x50, 0x47, 0x43, 0x4f, 0x50, 0x59, 0x0a, 0xff, 0x0d, 0x0a, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x0c, 0x00, 0x02, 0x00, 0x00, 0x40, 0x00, 0x00, 0x03, 0x00, 0x7b, 0x11, + 0xd0, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x01, 0xff, 0xfe, 0x00, 0x00, 0x00, + 0x08, 0x04, 0xd2, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x7b, 0x11, 0xd0, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0a, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0xff, 0xff}; + +TEST(PostgresCopyUtilsTest, PostgresCopyWriteNumeric) { + adbc_validation::Handle schema; + adbc_validation::Handle array; + struct ArrowError na_error; + constexpr enum ArrowType type = NANOARROW_TYPE_DECIMAL128; + constexpr int32_t size = 128; + constexpr int32_t precision = 38; + constexpr int32_t scale = 8; + + struct ArrowDecimal decimal1; + struct ArrowDecimal decimal2; + struct ArrowDecimal decimal3; + struct ArrowDecimal decimal4; + struct ArrowDecimal decimal5; + + ArrowDecimalInit(&decimal1, size, 19, 8); + ArrowDecimalSetInt(&decimal1, -12345600000); + ArrowDecimalInit(&decimal2, size, 19, 8); + ArrowDecimalSetInt(&decimal2, 1234); + ArrowDecimalInit(&decimal3, size, 19, 8); + ArrowDecimalSetInt(&decimal3, 100000000); + ArrowDecimalInit(&decimal4, size, 19, 8); + ArrowDecimalSetInt(&decimal4, 12345600000); + ArrowDecimalInit(&decimal5, size, 19, 8); + ArrowDecimalSetInt(&decimal5, 100000000000000); + + const std::vector> values = { + std::nullopt, &decimal1, &decimal2, &decimal3, &decimal4, &decimal5}; + + ArrowSchemaInit(&schema.value); + ASSERT_EQ(ArrowSchemaSetTypeStruct(&schema.value, 1), 0); + ASSERT_EQ(AdbcNsArrowSchemaSetTypeDecimal(schema.value.children[0], + type, precision, scale), 0); + ASSERT_EQ(ArrowSchemaSetName(schema.value.children[0], "col"), 0); + ASSERT_EQ(adbc_validation::MakeBatch(&schema.value, &array.value, + &na_error, values), ADBC_STATUS_OK); + + PostgresCopyStreamWriteTester tester; + ASSERT_EQ(tester.Init(&schema.value, &array.value), NANOARROW_OK); + ASSERT_EQ(tester.WriteAll(nullptr), ENODATA); + + const struct ArrowBuffer buf = tester.WriteBuffer(); + // The last 2 bytes of a message can be transmitted via PQputCopyData + // so no need to test those bytes from the Writer + constexpr size_t buf_size = sizeof(kTestPgCopyNumericWrite) - 2; + ASSERT_EQ(buf.size_bytes, buf_size); + for (size_t i = 0; i < buf_size; i++) { + ASSERT_EQ(buf.data[i], kTestPgCopyNumericWrite[i]) << " at position " << i; + } +} + // COPY (SELECT CAST(col AS TIMESTAMP) FROM ( VALUES ('1900-01-01 12:34:56'), // ('2100-01-01 12:34:56'), (NULL)) AS drvd("col")) TO STDOUT WITH (FORMAT BINARY); static uint8_t kTestPgCopyTimestamp[] = { diff --git a/c/driver/postgresql/postgresql_benchmark.cc b/c/driver/postgresql/postgresql_benchmark.cc index 239575a7d7..908269966e 100644 --- a/c/driver/postgresql/postgresql_benchmark.cc +++ b/c/driver/postgresql/postgresql_benchmark.cc @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include @@ -175,5 +176,156 @@ static void BM_PostgresqlExecute(benchmark::State& state) { &error)); } +static void BM_PostgresqlDecimalWrite(benchmark::State& state) { + const char* uri = std::getenv("ADBC_POSTGRESQL_TEST_URI"); + if (!uri || !strcmp(uri, "")) { + state.SkipWithError("ADBC_POSTGRESQL_TEST_URI not set!"); + return; + } + adbc_validation::Handle database; + struct AdbcError error; + + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcDatabaseNew(&database.value, &error)); + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcDatabaseSetOption(&database.value, + "uri", + uri, + &error)); + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcDatabaseInit(&database.value, &error)); + + adbc_validation::Handle connection; + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcConnectionNew(&connection.value, &error)); + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcConnectionInit(&connection.value, + &database.value, + &error)); + + adbc_validation::Handle statement; + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcStatementNew(&connection.value, + &statement.value, + &error)); + + const char* drop_query = "DROP TABLE IF EXISTS adbc_postgresql_ingest_benchmark"; + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcStatementSetSqlQuery(&statement.value, + drop_query, + &error)); + + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcStatementExecuteQuery(&statement.value, + nullptr, + nullptr, + &error)); + + adbc_validation::Handle schema; + adbc_validation::Handle array; + struct ArrowError na_error; + + constexpr enum ArrowType type = NANOARROW_TYPE_DECIMAL128; + constexpr int32_t bitwidth = 128; + constexpr int32_t precision = 38; + constexpr int32_t scale = 8; + constexpr size_t ncols = 5; + ArrowSchemaInit(&schema.value); + if (ArrowSchemaSetTypeStruct(&schema.value, ncols) != NANOARROW_OK) { + state.SkipWithError("Call to ArrowSchemaSetTypeStruct failed!"); + error.release(&error); + return; + } + + for (size_t i = 0; i < ncols; i++) { + if (AdbcNsArrowSchemaSetTypeDecimal(schema.value.children[i], + type, precision, scale) != NANOARROW_OK) { + state.SkipWithError("Call to ArrowSchemaSetTypeDecimal failed!"); + error.release(&error); + return; + } + + std::string colname = "col" + std::to_string(i); + if (ArrowSchemaSetName(schema.value.children[i], colname.c_str()) != NANOARROW_OK) { + state.SkipWithError("Call to ArrowSchemaSetName failed!"); + error.release(&error); + return; + } + } + if (ArrowArrayInitFromSchema(&array.value, &schema.value, &na_error) != NANOARROW_OK) { + state.SkipWithError("Call to ArrowArrayInitFromSchema failed!"); + error.release(&error); + return; + } + + if (ArrowArrayStartAppending(&array.value) != NANOARROW_OK) { + state.SkipWithError("Call to ArrowArrayStartAppending failed!"); + error.release(&error); + return; + } + + constexpr size_t nrows = 1000; + struct ArrowDecimal decimal; + ArrowDecimalInit(&decimal, bitwidth, precision, scale); + for (size_t i = 0; i < nrows; i++) { + for (size_t j = 0; j < ncols; j++) { + ArrowDecimalSetInt(&decimal, i + j); + if (ArrowArrayAppendDecimal(array.value.children[j], &decimal) != NANOARROW_OK) { + state.SkipWithError("Call to ArrowArrayAppendDecimal failed"); + error.release(&error); + return; + } + } + } + + for (int64_t i = 0; i < array.value.n_children; i++) { + array.value.children[i]->length = nrows; + } + array.value.length = nrows; + + if (ArrowArrayFinishBuildingDefault(&array.value, &na_error) != NANOARROW_OK) { + state.SkipWithError("Call to ArrowArrayFinishBuildingDefault failed"); + error.release(&error); + return; + } + + const char* create_query = + "CREATE TABLE adbc_postgresql_ingest_benchmark (col0 DECIMAL(38, 8), " + "col1 DECIMAL(38, 8), col2 DECIMAL(38, 8), col3 DECIMAL(38, 8), col4 DECIMAL(38, 8))"; + + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcStatementSetSqlQuery(&statement.value, + create_query, + &error)); + + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcStatementExecuteQuery(&statement.value, + nullptr, + nullptr, + &error)); + + adbc_validation::Handle insert_stmt; + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcStatementNew(&connection.value, + &insert_stmt.value, + &error)); + + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcStatementSetOption(&insert_stmt.value, + ADBC_INGEST_OPTION_TARGET_TABLE, + "adbc_postgresql_ingest_benchmark", + &error)); + + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcStatementSetOption(&insert_stmt.value, + ADBC_INGEST_OPTION_MODE, + ADBC_INGEST_OPTION_MODE_APPEND, + &error)); + + for (auto _ : state) { + AdbcStatementBind(&insert_stmt.value, &array.value, &schema.value, &error); + AdbcStatementExecuteQuery(&insert_stmt.value, nullptr, nullptr, &error); + } + + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcStatementSetSqlQuery(&statement.value, + drop_query, + &error)); + + ADBC_BENCHMARK_RETURN_NOT_OK(AdbcStatementExecuteQuery(&statement.value, + nullptr, + nullptr, + &error)); +} + +// TODO: we are limited to only 1 iteration as AdbcStatementBind is part of +// the benchmark loop, but releases the array when it is done BENCHMARK(BM_PostgresqlExecute)->Iterations(1); +BENCHMARK(BM_PostgresqlDecimalWrite)->Iterations(1); BENCHMARK_MAIN(); diff --git a/c/driver/postgresql/postgresql_test.cc b/c/driver/postgresql/postgresql_test.cc index 2327767a4e..8ac841d8a0 100644 --- a/c/driver/postgresql/postgresql_test.cc +++ b/c/driver/postgresql/postgresql_test.cc @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include @@ -119,6 +120,9 @@ class PostgresQuirks : public adbc_validation::DriverQuirks { return NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO; case NANOARROW_TYPE_LARGE_STRING: return NANOARROW_TYPE_STRING; + case NANOARROW_TYPE_DECIMAL128: + case NANOARROW_TYPE_DECIMAL256: + return NANOARROW_TYPE_STRING; default: return ingest_type; } @@ -1645,3 +1649,237 @@ INSTANTIATE_TEST_SUITE_P(TimeTypes, PostgresTypeTest, testing::ValuesIn(kTimeTyp INSTANTIATE_TEST_SUITE_P(TimestampTypes, PostgresTypeTest, testing::ValuesIn(kTimestampTypeCases), TypeTestCase::FormatName); + +struct DecimalTestCase { + const enum ArrowType type; + const int32_t precision; + const int32_t scale; + const std::vector> data; + const std::vector> expected; +}; + +class PostgresDecimalTest : public ::testing::TestWithParam { +public: + void SetUp() override { + ASSERT_THAT(AdbcDatabaseNew(&database_, &error_), IsOkStatus(&error_)); + ASSERT_THAT(quirks_.SetupDatabase(&database_, &error_), IsOkStatus(&error_)); + ASSERT_THAT(AdbcDatabaseInit(&database_, &error_), IsOkStatus(&error_)); + + ASSERT_THAT(AdbcConnectionNew(&connection_, &error_), IsOkStatus(&error_)); + ASSERT_THAT(AdbcConnectionInit(&connection_, &database_, &error_), + IsOkStatus(&error_)); + + ASSERT_THAT(AdbcStatementNew(&connection_, &statement_, &error_), + IsOkStatus(&error_)); + + ASSERT_THAT(quirks_.DropTable(&connection_, "bulk_ingest", &error_), + IsOkStatus(&error_)); + } + + void TearDown() override { + if (statement_.private_data) { + ASSERT_THAT(AdbcStatementRelease(&statement_, &error_), IsOkStatus(&error_)); + } + if (connection_.private_data) { + ASSERT_THAT(AdbcConnectionRelease(&connection_, &error_), IsOkStatus(&error_)); + } + if (database_.private_data) { + ASSERT_THAT(AdbcDatabaseRelease(&database_, &error_), IsOkStatus(&error_)); + } + + if (error_.release) error_.release(&error_); + } + +protected: + PostgresQuirks quirks_; + struct AdbcError error_ = {}; + struct AdbcDatabase database_ = {}; + struct AdbcConnection connection_ = {}; + struct AdbcStatement statement_ = {}; +}; + +TEST_P(PostgresDecimalTest, SelectValue) { + adbc_validation::Handle schema; + adbc_validation::Handle array; + struct ArrowError na_error; + + const enum ArrowType type = GetParam().type; + const int32_t precision = GetParam().precision; + const int32_t scale = GetParam().scale; + const auto data = GetParam().data; + const auto expected = GetParam().expected; + const size_t nrecords = expected.size(); + + int32_t bitwidth; + switch (type) { + case NANOARROW_TYPE_DECIMAL128: + bitwidth = 128; + break; + case NANOARROW_TYPE_DECIMAL256: + bitwidth = 256; + break; + default: + FAIL(); + } + + // this is a bit of a hack to make std::vector play nicely with + // a dynamic number of stack-allocated ArrowDecimal objects + constexpr size_t max_decimals = 10; + struct ArrowDecimal decimals[max_decimals]; + if (nrecords > max_decimals) { + FAIL() << + " max_decimals exceeded for test case - please change parametrization"; + } + + std::vector> values; + for (size_t i = 0; i < nrecords; i++) { + ArrowDecimalInit(&decimals[i], bitwidth, precision, scale); + uint8_t buf[32]; + const auto record = data[i]; + memcpy(buf, record.data(), sizeof(buf)); + ArrowDecimalSetBytes(&decimals[i], buf); + values.push_back(&decimals[i]); + } + + auto expected_with_null{expected}; + expected_with_null.insert(expected_with_null.begin(), std::nullopt); + values.push_back(std::nullopt); + + ArrowSchemaInit(&schema.value); + ASSERT_EQ(ArrowSchemaSetTypeStruct(&schema.value, 1), 0); + ASSERT_EQ(AdbcNsArrowSchemaSetTypeDecimal(schema.value.children[0], + type, precision, scale), 0); + ASSERT_EQ(ArrowSchemaSetName(schema.value.children[0], "col"), 0); + + ASSERT_THAT(adbc_validation::MakeBatch(&schema.value, &array.value, + &na_error, values), + adbc_validation::IsOkErrno()); + + ASSERT_THAT(AdbcStatementSetOption(&statement_, + ADBC_INGEST_OPTION_TARGET_TABLE, + "bulk_ingest", &error_), + IsOkStatus(&error_)); + ASSERT_THAT(AdbcStatementBind(&statement_, &array.value, &schema.value, &error_), + IsOkStatus(&error_)); + + int64_t rows_affected = 0; + ASSERT_THAT(AdbcStatementExecuteQuery(&statement_, nullptr, &rows_affected, &error_), + IsOkStatus(&error_)); + ASSERT_THAT(rows_affected, + ::testing::AnyOf(::testing::Eq(values.size()), ::testing::Eq(-1))); + + ASSERT_THAT(AdbcStatementSetSqlQuery( + &statement_, + "SELECT * FROM bulk_ingest ORDER BY \"col\" ASC NULLS FIRST", &error_), + IsOkStatus(&error_)); + + { + adbc_validation::StreamReader reader; + ASSERT_THAT(AdbcStatementExecuteQuery(&statement_, &reader.stream.value, + &reader.rows_affected, &error_), + IsOkStatus(&error_)); + ASSERT_THAT(reader.rows_affected, + ::testing::AnyOf(::testing::Eq(values.size()), ::testing::Eq(-1))); + + ASSERT_NO_FATAL_FAILURE(reader.GetSchema()); + ArrowType round_trip_type = quirks_.IngestSelectRoundTripType(type); + ASSERT_NO_FATAL_FAILURE(adbc_validation::CompareSchema(&reader.schema.value, + {{"col", + round_trip_type, true}})); + + ASSERT_NO_FATAL_FAILURE(reader.Next()); + ASSERT_NE(nullptr, reader.array->release); + ASSERT_EQ(values.size(), reader.array->length); + ASSERT_EQ(1, reader.array->n_children); + + ASSERT_NO_FATAL_FAILURE(adbc_validation::CompareArray< + std::string>(reader.array_view->children[0], + expected_with_null)); + + ASSERT_NO_FATAL_FAILURE(reader.Next()); + ASSERT_EQ(nullptr, reader.array->release); + } +} + +static std::vector> kDecimalData = { + // -12345600000 + {18446744061363951616ULL, 18446744073709551615ULL, 0, 0}, + // 1234 + {1234ULL, 0, 0, 0}, + // 100000000 + {100000000ULL, 0, 0, 0}, + // 12345600000 + {12345600000ULL, 0, 0, 0}, + // 100000000000000 + {100000000000000ULL, 0, 0, 0}, + // 2342394230592232349023094 + {8221368519775271798ULL, 126981ULL, 0, 0}, +}; + +static std::vector> kDecimal256Data = { + // 1234567890123456789012345678901234567890123456789012345678901234567890123456 + {17877984925544397504ULL, 5352188884907840935ULL, 234631617561833724ULL, + 196678011949953713ULL}, + // -1234567890123456789012345678901234567890123456789012345678901234567890123456 + {568759148165154112ULL, 13094555188801710680ULL, 18212112456147717891ULL, + 18250066061759597902ULL}, +}; + +static std::initializer_list kDecimal128Cases = { + { + NANOARROW_TYPE_DECIMAL128, 38, 8, kDecimalData, + {"-123.456", "0.00001234", "1", "123.456", "1000000", + "23423942305922323.49023094"} + }}; + +static std::initializer_list kDecimal128NoScaleCases = { + { + NANOARROW_TYPE_DECIMAL128, 38, 0, kDecimalData, + {"-12345600000", "1234", "100000000", "12345600000", "100000000000000", + "2342394230592232349023094"} + }}; + +static std::initializer_list kDecimal256Cases = { + { + NANOARROW_TYPE_DECIMAL256, 38, 8, kDecimalData, + {"-123.456", "0.00001234", "1", "123.456", "1000000", + "23423942305922323.49023094"} + }}; + +static std::initializer_list kDecimal256NoScaleCases = { + { + NANOARROW_TYPE_DECIMAL256, 38, 0, kDecimalData, + {"-12345600000", "1234", "100000000", "12345600000", "100000000000000", + "2342394230592232349023094"} + }}; + +static std::initializer_list kDecimal256LargeCases = { + { + NANOARROW_TYPE_DECIMAL256, 76, 8, kDecimal256Data, + { + "-12345678901234567890123456789012345678901234567890123456789012345678.90123456", + "12345678901234567890123456789012345678901234567890123456789012345678.90123456", + } + }}; + +static std::initializer_list kDecimal256LargeNoScaleCases = { + { + NANOARROW_TYPE_DECIMAL256, 76, 0, kDecimal256Data, + { + "-1234567890123456789012345678901234567890123456789012345678901234567890123456", + "1234567890123456789012345678901234567890123456789012345678901234567890123456", + } + }}; + +INSTANTIATE_TEST_SUITE_P(Decimal128Tests, PostgresDecimalTest, + testing::ValuesIn(kDecimal128Cases)); +INSTANTIATE_TEST_SUITE_P(Decimal128NoScale, PostgresDecimalTest, + testing::ValuesIn(kDecimal128NoScaleCases)); +INSTANTIATE_TEST_SUITE_P(Decimal256Tests, PostgresDecimalTest, + testing::ValuesIn(kDecimal128Cases)); +INSTANTIATE_TEST_SUITE_P(Decimal256NoScale, PostgresDecimalTest, + testing::ValuesIn(kDecimal128NoScaleCases)); +INSTANTIATE_TEST_SUITE_P(Decimal256LargeTests, PostgresDecimalTest, + testing::ValuesIn(kDecimal256LargeCases)); +INSTANTIATE_TEST_SUITE_P(Decimal256LargeNoScale, PostgresDecimalTest, + testing::ValuesIn(kDecimal256LargeNoScaleCases)); diff --git a/c/driver/postgresql/statement.cc b/c/driver/postgresql/statement.cc index 811a086a4f..68fd45a944 100644 --- a/c/driver/postgresql/statement.cc +++ b/c/driver/postgresql/statement.cc @@ -214,6 +214,11 @@ struct BindStream { type_id = PostgresTypeId::kInterval; param_lengths[i] = 16; break; + case ArrowType::NANOARROW_TYPE_DECIMAL128: + case ArrowType::NANOARROW_TYPE_DECIMAL256: + type_id = PostgresTypeId::kNumeric; + param_lengths[i] = 0; + break; case ArrowType::NANOARROW_TYPE_DICTIONARY: { struct ArrowSchemaView value_view; CHECK_NA(INTERNAL, @@ -1062,6 +1067,10 @@ AdbcStatusCode PostgresStatement::CreateBulkTable( case ArrowType::NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO: create += " INTERVAL"; break; + case ArrowType::NANOARROW_TYPE_DECIMAL128: + case ArrowType::NANOARROW_TYPE_DECIMAL256: + create += " DECIMAL"; + break; case ArrowType::NANOARROW_TYPE_DICTIONARY: { struct ArrowSchemaView value_view; CHECK_NA(INTERNAL, diff --git a/c/validation/adbc_validation_util.h b/c/validation/adbc_validation_util.h index da71e0d9e8..321b10f9d8 100644 --- a/c/validation/adbc_validation_util.h +++ b/c/validation/adbc_validation_util.h @@ -283,6 +283,10 @@ int MakeArray(struct ArrowArray* parent, struct ArrowArray* array, if (int errno_res = ArrowArrayAppendInterval(array, *v); errno_res != 0) { return errno_res; } + } else if constexpr (std::is_same::value) { + if (int errno_res = ArrowArrayAppendDecimal(array, *v); errno_res != 0) { + return errno_res; + } } else { static_assert(!sizeof(T), "Not yet implemented"); return ENOTSUP; From 76ab8cb675e9c11afa068522a01d07fb87847876 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 3 Jan 2024 08:29:23 -0500 Subject: [PATCH 50/51] python: switch to static versioning for simplicity (#1419) We tried to derive versions from Git. But since we release off of a branch, this meant that all non-branch builds would have version 0.0.0. This breaks things like Polars that rely on checking the installed version. Instead, since we already bump versions as part of the release process, just hardcode the version and drop most of miniver. Fixes #1363. --- .github/workflows/integration.yml | 1 - dev/release/utils-prepare.sh | 6 + .../adbc_driver_flightsql/_static_version.py | 6 +- .../adbc_driver_flightsql/_version.py | 172 +----------------- python/adbc_driver_flightsql/setup.py | 7 +- .../adbc_driver_manager/_static_version.py | 6 +- .../adbc_driver_manager/_version.py | 172 +----------------- python/adbc_driver_manager/setup.py | 7 +- .../adbc_driver_postgresql/_static_version.py | 6 +- .../adbc_driver_postgresql/_version.py | 172 +----------------- python/adbc_driver_postgresql/setup.py | 7 +- .../adbc_driver_snowflake/_static_version.py | 6 +- .../adbc_driver_snowflake/_version.py | 172 +----------------- python/adbc_driver_snowflake/setup.py | 7 +- .../adbc_driver_sqlite/_static_version.py | 6 +- .../adbc_driver_sqlite/_version.py | 172 +----------------- python/adbc_driver_sqlite/setup.py | 7 +- 17 files changed, 36 insertions(+), 896 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bd6d79d16f..300887fa3f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -229,7 +229,6 @@ jobs: BUILD_ALL: "0" BUILD_DRIVER_MANAGER: "1" BUILD_DRIVER_POSTGRESQL: "1" - SETUPTOOLS_SCM_PRETEND_VERSION: "0.9.0" run: | ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh index 2cf5b557e5..5eb08d81cc 100644 --- a/dev/release/utils-prepare.sh +++ b/dev/release/utils-prepare.sh @@ -27,12 +27,14 @@ update_versions() { local version=${base_version} local conda_version=${base_version} local docs_version=${base_version} + local py_version=${base_version} local r_version=${base_version} ;; snapshot) local version=${next_version}-SNAPSHOT local conda_version=${next_version} local docs_version="${next_version} (dev)" + local py_version="${next_version}dev" local r_version="${base_version}.9000" ;; esac @@ -70,6 +72,10 @@ update_versions() { rm "${ADBC_DIR}/glib/meson.build.bak" git add "${ADBC_DIR}/glib/meson.build" + sed -i.bak -E "s/version = \".+\"/version = \"${py_version}\"/g" "${ADBC_DIR}"/python/adbc_*/adbc_*/_static_version.py + rm "${ADBC_DIR}"/python/adbc_*/adbc_*/_static_version.py.bak + git add "${ADBC_DIR}"/python/adbc_*/adbc_*/_static_version.py + sed -i.bak -E "s/VERSION = \".+\"/VERSION = \"${version}\"/g" "${ADBC_DIR}/ruby/lib/adbc/version.rb" rm "${ADBC_DIR}/ruby/lib/adbc/version.rb.bak" git add "${ADBC_DIR}/ruby/lib/adbc/version.rb" diff --git a/python/adbc_driver_flightsql/adbc_driver_flightsql/_static_version.py b/python/adbc_driver_flightsql/adbc_driver_flightsql/_static_version.py index 4c31c3b721..bd1f923e3b 100644 --- a/python/adbc_driver_flightsql/adbc_driver_flightsql/_static_version.py +++ b/python/adbc_driver_flightsql/adbc_driver_flightsql/_static_version.py @@ -19,12 +19,8 @@ # Generated by miniver (CC0). # This file is part of 'miniver': https://github.com/jbweston/miniver -# -# This file will be overwritten by setup.py when a source or binary -# distribution is made. The magic value "__use_git__" is interpreted by -# _version.py. -version = "__use_git__" +version = "0.9.0dev" # These values are only set if the distribution was created with 'git archive' # NOTE: must add an export-subst to .gitattributes! diff --git a/python/adbc_driver_flightsql/adbc_driver_flightsql/_version.py b/python/adbc_driver_flightsql/adbc_driver_flightsql/_version.py index 8eadf0e6c1..5483d6d65f 100644 --- a/python/adbc_driver_flightsql/adbc_driver_flightsql/_version.py +++ b/python/adbc_driver_flightsql/adbc_driver_flightsql/_version.py @@ -17,15 +17,9 @@ # under the License. # Generated by miniver (CC0). - -import os -import re - # This file is part of 'miniver': https://github.com/jbweston/miniver -# -from collections import namedtuple -Version = namedtuple("Version", ("release", "dev", "labels")) +import os # No public API __all__ = [] @@ -34,9 +28,6 @@ package_name = os.path.basename(package_root) STATIC_VERSION_FILE = "_static_version.py" -TAG_RELEASE_FORMAT = re.compile( - r"^apache-arrow-adbc-([0-9]+\.[0-9]+\.[0-9]+)(?:-rc[0-9]+)?$" -) def get_version(version_file=STATIC_VERSION_FILE): @@ -45,15 +36,7 @@ def get_version(version_file=STATIC_VERSION_FILE): return override version_info = get_static_version_info(version_file) version = version_info["version"] - if version == "__use_git__": - version = get_version_from_git() - if not version: - version = get_version_from_git_archive(version_info) - if not version: - version = Version("unknown", None, None) - return pep440_format(version) - else: - return version + return version def get_static_version_info(version_file=STATIC_VERSION_FILE): @@ -63,158 +46,7 @@ def get_static_version_info(version_file=STATIC_VERSION_FILE): return version_info -def version_is_from_git(version_file=STATIC_VERSION_FILE): - return get_static_version_info(version_file)["version"] == "__use_git__" - - -def pep440_format(version_info): - release, dev, labels = version_info - - version_parts = [release] - if dev: - if release.endswith("-dev") or release.endswith(".dev"): - version_parts.append(dev) - else: # prefer PEP440 over strict adhesion to semver - version_parts.append(".dev{}".format(dev)) - - if labels: - version_parts.append("+") - version_parts.append(".".join(labels)) - - return "".join(version_parts) - - -def get_version_from_git(): - import subprocess - - # git describe --first-parent does not take into account tags from branches - # that were merged-in. The '--long' flag gets us the 'dev' version and - # git hash, '--always' returns the git hash even if there are no tags. - for opts in [["--first-parent"], []]: - try: - p = subprocess.Popen( - ["git", "describe", "--long", "--always"] + opts, - cwd=package_root, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - except OSError: - return - if p.wait() == 0: - break - else: - return - - description = ( - p.communicate()[0] - .decode() - .strip("v") # Tags can have a leading 'v', but the version should not - .rstrip("\n") - .rsplit("-", 2) # Split the latest tag, commits since tag, and hash - ) - - try: - release, dev, git = description - release = TAG_RELEASE_FORMAT.match(release).group(1) - except (AttributeError, ValueError): - # Invalid tag; or no tags, only the git hash - # prepend 'g' to match with format returned by 'git describe' - git = "g{}".format(*description) - # XXX: assume version if not given - release = "0.0.0" - dev = None - - labels = [] - if dev == "0": - dev = None - else: - labels.append(git) - - try: - p = subprocess.Popen(["git", "diff", "--quiet"], cwd=package_root) - except OSError: - labels.append("confused") # This should never happen. - else: - if p.wait() == 1: - labels.append("dirty") - - return Version(release, dev, labels) - - -# TODO: change this logic when there is a git pretty-format -# that gives the same output as 'git describe'. -# Currently we can only tell the tag the current commit is -# pointing to, or its hash (with no version info) -# if it is not tagged. -def get_version_from_git_archive(version_info): - try: - refnames = version_info["refnames"] - git_hash = version_info["git_hash"] - except KeyError: - # These fields are not present if we are running from an sdist. - # Execution should never reach here, though - return None - - if git_hash.startswith("$Format") or refnames.startswith("$Format"): - # variables not expanded during 'git archive' - return None - - VTAG = "tag: " - refs = set(r.strip() for r in refnames.split(",")) - version_tags = set(r[len(VTAG) :] for r in refs if r.startswith(VTAG)) - if version_tags: - release, *_ = sorted(version_tags) # prefer e.g. "2.0" over "2.0rc1" - release = TAG_RELEASE_FORMAT.match(release).group(1) - return Version(release, dev=None, labels=None) - else: - return Version("unknown", dev=None, labels=["g{}".format(git_hash)]) - - __version__ = get_version() - -# The following section defines a 'get_cmdclass' function -# that can be used from setup.py. The '__version__' module -# global is used (but not modified). - - -def _write_version(fname): - # This could be a hard link, so try to delete it first. Is there any way - # to do this atomically together with opening? - try: - os.remove(fname) - except OSError: - pass - with open(fname, "w") as f: - f.write( - "# This file has been created by setup.py.\n" - "version = '{}'\n".format(__version__) - ) - - -def get_cmdclass(pkg_source_path): - from setuptools.command.build_py import build_py as build_py_orig - from setuptools.command.sdist import sdist as sdist_orig - - class _build_py(build_py_orig): - def run(self): - super().run() - - src_marker = "".join(["src", os.path.sep]) - - if pkg_source_path.startswith(src_marker): - path = pkg_source_path[len(src_marker) :] - else: - path = pkg_source_path - _write_version(os.path.join(self.build_lib, path, STATIC_VERSION_FILE)) - - class _sdist(sdist_orig): - def make_release_tree(self, base_dir, files): - super().make_release_tree(base_dir, files) - _write_version(os.path.join(base_dir, pkg_source_path, STATIC_VERSION_FILE)) - - return dict(sdist=_sdist, build_py=_build_py) - - if __name__ == "__main__": print("Version: ", get_version()) diff --git a/python/adbc_driver_flightsql/setup.py b/python/adbc_driver_flightsql/setup.py index d8424b73e2..8be464d29d 100644 --- a/python/adbc_driver_flightsql/setup.py +++ b/python/adbc_driver_flightsql/setup.py @@ -49,7 +49,7 @@ # Resolve Version (miniver) -def get_version_and_cmdclass(pkg_path): +def get_version(pkg_path): """ Load version.py module without importing the whole package. @@ -60,15 +60,14 @@ def get_version_and_cmdclass(pkg_path): spec = spec_from_file_location("version", os.path.join(pkg_path, "_version.py")) module = module_from_spec(spec) spec.loader.exec_module(module) - return module.__version__, module.get_cmdclass(pkg_path) + return module.__version__ -version, cmdclass = get_version_and_cmdclass("adbc_driver_flightsql") +version = get_version("adbc_driver_flightsql") # ------------------------------------------------------------ # Setup setup( - cmdclass=cmdclass, version=version, ) diff --git a/python/adbc_driver_manager/adbc_driver_manager/_static_version.py b/python/adbc_driver_manager/adbc_driver_manager/_static_version.py index 4c31c3b721..bd1f923e3b 100644 --- a/python/adbc_driver_manager/adbc_driver_manager/_static_version.py +++ b/python/adbc_driver_manager/adbc_driver_manager/_static_version.py @@ -19,12 +19,8 @@ # Generated by miniver (CC0). # This file is part of 'miniver': https://github.com/jbweston/miniver -# -# This file will be overwritten by setup.py when a source or binary -# distribution is made. The magic value "__use_git__" is interpreted by -# _version.py. -version = "__use_git__" +version = "0.9.0dev" # These values are only set if the distribution was created with 'git archive' # NOTE: must add an export-subst to .gitattributes! diff --git a/python/adbc_driver_manager/adbc_driver_manager/_version.py b/python/adbc_driver_manager/adbc_driver_manager/_version.py index 8eadf0e6c1..5483d6d65f 100644 --- a/python/adbc_driver_manager/adbc_driver_manager/_version.py +++ b/python/adbc_driver_manager/adbc_driver_manager/_version.py @@ -17,15 +17,9 @@ # under the License. # Generated by miniver (CC0). - -import os -import re - # This file is part of 'miniver': https://github.com/jbweston/miniver -# -from collections import namedtuple -Version = namedtuple("Version", ("release", "dev", "labels")) +import os # No public API __all__ = [] @@ -34,9 +28,6 @@ package_name = os.path.basename(package_root) STATIC_VERSION_FILE = "_static_version.py" -TAG_RELEASE_FORMAT = re.compile( - r"^apache-arrow-adbc-([0-9]+\.[0-9]+\.[0-9]+)(?:-rc[0-9]+)?$" -) def get_version(version_file=STATIC_VERSION_FILE): @@ -45,15 +36,7 @@ def get_version(version_file=STATIC_VERSION_FILE): return override version_info = get_static_version_info(version_file) version = version_info["version"] - if version == "__use_git__": - version = get_version_from_git() - if not version: - version = get_version_from_git_archive(version_info) - if not version: - version = Version("unknown", None, None) - return pep440_format(version) - else: - return version + return version def get_static_version_info(version_file=STATIC_VERSION_FILE): @@ -63,158 +46,7 @@ def get_static_version_info(version_file=STATIC_VERSION_FILE): return version_info -def version_is_from_git(version_file=STATIC_VERSION_FILE): - return get_static_version_info(version_file)["version"] == "__use_git__" - - -def pep440_format(version_info): - release, dev, labels = version_info - - version_parts = [release] - if dev: - if release.endswith("-dev") or release.endswith(".dev"): - version_parts.append(dev) - else: # prefer PEP440 over strict adhesion to semver - version_parts.append(".dev{}".format(dev)) - - if labels: - version_parts.append("+") - version_parts.append(".".join(labels)) - - return "".join(version_parts) - - -def get_version_from_git(): - import subprocess - - # git describe --first-parent does not take into account tags from branches - # that were merged-in. The '--long' flag gets us the 'dev' version and - # git hash, '--always' returns the git hash even if there are no tags. - for opts in [["--first-parent"], []]: - try: - p = subprocess.Popen( - ["git", "describe", "--long", "--always"] + opts, - cwd=package_root, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - except OSError: - return - if p.wait() == 0: - break - else: - return - - description = ( - p.communicate()[0] - .decode() - .strip("v") # Tags can have a leading 'v', but the version should not - .rstrip("\n") - .rsplit("-", 2) # Split the latest tag, commits since tag, and hash - ) - - try: - release, dev, git = description - release = TAG_RELEASE_FORMAT.match(release).group(1) - except (AttributeError, ValueError): - # Invalid tag; or no tags, only the git hash - # prepend 'g' to match with format returned by 'git describe' - git = "g{}".format(*description) - # XXX: assume version if not given - release = "0.0.0" - dev = None - - labels = [] - if dev == "0": - dev = None - else: - labels.append(git) - - try: - p = subprocess.Popen(["git", "diff", "--quiet"], cwd=package_root) - except OSError: - labels.append("confused") # This should never happen. - else: - if p.wait() == 1: - labels.append("dirty") - - return Version(release, dev, labels) - - -# TODO: change this logic when there is a git pretty-format -# that gives the same output as 'git describe'. -# Currently we can only tell the tag the current commit is -# pointing to, or its hash (with no version info) -# if it is not tagged. -def get_version_from_git_archive(version_info): - try: - refnames = version_info["refnames"] - git_hash = version_info["git_hash"] - except KeyError: - # These fields are not present if we are running from an sdist. - # Execution should never reach here, though - return None - - if git_hash.startswith("$Format") or refnames.startswith("$Format"): - # variables not expanded during 'git archive' - return None - - VTAG = "tag: " - refs = set(r.strip() for r in refnames.split(",")) - version_tags = set(r[len(VTAG) :] for r in refs if r.startswith(VTAG)) - if version_tags: - release, *_ = sorted(version_tags) # prefer e.g. "2.0" over "2.0rc1" - release = TAG_RELEASE_FORMAT.match(release).group(1) - return Version(release, dev=None, labels=None) - else: - return Version("unknown", dev=None, labels=["g{}".format(git_hash)]) - - __version__ = get_version() - -# The following section defines a 'get_cmdclass' function -# that can be used from setup.py. The '__version__' module -# global is used (but not modified). - - -def _write_version(fname): - # This could be a hard link, so try to delete it first. Is there any way - # to do this atomically together with opening? - try: - os.remove(fname) - except OSError: - pass - with open(fname, "w") as f: - f.write( - "# This file has been created by setup.py.\n" - "version = '{}'\n".format(__version__) - ) - - -def get_cmdclass(pkg_source_path): - from setuptools.command.build_py import build_py as build_py_orig - from setuptools.command.sdist import sdist as sdist_orig - - class _build_py(build_py_orig): - def run(self): - super().run() - - src_marker = "".join(["src", os.path.sep]) - - if pkg_source_path.startswith(src_marker): - path = pkg_source_path[len(src_marker) :] - else: - path = pkg_source_path - _write_version(os.path.join(self.build_lib, path, STATIC_VERSION_FILE)) - - class _sdist(sdist_orig): - def make_release_tree(self, base_dir, files): - super().make_release_tree(base_dir, files) - _write_version(os.path.join(base_dir, pkg_source_path, STATIC_VERSION_FILE)) - - return dict(sdist=_sdist, build_py=_build_py) - - if __name__ == "__main__": print("Version: ", get_version()) diff --git a/python/adbc_driver_manager/setup.py b/python/adbc_driver_manager/setup.py index 075ae8612d..bbec1a01c6 100644 --- a/python/adbc_driver_manager/setup.py +++ b/python/adbc_driver_manager/setup.py @@ -53,7 +53,7 @@ # Resolve Version (miniver) -def get_version_and_cmdclass(pkg_path): +def get_version(pkg_path): """ Load version.py module without importing the whole package. @@ -64,10 +64,10 @@ def get_version_and_cmdclass(pkg_path): spec = spec_from_file_location("version", os.path.join(pkg_path, "_version.py")) module = module_from_spec(spec) spec.loader.exec_module(module) - return module.__version__, module.get_cmdclass(pkg_path) + return module.__version__ -version, cmdclass = get_version_and_cmdclass("adbc_driver_manager") +version = get_version("adbc_driver_manager") # ------------------------------------------------------------ # Resolve compiler flags @@ -86,7 +86,6 @@ def get_version_and_cmdclass(pkg_path): # Setup setup( - cmdclass=cmdclass, ext_modules=[ Extension( name="adbc_driver_manager._lib", diff --git a/python/adbc_driver_postgresql/adbc_driver_postgresql/_static_version.py b/python/adbc_driver_postgresql/adbc_driver_postgresql/_static_version.py index 4c31c3b721..bd1f923e3b 100644 --- a/python/adbc_driver_postgresql/adbc_driver_postgresql/_static_version.py +++ b/python/adbc_driver_postgresql/adbc_driver_postgresql/_static_version.py @@ -19,12 +19,8 @@ # Generated by miniver (CC0). # This file is part of 'miniver': https://github.com/jbweston/miniver -# -# This file will be overwritten by setup.py when a source or binary -# distribution is made. The magic value "__use_git__" is interpreted by -# _version.py. -version = "__use_git__" +version = "0.9.0dev" # These values are only set if the distribution was created with 'git archive' # NOTE: must add an export-subst to .gitattributes! diff --git a/python/adbc_driver_postgresql/adbc_driver_postgresql/_version.py b/python/adbc_driver_postgresql/adbc_driver_postgresql/_version.py index 8eadf0e6c1..5483d6d65f 100644 --- a/python/adbc_driver_postgresql/adbc_driver_postgresql/_version.py +++ b/python/adbc_driver_postgresql/adbc_driver_postgresql/_version.py @@ -17,15 +17,9 @@ # under the License. # Generated by miniver (CC0). - -import os -import re - # This file is part of 'miniver': https://github.com/jbweston/miniver -# -from collections import namedtuple -Version = namedtuple("Version", ("release", "dev", "labels")) +import os # No public API __all__ = [] @@ -34,9 +28,6 @@ package_name = os.path.basename(package_root) STATIC_VERSION_FILE = "_static_version.py" -TAG_RELEASE_FORMAT = re.compile( - r"^apache-arrow-adbc-([0-9]+\.[0-9]+\.[0-9]+)(?:-rc[0-9]+)?$" -) def get_version(version_file=STATIC_VERSION_FILE): @@ -45,15 +36,7 @@ def get_version(version_file=STATIC_VERSION_FILE): return override version_info = get_static_version_info(version_file) version = version_info["version"] - if version == "__use_git__": - version = get_version_from_git() - if not version: - version = get_version_from_git_archive(version_info) - if not version: - version = Version("unknown", None, None) - return pep440_format(version) - else: - return version + return version def get_static_version_info(version_file=STATIC_VERSION_FILE): @@ -63,158 +46,7 @@ def get_static_version_info(version_file=STATIC_VERSION_FILE): return version_info -def version_is_from_git(version_file=STATIC_VERSION_FILE): - return get_static_version_info(version_file)["version"] == "__use_git__" - - -def pep440_format(version_info): - release, dev, labels = version_info - - version_parts = [release] - if dev: - if release.endswith("-dev") or release.endswith(".dev"): - version_parts.append(dev) - else: # prefer PEP440 over strict adhesion to semver - version_parts.append(".dev{}".format(dev)) - - if labels: - version_parts.append("+") - version_parts.append(".".join(labels)) - - return "".join(version_parts) - - -def get_version_from_git(): - import subprocess - - # git describe --first-parent does not take into account tags from branches - # that were merged-in. The '--long' flag gets us the 'dev' version and - # git hash, '--always' returns the git hash even if there are no tags. - for opts in [["--first-parent"], []]: - try: - p = subprocess.Popen( - ["git", "describe", "--long", "--always"] + opts, - cwd=package_root, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - except OSError: - return - if p.wait() == 0: - break - else: - return - - description = ( - p.communicate()[0] - .decode() - .strip("v") # Tags can have a leading 'v', but the version should not - .rstrip("\n") - .rsplit("-", 2) # Split the latest tag, commits since tag, and hash - ) - - try: - release, dev, git = description - release = TAG_RELEASE_FORMAT.match(release).group(1) - except (AttributeError, ValueError): - # Invalid tag; or no tags, only the git hash - # prepend 'g' to match with format returned by 'git describe' - git = "g{}".format(*description) - # XXX: assume version if not given - release = "0.0.0" - dev = None - - labels = [] - if dev == "0": - dev = None - else: - labels.append(git) - - try: - p = subprocess.Popen(["git", "diff", "--quiet"], cwd=package_root) - except OSError: - labels.append("confused") # This should never happen. - else: - if p.wait() == 1: - labels.append("dirty") - - return Version(release, dev, labels) - - -# TODO: change this logic when there is a git pretty-format -# that gives the same output as 'git describe'. -# Currently we can only tell the tag the current commit is -# pointing to, or its hash (with no version info) -# if it is not tagged. -def get_version_from_git_archive(version_info): - try: - refnames = version_info["refnames"] - git_hash = version_info["git_hash"] - except KeyError: - # These fields are not present if we are running from an sdist. - # Execution should never reach here, though - return None - - if git_hash.startswith("$Format") or refnames.startswith("$Format"): - # variables not expanded during 'git archive' - return None - - VTAG = "tag: " - refs = set(r.strip() for r in refnames.split(",")) - version_tags = set(r[len(VTAG) :] for r in refs if r.startswith(VTAG)) - if version_tags: - release, *_ = sorted(version_tags) # prefer e.g. "2.0" over "2.0rc1" - release = TAG_RELEASE_FORMAT.match(release).group(1) - return Version(release, dev=None, labels=None) - else: - return Version("unknown", dev=None, labels=["g{}".format(git_hash)]) - - __version__ = get_version() - -# The following section defines a 'get_cmdclass' function -# that can be used from setup.py. The '__version__' module -# global is used (but not modified). - - -def _write_version(fname): - # This could be a hard link, so try to delete it first. Is there any way - # to do this atomically together with opening? - try: - os.remove(fname) - except OSError: - pass - with open(fname, "w") as f: - f.write( - "# This file has been created by setup.py.\n" - "version = '{}'\n".format(__version__) - ) - - -def get_cmdclass(pkg_source_path): - from setuptools.command.build_py import build_py as build_py_orig - from setuptools.command.sdist import sdist as sdist_orig - - class _build_py(build_py_orig): - def run(self): - super().run() - - src_marker = "".join(["src", os.path.sep]) - - if pkg_source_path.startswith(src_marker): - path = pkg_source_path[len(src_marker) :] - else: - path = pkg_source_path - _write_version(os.path.join(self.build_lib, path, STATIC_VERSION_FILE)) - - class _sdist(sdist_orig): - def make_release_tree(self, base_dir, files): - super().make_release_tree(base_dir, files) - _write_version(os.path.join(base_dir, pkg_source_path, STATIC_VERSION_FILE)) - - return dict(sdist=_sdist, build_py=_build_py) - - if __name__ == "__main__": print("Version: ", get_version()) diff --git a/python/adbc_driver_postgresql/setup.py b/python/adbc_driver_postgresql/setup.py index 998720939f..6675f42bb0 100644 --- a/python/adbc_driver_postgresql/setup.py +++ b/python/adbc_driver_postgresql/setup.py @@ -49,7 +49,7 @@ # Resolve Version (miniver) -def get_version_and_cmdclass(pkg_path): +def get_version(pkg_path): """ Load version.py module without importing the whole package. @@ -60,15 +60,14 @@ def get_version_and_cmdclass(pkg_path): spec = spec_from_file_location("version", os.path.join(pkg_path, "_version.py")) module = module_from_spec(spec) spec.loader.exec_module(module) - return module.__version__, module.get_cmdclass(pkg_path) + return module.__version__ -version, cmdclass = get_version_and_cmdclass("adbc_driver_postgresql") +version = get_version("adbc_driver_postgresql") # ------------------------------------------------------------ # Setup setup( - cmdclass=cmdclass, version=version, ) diff --git a/python/adbc_driver_snowflake/adbc_driver_snowflake/_static_version.py b/python/adbc_driver_snowflake/adbc_driver_snowflake/_static_version.py index 4c31c3b721..bd1f923e3b 100644 --- a/python/adbc_driver_snowflake/adbc_driver_snowflake/_static_version.py +++ b/python/adbc_driver_snowflake/adbc_driver_snowflake/_static_version.py @@ -19,12 +19,8 @@ # Generated by miniver (CC0). # This file is part of 'miniver': https://github.com/jbweston/miniver -# -# This file will be overwritten by setup.py when a source or binary -# distribution is made. The magic value "__use_git__" is interpreted by -# _version.py. -version = "__use_git__" +version = "0.9.0dev" # These values are only set if the distribution was created with 'git archive' # NOTE: must add an export-subst to .gitattributes! diff --git a/python/adbc_driver_snowflake/adbc_driver_snowflake/_version.py b/python/adbc_driver_snowflake/adbc_driver_snowflake/_version.py index 8eadf0e6c1..5483d6d65f 100644 --- a/python/adbc_driver_snowflake/adbc_driver_snowflake/_version.py +++ b/python/adbc_driver_snowflake/adbc_driver_snowflake/_version.py @@ -17,15 +17,9 @@ # under the License. # Generated by miniver (CC0). - -import os -import re - # This file is part of 'miniver': https://github.com/jbweston/miniver -# -from collections import namedtuple -Version = namedtuple("Version", ("release", "dev", "labels")) +import os # No public API __all__ = [] @@ -34,9 +28,6 @@ package_name = os.path.basename(package_root) STATIC_VERSION_FILE = "_static_version.py" -TAG_RELEASE_FORMAT = re.compile( - r"^apache-arrow-adbc-([0-9]+\.[0-9]+\.[0-9]+)(?:-rc[0-9]+)?$" -) def get_version(version_file=STATIC_VERSION_FILE): @@ -45,15 +36,7 @@ def get_version(version_file=STATIC_VERSION_FILE): return override version_info = get_static_version_info(version_file) version = version_info["version"] - if version == "__use_git__": - version = get_version_from_git() - if not version: - version = get_version_from_git_archive(version_info) - if not version: - version = Version("unknown", None, None) - return pep440_format(version) - else: - return version + return version def get_static_version_info(version_file=STATIC_VERSION_FILE): @@ -63,158 +46,7 @@ def get_static_version_info(version_file=STATIC_VERSION_FILE): return version_info -def version_is_from_git(version_file=STATIC_VERSION_FILE): - return get_static_version_info(version_file)["version"] == "__use_git__" - - -def pep440_format(version_info): - release, dev, labels = version_info - - version_parts = [release] - if dev: - if release.endswith("-dev") or release.endswith(".dev"): - version_parts.append(dev) - else: # prefer PEP440 over strict adhesion to semver - version_parts.append(".dev{}".format(dev)) - - if labels: - version_parts.append("+") - version_parts.append(".".join(labels)) - - return "".join(version_parts) - - -def get_version_from_git(): - import subprocess - - # git describe --first-parent does not take into account tags from branches - # that were merged-in. The '--long' flag gets us the 'dev' version and - # git hash, '--always' returns the git hash even if there are no tags. - for opts in [["--first-parent"], []]: - try: - p = subprocess.Popen( - ["git", "describe", "--long", "--always"] + opts, - cwd=package_root, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - except OSError: - return - if p.wait() == 0: - break - else: - return - - description = ( - p.communicate()[0] - .decode() - .strip("v") # Tags can have a leading 'v', but the version should not - .rstrip("\n") - .rsplit("-", 2) # Split the latest tag, commits since tag, and hash - ) - - try: - release, dev, git = description - release = TAG_RELEASE_FORMAT.match(release).group(1) - except (AttributeError, ValueError): - # Invalid tag; or no tags, only the git hash - # prepend 'g' to match with format returned by 'git describe' - git = "g{}".format(*description) - # XXX: assume version if not given - release = "0.0.0" - dev = None - - labels = [] - if dev == "0": - dev = None - else: - labels.append(git) - - try: - p = subprocess.Popen(["git", "diff", "--quiet"], cwd=package_root) - except OSError: - labels.append("confused") # This should never happen. - else: - if p.wait() == 1: - labels.append("dirty") - - return Version(release, dev, labels) - - -# TODO: change this logic when there is a git pretty-format -# that gives the same output as 'git describe'. -# Currently we can only tell the tag the current commit is -# pointing to, or its hash (with no version info) -# if it is not tagged. -def get_version_from_git_archive(version_info): - try: - refnames = version_info["refnames"] - git_hash = version_info["git_hash"] - except KeyError: - # These fields are not present if we are running from an sdist. - # Execution should never reach here, though - return None - - if git_hash.startswith("$Format") or refnames.startswith("$Format"): - # variables not expanded during 'git archive' - return None - - VTAG = "tag: " - refs = set(r.strip() for r in refnames.split(",")) - version_tags = set(r[len(VTAG) :] for r in refs if r.startswith(VTAG)) - if version_tags: - release, *_ = sorted(version_tags) # prefer e.g. "2.0" over "2.0rc1" - release = TAG_RELEASE_FORMAT.match(release).group(1) - return Version(release, dev=None, labels=None) - else: - return Version("unknown", dev=None, labels=["g{}".format(git_hash)]) - - __version__ = get_version() - -# The following section defines a 'get_cmdclass' function -# that can be used from setup.py. The '__version__' module -# global is used (but not modified). - - -def _write_version(fname): - # This could be a hard link, so try to delete it first. Is there any way - # to do this atomically together with opening? - try: - os.remove(fname) - except OSError: - pass - with open(fname, "w") as f: - f.write( - "# This file has been created by setup.py.\n" - "version = '{}'\n".format(__version__) - ) - - -def get_cmdclass(pkg_source_path): - from setuptools.command.build_py import build_py as build_py_orig - from setuptools.command.sdist import sdist as sdist_orig - - class _build_py(build_py_orig): - def run(self): - super().run() - - src_marker = "".join(["src", os.path.sep]) - - if pkg_source_path.startswith(src_marker): - path = pkg_source_path[len(src_marker) :] - else: - path = pkg_source_path - _write_version(os.path.join(self.build_lib, path, STATIC_VERSION_FILE)) - - class _sdist(sdist_orig): - def make_release_tree(self, base_dir, files): - super().make_release_tree(base_dir, files) - _write_version(os.path.join(base_dir, pkg_source_path, STATIC_VERSION_FILE)) - - return dict(sdist=_sdist, build_py=_build_py) - - if __name__ == "__main__": print("Version: ", get_version()) diff --git a/python/adbc_driver_snowflake/setup.py b/python/adbc_driver_snowflake/setup.py index e227297fd9..9dad7dcc2b 100644 --- a/python/adbc_driver_snowflake/setup.py +++ b/python/adbc_driver_snowflake/setup.py @@ -49,7 +49,7 @@ # Resolve Version (miniver) -def get_version_and_cmdclass(pkg_path): +def get_version(pkg_path): """ Load version.py module without importing the whole package. @@ -60,15 +60,14 @@ def get_version_and_cmdclass(pkg_path): spec = spec_from_file_location("version", os.path.join(pkg_path, "_version.py")) module = module_from_spec(spec) spec.loader.exec_module(module) - return module.__version__, module.get_cmdclass(pkg_path) + return module.__version__ -version, cmdclass = get_version_and_cmdclass("adbc_driver_snowflake") +version = get_version("adbc_driver_snowflake") # ------------------------------------------------------------ # Setup setup( - cmdclass=cmdclass, version=version, ) diff --git a/python/adbc_driver_sqlite/adbc_driver_sqlite/_static_version.py b/python/adbc_driver_sqlite/adbc_driver_sqlite/_static_version.py index 4c31c3b721..bd1f923e3b 100644 --- a/python/adbc_driver_sqlite/adbc_driver_sqlite/_static_version.py +++ b/python/adbc_driver_sqlite/adbc_driver_sqlite/_static_version.py @@ -19,12 +19,8 @@ # Generated by miniver (CC0). # This file is part of 'miniver': https://github.com/jbweston/miniver -# -# This file will be overwritten by setup.py when a source or binary -# distribution is made. The magic value "__use_git__" is interpreted by -# _version.py. -version = "__use_git__" +version = "0.9.0dev" # These values are only set if the distribution was created with 'git archive' # NOTE: must add an export-subst to .gitattributes! diff --git a/python/adbc_driver_sqlite/adbc_driver_sqlite/_version.py b/python/adbc_driver_sqlite/adbc_driver_sqlite/_version.py index 8eadf0e6c1..5483d6d65f 100644 --- a/python/adbc_driver_sqlite/adbc_driver_sqlite/_version.py +++ b/python/adbc_driver_sqlite/adbc_driver_sqlite/_version.py @@ -17,15 +17,9 @@ # under the License. # Generated by miniver (CC0). - -import os -import re - # This file is part of 'miniver': https://github.com/jbweston/miniver -# -from collections import namedtuple -Version = namedtuple("Version", ("release", "dev", "labels")) +import os # No public API __all__ = [] @@ -34,9 +28,6 @@ package_name = os.path.basename(package_root) STATIC_VERSION_FILE = "_static_version.py" -TAG_RELEASE_FORMAT = re.compile( - r"^apache-arrow-adbc-([0-9]+\.[0-9]+\.[0-9]+)(?:-rc[0-9]+)?$" -) def get_version(version_file=STATIC_VERSION_FILE): @@ -45,15 +36,7 @@ def get_version(version_file=STATIC_VERSION_FILE): return override version_info = get_static_version_info(version_file) version = version_info["version"] - if version == "__use_git__": - version = get_version_from_git() - if not version: - version = get_version_from_git_archive(version_info) - if not version: - version = Version("unknown", None, None) - return pep440_format(version) - else: - return version + return version def get_static_version_info(version_file=STATIC_VERSION_FILE): @@ -63,158 +46,7 @@ def get_static_version_info(version_file=STATIC_VERSION_FILE): return version_info -def version_is_from_git(version_file=STATIC_VERSION_FILE): - return get_static_version_info(version_file)["version"] == "__use_git__" - - -def pep440_format(version_info): - release, dev, labels = version_info - - version_parts = [release] - if dev: - if release.endswith("-dev") or release.endswith(".dev"): - version_parts.append(dev) - else: # prefer PEP440 over strict adhesion to semver - version_parts.append(".dev{}".format(dev)) - - if labels: - version_parts.append("+") - version_parts.append(".".join(labels)) - - return "".join(version_parts) - - -def get_version_from_git(): - import subprocess - - # git describe --first-parent does not take into account tags from branches - # that were merged-in. The '--long' flag gets us the 'dev' version and - # git hash, '--always' returns the git hash even if there are no tags. - for opts in [["--first-parent"], []]: - try: - p = subprocess.Popen( - ["git", "describe", "--long", "--always"] + opts, - cwd=package_root, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - except OSError: - return - if p.wait() == 0: - break - else: - return - - description = ( - p.communicate()[0] - .decode() - .strip("v") # Tags can have a leading 'v', but the version should not - .rstrip("\n") - .rsplit("-", 2) # Split the latest tag, commits since tag, and hash - ) - - try: - release, dev, git = description - release = TAG_RELEASE_FORMAT.match(release).group(1) - except (AttributeError, ValueError): - # Invalid tag; or no tags, only the git hash - # prepend 'g' to match with format returned by 'git describe' - git = "g{}".format(*description) - # XXX: assume version if not given - release = "0.0.0" - dev = None - - labels = [] - if dev == "0": - dev = None - else: - labels.append(git) - - try: - p = subprocess.Popen(["git", "diff", "--quiet"], cwd=package_root) - except OSError: - labels.append("confused") # This should never happen. - else: - if p.wait() == 1: - labels.append("dirty") - - return Version(release, dev, labels) - - -# TODO: change this logic when there is a git pretty-format -# that gives the same output as 'git describe'. -# Currently we can only tell the tag the current commit is -# pointing to, or its hash (with no version info) -# if it is not tagged. -def get_version_from_git_archive(version_info): - try: - refnames = version_info["refnames"] - git_hash = version_info["git_hash"] - except KeyError: - # These fields are not present if we are running from an sdist. - # Execution should never reach here, though - return None - - if git_hash.startswith("$Format") or refnames.startswith("$Format"): - # variables not expanded during 'git archive' - return None - - VTAG = "tag: " - refs = set(r.strip() for r in refnames.split(",")) - version_tags = set(r[len(VTAG) :] for r in refs if r.startswith(VTAG)) - if version_tags: - release, *_ = sorted(version_tags) # prefer e.g. "2.0" over "2.0rc1" - release = TAG_RELEASE_FORMAT.match(release).group(1) - return Version(release, dev=None, labels=None) - else: - return Version("unknown", dev=None, labels=["g{}".format(git_hash)]) - - __version__ = get_version() - -# The following section defines a 'get_cmdclass' function -# that can be used from setup.py. The '__version__' module -# global is used (but not modified). - - -def _write_version(fname): - # This could be a hard link, so try to delete it first. Is there any way - # to do this atomically together with opening? - try: - os.remove(fname) - except OSError: - pass - with open(fname, "w") as f: - f.write( - "# This file has been created by setup.py.\n" - "version = '{}'\n".format(__version__) - ) - - -def get_cmdclass(pkg_source_path): - from setuptools.command.build_py import build_py as build_py_orig - from setuptools.command.sdist import sdist as sdist_orig - - class _build_py(build_py_orig): - def run(self): - super().run() - - src_marker = "".join(["src", os.path.sep]) - - if pkg_source_path.startswith(src_marker): - path = pkg_source_path[len(src_marker) :] - else: - path = pkg_source_path - _write_version(os.path.join(self.build_lib, path, STATIC_VERSION_FILE)) - - class _sdist(sdist_orig): - def make_release_tree(self, base_dir, files): - super().make_release_tree(base_dir, files) - _write_version(os.path.join(base_dir, pkg_source_path, STATIC_VERSION_FILE)) - - return dict(sdist=_sdist, build_py=_build_py) - - if __name__ == "__main__": print("Version: ", get_version()) diff --git a/python/adbc_driver_sqlite/setup.py b/python/adbc_driver_sqlite/setup.py index 4bc1336c9a..e5a3a5dc44 100644 --- a/python/adbc_driver_sqlite/setup.py +++ b/python/adbc_driver_sqlite/setup.py @@ -47,7 +47,7 @@ # Resolve Version (miniver) -def get_version_and_cmdclass(pkg_path): +def get_version(pkg_path): """ Load version.py module without importing the whole package. @@ -58,15 +58,14 @@ def get_version_and_cmdclass(pkg_path): spec = spec_from_file_location("version", os.path.join(pkg_path, "_version.py")) module = module_from_spec(spec) spec.loader.exec_module(module) - return module.__version__, module.get_cmdclass(pkg_path) + return module.__version__ -version, cmdclass = get_version_and_cmdclass("adbc_driver_sqlite") +version = get_version("adbc_driver_sqlite") # ------------------------------------------------------------ # Setup setup( - cmdclass=cmdclass, version=version, ) From 61c9236a822b01748dd66dfa4d340a6d236d416e Mon Sep 17 00:00:00 2001 From: rtadepalli <105760760+rtadepalli@users.noreply.github.com> Date: Wed, 3 Jan 2024 08:34:58 -0500 Subject: [PATCH 51/51] fix(java/driver/jdbc): fix connection leak in `JdbcDataSourceDatabase` constructor (#1418) I think there is a connection leak in the constructor for `JdbcDataSourceDatabase` where we're obtaining a connection in the constructor but we're obtaining another one in the `connect` method using the appropriate username/password. We can remove this and fix the bug as described in #1415. Fixes #1415. --- .../driver/jdbc/JdbcDataSourceDatabase.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/java/driver/jdbc/src/main/java/org/apache/arrow/adbc/driver/jdbc/JdbcDataSourceDatabase.java b/java/driver/jdbc/src/main/java/org/apache/arrow/adbc/driver/jdbc/JdbcDataSourceDatabase.java index 4059759b71..6348e05c9c 100644 --- a/java/driver/jdbc/src/main/java/org/apache/arrow/adbc/driver/jdbc/JdbcDataSourceDatabase.java +++ b/java/driver/jdbc/src/main/java/org/apache/arrow/adbc/driver/jdbc/JdbcDataSourceDatabase.java @@ -34,8 +34,8 @@ public final class JdbcDataSourceDatabase implements AdbcDatabase { private final String username; private final String password; private final JdbcQuirks quirks; - private final Connection connection; private final AtomicInteger counter; + private Connection connection; JdbcDataSourceDatabase( BufferAllocator allocator, @@ -49,22 +49,19 @@ public final class JdbcDataSourceDatabase implements AdbcDatabase { this.username = username; this.password = password; this.quirks = Objects.requireNonNull(quirks); - try { - this.connection = dataSource.getConnection(); - } catch (SQLException e) { - throw JdbcDriverUtil.fromSqlException(e); - } + this.connection = null; this.counter = new AtomicInteger(); } @Override public AdbcConnection connect() throws AdbcException { - final Connection connection; try { - if (username != null && password != null) { - connection = dataSource.getConnection(username, password); - } else { - connection = dataSource.getConnection(); + if (connection == null) { + if (username != null && password != null) { + connection = dataSource.getConnection(username, password); + } else { + connection = dataSource.getConnection(); + } } } catch (SQLException e) { throw JdbcDriverUtil.fromSqlException(e); @@ -79,7 +76,10 @@ public AdbcConnection connect() throws AdbcException { @Override public void close() throws Exception { - connection.close(); + if (connection != null) { + connection.close(); + } + connection = null; } @Override