https://github.com/manifold-systems/manifold/issues/608

- streamline duckdb's appender usage within manifold-sql

manifold-deps-parent/manifold-json-rt/src/main/java/manifold/json/rt/api/DataBindings.java

@@ -57,7 +57,7 @@ public DataBindings( Map<String, Object> map )
   }
 
   /**
-   * Default constructor uses a {@code LinkedHashMap} to maintain the nature order of entries.
+   * Default constructor uses a {@code LinkedHashMap} to maintain the insert order of entries.
    */
   public DataBindings()
   {

manifold-deps-parent/manifold-sql-inproc-test/src/test/java/manifold/sql/schema/duckdb/AppendTest.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2023 - Manifold Systems LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package manifold.sql.schema.duckdb;
+
+import manifold.sql.schema.duckdb.base.DuckdbDdlServerTest;
+import manifold.sql.schema.simple.duckdb.DuckdbSakila;
+import manifold.sql.schema.simple.duckdb.DuckdbSakila.Country;
+import org.junit.Test;
+
+import java.sql.SQLException;
+import java.time.LocalDateTime;
+
+import static org.junit.Assert.*;
+
+public class AppendTest extends DuckdbDdlServerTest
+{
+  @Test
+  public void testAppendNoTransaction() throws SQLException
+  {
+    LocalDateTime now = LocalDateTime.now();
+    Country.append( a -> {
+      a.append( 1, "Canada", now );
+      a.append( 2, "Mexico", now );
+      a.append( 3, "Belize", now );
+      a.append( 4, "Brazil", now );
+    } );
+    StringBuilder sb = new StringBuilder();
+    for( Country country : Country.fetchAll() )
+    {
+      sb.append( country.getCountryId() ).append( ',' )
+        .append( country.getCountry() ).append( ',' )
+        .append( country.getLastUpdate() ).append( '\n' );
+    }
+    assertEquals(
+      "1,Canada," + now + "\n" +
+      "2,Mexico," + now + "\n" +
+      "3,Belize," + now + "\n" +
+      "4,Brazil," + now + "\n",
+      sb.toString() );
+  }
+
+  @Test
+  public void testAppendWithinTransaction() throws SQLException
+  {
+    LocalDateTime now = LocalDateTime.now();
+    DuckdbSakila.commit( ctx -> {
+      Country.builder( "first" ).withLastUpdate( now ).build();
+      "[.sql:DuckdbSakila/] insert into country (country, last_update) values('second', :now)".execute( ctx, now );
+      Country.append( a -> {
+        a.append( 10, "Canada", now );
+        a.append( 20, "Mexico", now );
+        a.append( 30, "Belize", now );
+        a.append( 40, "Brazil", now );
+      } );
+    } );
+    StringBuilder sb = new StringBuilder();
+    for( Country country : Country.fetchAll() )
+    {
+      sb.append( country.getCountryId() ).append( ',' )
+        .append( country.getCountry() ).append( ',' )
+        .append( country.getLastUpdate() ).append( '\n' );
+    }
+    assertEquals(
+      "1,first," + now + "\n" +
+      "2,second," + now + "\n" +
+      "10,Canada," + now + "\n" +
+      "20,Mexico," + now + "\n" +
+      "30,Belize," + now + "\n" +
+      "40,Brazil," + now + "\n",
+      sb.toString() );
+  }
+}

manifold-deps-parent/manifold-sql-rt/pom.xml

@@ -47,6 +47,13 @@
       <artifactId>annotations</artifactId>
       <version>24.0.1</version>
     </dependency>
+    <!-- duckdb for compile-time only, exclusively for Appender integration -->
+    <dependency>
+      <groupId>org.duckdb</groupId>
+      <artifactId>duckdb_jdbc</artifactId>
+      <version>1.0.0</version>
+      <scope>provided</scope>
+    </dependency>
   </dependencies>
 
   <build>

manifold-deps-parent/manifold-sql-rt/src/main/java/manifold/sql/rt/api/OperableTxScope.java

@@ -17,6 +17,7 @@
 package manifold.sql.rt.api;
 
 import java.sql.Connection;
+import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Set;
 import java.util.function.Consumer;
@@ -37,4 +38,7 @@ public interface OperableTxScope extends TxScope
   BatchSqlChangeCtx newBatchSqlChangeCtx( Connection c );
 
   void addBatch( Executor exec, Consumer<Statement> consumer );
+
+  // specific to duckdb (for now)
+  <T extends SchemaAppender> void append( Consumer<T> consumer, T appender ) throws SQLException;
 }

manifold-deps-parent/manifold-sql-rt/src/main/java/manifold/sql/rt/api/SchemaAppender.java

@@ -0,0 +1,124 @@
+package manifold.sql.rt.api;
+
+import manifold.json.rt.api.DataBindings;
+import manifold.util.ManExceptionUtil;
+import org.duckdb.DuckDBAppender;
+import org.duckdb.DuckDBConnection;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.util.Map;
+import java.util.function.Consumer;
+
+public abstract class SchemaAppender
+{
+  private final String _schema;
+  private final String _table;
+  private DuckDBAppender _appender;
+
+  public SchemaAppender( String schema, String table )
+  {
+    _schema = schema;
+    _table = table;
+  }
+
+  // called from generated code
+  @SuppressWarnings( "unused" )
+  protected void appendRow( DataBindings bindings )
+  {
+    try
+    {
+      _appender.beginRow();
+      appendBindings( _appender, bindings );
+      _appender.endRow();
+    }
+    catch( SQLException e )
+    {
+      throw ManExceptionUtil.unchecked( e );
+    }
+  }
+
+  public <T extends SchemaAppender> void execute( Connection c, Consumer<T> consumer ) throws SQLException
+  {
+    DuckDBConnection duckdbConnection = c.unwrap( DuckDBConnection.class );
+    try( DuckDBAppender appender = duckdbConnection.createAppender( _schema, _table ) )
+    {
+      _appender = appender;
+      //noinspection unchecked
+      consumer.accept( (T)this );
+    }
+    catch( Exception e )
+    {
+      throw new SQLException( e );
+    }
+  }
+
+  private void appendBindings( DuckDBAppender appender, DataBindings bindings ) throws SQLException
+  {
+    for( Map.Entry<String, Object> entry : bindings.entrySet() )
+    {
+      Object value = entry.getValue();
+      if( value == null )
+      {
+        appender.append( null );
+      }
+      else if( value instanceof BigDecimal )
+      {
+        appender.appendBigDecimal( (BigDecimal)value );
+      }
+      else if( value instanceof BigInteger )
+      {
+        appender.append( ((BigInteger)value).longValueExact() );
+      }
+      else if( value instanceof Long )
+      {
+        appender.append( (Long)value );
+      }
+      else if( value instanceof Integer )
+      {
+        appender.append( (Integer)value );
+      }
+      else if( value instanceof Short )
+      {
+        appender.append( (Short)value );
+      }
+      else if( value instanceof Byte )
+      {
+        appender.append( (Byte)value );
+      }
+      else if( value instanceof Double )
+      {
+        appender.append( (Double)value );
+      }
+      else if( value instanceof Float )
+      {
+        appender.append( (Float)value );
+      }
+      else if( value instanceof Boolean )
+      {
+        appender.append( (Boolean)value );
+      }
+      else if( value instanceof LocalDateTime )
+      {
+        appender.appendLocalDateTime( (LocalDateTime)value );
+      }
+      else if( value instanceof OffsetDateTime )
+      {
+        appender.appendLocalDateTime( ((OffsetDateTime)value).toLocalDateTime() );
+      }
+      else if( value instanceof LocalDate )
+      {
+        appender.appendLocalDateTime( ((LocalDate)value).atStartOfDay() );
+      }
+      else
+      {
+        appender.append( value.toString() );
+      }
+    }
+  }
+}

manifold-deps-parent/manifold-sql-rt/src/main/java/manifold/sql/rt/impl/BasicTxScope.java

@@ -254,6 +254,22 @@ public void accept( SqlChangeCtx ctx ) throws SQLException
     }
   }
 
+  @Override
+  public <T extends SchemaAppender> void append( Consumer<T> consumer, T appender ) throws SQLException
+  {
+    // duckdb will use the transaction in an existing connection, or it will commit on its own with a new connection
+
+    Connection activeConnection = getActiveConnection();
+    ConnectionProvider cp = Dependencies.instance().getConnectionProvider();
+    try( Connection newConnection = activeConnection == null
+      ? cp.getConnection( getDbConfig().getName(), appender.getClass() )
+      : null )
+    {
+      Connection c = activeConnection == null ? newConnection : activeConnection;
+      appender.execute( c, consumer );
+    }
+  }
+
   @Override
   public void commit() throws SQLException
   {

manifold-deps-parent/manifold-sql/readme.md

@@ -145,6 +145,8 @@ these APIs are _always_ 100% type-safe, in-sync, and tailored for use with your
     * [Coupled queries](#coupled-queries)
     * [Have it your way](#have-it-your-way)
   * [DML & DDL commands](#dml--ddl-commands)
+  * [Batch commands](#batch-commands)
+  * [Bulk insert](#bulk-insert)
   * [Customizations](#customizations)
   * [Dependency interfaces](#dependency-interfaces)
     * [DbConfigProvider](#dbconfigprovider)
@@ -1111,14 +1113,77 @@ String country = "United States";
 . . .
 Sakila.addSqlChange(ctx -> {
   "[.sql/] DELETE FROM country WHERE country = :country".execute(ctx, country);
-});
+}); 
+. . .
+Sakila.commit();
 ```
 Notice the `execute` method requires the `ctx` parameter, which is supplied exclusively by the transaction scope. This
 ensures SQL commands execute only from calls to `addSqlChange`, which enables such changes to be bundled within the same
 transaction scope as other DML statements and CRUD operations.
 
 This example also illustrates how type-safe, injection-safe SQL parameters can be used with any SQL command.
 
+## Batch commands
+
+---
+
+Batch commands address both performance and atomicity by grouping multiple SQL commands, primarily INSERT and UPDATE, into
+a single database call. Use `TxScope#addBatchChange` to streamline this process with both parameterized and vanilla SQL
+statements.
+
+Use `addBatchChange` to batch many invocations of the same parameterized statement as one database call. 
+```java
+List<String> countries = loadCountries)();
+. . .
+Sakila.addBatchChange(ctx -> {
+  for(String country: countries) {
+    "[.sql/] INSERT INTO country (country) VALUES (:name)".execute(ctx, country);
+  }
+}  
+. . .
+Sakila.commit();
+``` 
+
+Use `addBatchChange` to batch many non-parameterized SQL statements as one database call. 
+```java
+Sakila.addBatchChange(ctx -> {
+    "[.sql/] INSERT INTO country (country) VALUES ('Canada')".execute(ctx);
+    "[.sql/] INSERT INTO country (country) VALUES ('Mexico')".execute(ctx);
+    "[.sql/] INSERT INTO country (country) VALUES ('Belize')".execute(ctx);
+    . . .
+  }
+}
+. . .
+Sakila.commit();
+``` 
+
+## Bulk insert
+
+---
+
+Some databases such as DuckDB offer higher performance for bulk inserts. In this case you can use the `append` method on
+table entities to type-safely load bulk data more efficiently.
+
+```java
+LocalDateTime now = LocalDateTime.now();
+Country.append(a -> {
+  a.append(1, "Canada", now);
+  a.append(2, "Mexico", now);
+  a.append(3, "Belize", now);
+  a.append(4, "Brazil", now);
+  . . .
+});
+```
+
+If the `append` call is made within the context of a [transaction scope](#transaction-scopes), the inserts are bound to
+the transaction. For instance, multiple `append` calls within a [addSqlChange](#dml--ddl-commands) or [addBatchChange](#batch-commands)
+are grouped into the same transaction. Otherwise, the inserts automatically commit in a separate transaction directly after
+the call completes.
+
+>&#9888; The `append` call is not a standard JDBC feature, it is available only with select drivers that offer bulk insert
+> capability, such as DuckDB. [Let us know](https://github.com/manifold-systems/manifold/issues/new?assignees=&labels=&projects=&template=feature_request.md)
+> if your driver implements this feature and is not supported in manifold-sql. 
+
 ## Customizations
 
 ---