Skip to content

SYM-7015: DDL capture support for cross platform #431

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: release/3.16
Choose a base branch
from
Open

SYM-7015: DDL capture support for cross platform #431

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
91ed69f
0007015: DDL capture support for cross platform
joshahicks Aug 6, 2025
df7559a
0007015: DDL capture support for cross platform
joshahicks Aug 11, 2025
a38f252
0007015: DDL capture support for cross platform
joshahicks Aug 28, 2025
eae5965
Merge branch 'release/3.16' into feature/ddl-cross-platform
awebbdev1 Nov 3, 2025
342aebf
0007015: fix style violations
awebbdev1 Nov 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions symmetric-client/src/main/java/org/jumpmind/symmetric/db/mssql/MsSqlTriggerTemplate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ public MsSqlTriggerTemplate(ISymmetricDialect symmetricDialect) {
String triggerExecuteAs = symmetricDialect.getParameterService().getString(ParameterConstants.MSSQL_TRIGGER_EXECUTE_AS, "self");
String defaultCatalog = symmetricDialect.getParameterService().is(ParameterConstants.MSSQL_INCLUDE_CATALOG_IN_TRIGGERS, true) ? "$(defaultCatalog)"
: "";
boolean ddlSendTable = symmetricDialect.getParameterService().is(ParameterConstants.TRIGGER_CAPTURE_DDL_SEND_TABLE);
String ddlEventType = ddlSendTable ? DataEventType.CREATE.getCode() : DataEventType.SQL.getCode();
String ddlRowData = ddlSendTable ? "'" + AbstractTriggerTemplate.CREATE_EVENT_DDL_GENERATED + "'"
: " '\"delimiter " + delimiter
+ ";' + CHAR(13) + char(10) + replace(replace(@data.value('(/EVENT_INSTANCE/TSQLCommand/CommandText)[1]', 'nvarchar(max)'),'\\','\\\\'),'\"','\\\"') + '\",ddl'";
// @formatter:off
emptyColumnTemplate = "''" ;
stringColumnTemplate = "case when $(tableAlias).\"$(columnName)\" is null then '' else '\"' + replace(replace(convert("+
Expand Down Expand Up @@ -449,8 +454,8 @@ public MsSqlTriggerTemplate(ISymmetricDialect symmetricDialect) {
" set @channelId = 'config'\n" +
" insert into " + defaultCatalog + "$(defaultSchema)$(prefixName)_data\n" +
" (table_name, event_type, trigger_hist_id, row_data, channel_id, source_node_id, create_time)\n" +
" values (@tableName, '" + DataEventType.SQL.getCode() + "', @histId,\n" +
" '\"delimiter " + delimiter + ";' + CHAR(13) + char(10) + replace(replace(@data.value('(/EVENT_INSTANCE/TSQLCommand/CommandText)[1]', 'nvarchar(max)'),'\\','\\\\'),'\"','\\\"') + '\",ddl',\n" +
" values (@tableName, '" + ddlEventType + "', @histId,\n" +
Copy link
Contributor

@evan-miller-jumpmind evan-miller-jumpmind Nov 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What happens when the captured DDL isn't for a specific table and @tableName is sym_node? It looks it it would end up sending a create event for sym_node.

" " + ddlRowData + ",\n" +
" @channelId, " + defaultCatalog + "$(defaultSchema)$(prefixName)_node_disabled(), " + getCreateTimeExpression() + ")\n" +
" end\n" +
"end\n" + "---- go");
Expand Down
1 change: 1 addition & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,7 @@ private ParameterConstants() {
public final static String TRIGGER_CAPTURE_DDL_CHANGES = "trigger.capture.ddl.changes";
public final static String TRIGGER_CAPTURE_DDL_DELIMITER = "trigger.capture.ddl.delimiter";
public final static String TRIGGER_CAPTURE_DDL_CHECK_TRIGGER_HIST = "trigger.capture.ddl.check.trigger.hist";
public final static String TRIGGER_CAPTURE_DDL_SEND_TABLE = "trigger.capture.ddl.send.table";
public final static String TRIGGER_USE_INSERT_DELETE_FOR_PRIMARY_KEY_CHANGES = "trigger.use.insert.delete.for.primary.key.changes";
public final static String DB_METADATA_IGNORE_CASE = "db.metadata.ignore.case";
public final static String DB_NATIVE_EXTRACTOR = "db.native.extractor";
Expand Down
1 change: 1 addition & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractTriggerTemplate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ abstract public class AbstractTriggerTemplate {
protected static final String UPDATE_WITH_RELOAD_TRIGGER_TEMPLATE = "updateReloadTriggerTemplate";
protected static final String DELETE_TRIGGER_TEMPLATE = "deleteTriggerTemplate";
protected static final String INITIAL_LOAD_SQL_TEMPLATE = "initialLoadSqlTemplate";
public static final String CREATE_EVENT_DDL_GENERATED = "ddl.generated";
protected Map<String, String> sqlTemplates;
protected String emptyColumnTemplate = "''";
protected String stringColumnTemplate;
Expand Down
11 changes: 11 additions & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/extract/SelectFromSymDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
package org.jumpmind.symmetric.extract;

import java.sql.Types;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -41,6 +42,7 @@
import org.jumpmind.symmetric.common.Constants;
import org.jumpmind.symmetric.common.ErrorConstants;
import org.jumpmind.symmetric.common.ParameterConstants;
import org.jumpmind.symmetric.db.AbstractTriggerTemplate;
import org.jumpmind.symmetric.io.data.Batch;
import org.jumpmind.symmetric.io.data.Batch.BatchType;
import org.jumpmind.symmetric.io.data.CsvData;
Expand Down Expand Up @@ -282,6 +284,15 @@ protected boolean evaluateDeferTableLogging(OutgoingBatch batch, boolean deferIn
}

protected boolean processCreateEvent(TriggerHistory triggerHistory, String routerId, Data data) {
if (data.getRowData() != null && data.getRowData().contains(AbstractTriggerTemplate.CREATE_EVENT_DDL_GENERATED)) {
data.putCsvData(CsvData.ROW_DATA, "");
Trigger trigger = engine.getTriggerRouterService().getTriggerById(triggerHistory.getTriggerId());
engine.getTriggerRouterService().syncTriggers(Collections.singletonList(trigger), null, true, false, false);
List<TriggerHistory> latestTriggerHistory = engine.getTriggerRouterService().getActiveTriggerHistories(trigger);
for (TriggerHistory th : latestTriggerHistory) {
triggerHistory = th;
}
}
String oldData = data.getCsvData(CsvData.OLD_DATA);
boolean sendSchemaExcludeIndices = false;
boolean sendSchemaExcludeForeignKeys = false;
Expand Down
2 changes: 2 additions & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/service/ITriggerRouterService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,8 @@ public String getTriggerName(DataEventType dml, int maxTriggerNameLength, Trigge

public boolean syncTriggers(List<Trigger> triggers, ITriggerCreationListener listener, boolean force, boolean verifyInDatabase);

public boolean syncTriggers(List<Trigger> triggers, ITriggerCreationListener listener, boolean force, boolean verifyInDatabase, boolean useTableCache);

public boolean syncTriggers(Table table, boolean genAlways);

public boolean syncTriggers(List<Table> tables, boolean genAlways);
Expand Down
6 changes: 5 additions & 1 deletion symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/TriggerRouterService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2225,6 +2225,10 @@ public void syncTrigger(Trigger trigger, ITriggerCreationListener listener, bool
}

public boolean syncTriggers(List<Trigger> triggers, ITriggerCreationListener listener, boolean force, boolean verifyInDatabase) {
return syncTriggers(triggers, listener, force, verifyInDatabase, true);
}

public boolean syncTriggers(List<Trigger> triggers, ITriggerCreationListener listener, boolean force, boolean verifyInDatabase, boolean useTableCache) {
if (clusterService.lock(ClusterConstants.SYNC_TRIGGERS)) {
TriggerRouterContext context = new TriggerRouterContext();
long startTime = System.currentTimeMillis();
Expand Down Expand Up @@ -2291,7 +2295,7 @@ public boolean syncTriggers(List<Trigger> triggers, ITriggerCreationListener lis
}
}
Map<String, List<TriggerTableSupportingInfo>> triggerToTableSupportingInfo = getTriggerToTableSupportingInfo(
Collections.singletonList(trigger), allHistories, true, context);
Collections.singletonList(trigger), allHistories, useTableCache, context);
updateOrCreateDatabaseTrigger(trigger, triggersForCurrentNode, null, force,
verifyInDatabase, allHistories, false, triggerToTableSupportingInfo, context);
} else {
Expand Down
11 changes: 11 additions & 0 deletions symmetric-core/src/main/resources/symmetric-default.properties
View file
Open in desktop
Copy link
Contributor

@evan-miller-jumpmind evan-miller-jumpmind Nov 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should document that this parameter is only compatible with SQL Server.

Original file line number Diff line number Diff line change
Expand Up @@ -2402,6 +2402,17 @@ trigger.capture.ddl.delimiter=$
# Type: boolean
trigger.capture.ddl.check.trigger.hist=true

# Determines if the DDL captured will be the actual ddl run against the source or if a Create table event
# will be captured for the changed table. This allows cross compatibility where an add columnn syntax
# may vary from one dialect to another. By sending a full create or alter table it will naturally apply
# the proper alters to match the target.

# See: trigger.capture.ddl.changes
# DatabaseOverridable: false
# Tags: trigger
# Type: boolean
trigger.capture.ddl.send.table=false

# Enable or disabled use of create or replace syntax on Oracle and MS-SQL 2016 SP1 and newer.
#
# DatabaseOverridable: false
Expand Down
2 changes: 1 addition & 1 deletion symmetric-db/src/main/java/org/jumpmind/db/alter/ModelComparator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,7 @@ public List<TableChange> compareColumns(Table sourceTable, Column sourceColumn,
changes.add(new ColumnSizeChange(sourceTable, sourceColumn, targetColumn.getSizeAsInt(), targetColumn.getScale()));
}
}
if (supportsDefaultValues() && !defaultValuesAreEqual(sourceColumn, targetColumn)) {
if (supportsDefaultValues() && ddlBuilder.supportDefaultValues() && !defaultValuesAreEqual(sourceColumn, targetColumn)) {
log.info("The {} column on the {} table changed default value from {} to {} ", sourceColumn.getName(), sourceTable.getName(),
sourceColumn.getDefaultValue(), targetColumn.getDefaultValue());
changes.add(new ColumnDefaultValueChange(sourceTable, sourceColumn, targetColumn.getDefaultValue()));
Expand Down
5 changes: 5 additions & 0 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDdlBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2737,4 +2737,9 @@ public String getTriggerDelimiterReplacementCharacters() {
public void setTriggerDelimiterReplacementCharacters(String triggerDelimiterReplacementCharacters) {
this.triggerDelimiterReplacementCharacters = triggerDelimiterReplacementCharacters;
}

@Override
public boolean supportDefaultValues() {
return true;
}
}
2 changes: 2 additions & 0 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/IDdlBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,4 +86,6 @@ public interface IDdlBuilder {
public void setTriggerDelimiterReplacementCharacters(String triggerDelimiterReplacementCharacters);

public String getTriggerDelimiterReplacementCharacters();

public boolean supportDefaultValues();
}
13 changes: 8 additions & 5 deletions symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/writer/DefaultDatabaseWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ public class DefaultDatabaseWriter extends AbstractDatabaseWriter {
protected LogSqlBuilder logSqlBuilder = new LogSqlBuilder();
protected Boolean isCteExpression;
protected boolean hasUncommittedDdl;
protected List<Column> lookupKeys = null;
protected ArrayList<Column> changedColumnsList = new ArrayList<>();
boolean[] nullKeyValues = null;

public DefaultDatabaseWriter(IDatabasePlatform platform) {
this(platform, null, null);
Expand Down Expand Up @@ -346,7 +349,7 @@ protected LoadStatus delete(CsvData data, boolean useConflictDetection) {
if (requireNewStatement(DmlType.DELETE, data, useConflictDetection, useConflictDetection,
conflict.getDetectType())) {
lastUseConflictDetection = useConflictDetection;
List<Column> lookupKeys = null;
lookupKeys = null;
if (!useConflictDetection) {
lookupKeys = targetTable.getPrimaryKeyColumnsAsList();
} else {
Expand Down Expand Up @@ -409,7 +412,7 @@ protected LoadStatus delete(CsvData data, boolean useConflictDetection) {
throw new IllegalStateException(msg);
}
lookupDataMap = getLookupDataMap(data, conflict);
boolean[] nullKeyValues = new boolean[lookupKeys.size()];
nullKeyValues = new boolean[lookupKeys.size()];
for (int i = 0; i < lookupKeys.size(); i++) {
Column column = lookupKeys.get(i);
nullKeyValues[i] = !column.isRequired()
Expand Down Expand Up @@ -468,7 +471,7 @@ protected LoadStatus update(CsvData data, boolean applyChangesOnly, boolean useC
String[] rowData = getRowData(data, CsvData.ROW_DATA);
String[] oldData = getRowData(data, CsvData.OLD_DATA);
ArrayList<String> changedColumnValueList = new ArrayList<>();
ArrayList<Column> changedColumnsList = new ArrayList<>();
changedColumnsList = new ArrayList<>();
for (int i = 0; i < targetTable.getColumnCount(); i++) {
Column column = targetTable.getColumn(i);
if (column != null) {
Expand All @@ -485,7 +488,7 @@ protected LoadStatus update(CsvData data, boolean applyChangesOnly, boolean useC
useConflictDetection, conflict.getDetectType())) {
lastApplyChangesOnly = applyChangesOnly;
lastUseConflictDetection = useConflictDetection;
List<Column> lookupKeys = null;
lookupKeys = null;
if (!useConflictDetection) {
lookupKeys = targetTable.getPrimaryKeyColumnsAsList();
} else {
Expand Down Expand Up @@ -566,7 +569,7 @@ protected LoadStatus update(CsvData data, boolean applyChangesOnly, boolean useC
throw new IllegalStateException(msg);
}
lookupDataMap = getLookupDataMap(data, conflict);
boolean[] nullKeyValues = new boolean[lookupKeys.size()];
nullKeyValues = new boolean[lookupKeys.size()];
for (int i = 0; i < lookupKeys.size(); i++) {
Column column = lookupKeys.get(i);
// the isRequired is a bit of a hack. This nullKeyValues
Expand Down
Loading