From dec06cbbde3cfe1c731bfa17463376ceb59674a3 Mon Sep 17 00:00:00 2001 From: Mayura Deshmukh Date: Thu, 26 Oct 2023 16:12:00 +0530 Subject: [PATCH 1/3] MOSIP-29932 fixing auto update issue Signed-off-by: Mayura Deshmukh --- .../70.toolkit_biometric_scores.api | 15 ++++++--------- .../71.toolkit_biometric_scores_new.api | 9 ++++++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/kafka-connect/ref_connector_api_calls/70.toolkit_biometric_scores.api b/kafka-connect/ref_connector_api_calls/70.toolkit_biometric_scores.api index 4f44aa6..d161454 100644 --- a/kafka-connect/ref_connector_api_calls/70.toolkit_biometric_scores.api +++ b/kafka-connect/ref_connector_api_calls/70.toolkit_biometric_scores.api @@ -36,13 +36,7 @@ curl \ "behavior.on.null.values": "DELETE", - "transforms": "debezExtract,ext1,ext2,tsconvert01,stringtojson01,tsSelect", - - "transforms.debezExtract.type": "io.debezium.transforms.ExtractNewRecordState", - "transforms.debezExtract.delete.handling.mode": "drop", - "transforms.debezExtract.drop.tombstones": "false", - "transforms.debezExtract.add.fields": "source.ts_ms:ts_ms,table,lsn", - "transforms.debezExtract.add.fields.prefix": "source_", + "transforms": "ext1,ext2,tsconvert01,tsconvert02,stringtojson01,tsSelect", "transforms.ext1.type": "org.apache.kafka.connect.transforms.ExtractField$Value", "transforms.ext1.field": "payload", @@ -50,8 +44,11 @@ curl \ "transforms.ext2.field": "after", "transforms.tsconvert01.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", - "transforms.tsconvert01.field": "cr_dtimes", - "transforms.tsconvert01.input.type": "micro_sec", + "transforms.tsconvert01.field": "source_ts_ms" + + "transforms.tsconvert02.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", + "transforms.tsconvert02.field": "cr_dtimes", + "transforms.tsconvert02.input.type": "micro_sec", "transforms.stringtojson01.type": "io.mosip.kafka.connect.transforms.StringToJson$Value", "transforms.stringtojson01.input.field": "scores_json", diff --git a/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api b/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api index 234c362..6db9f5e 100644 --- a/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api +++ b/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api @@ -36,7 +36,7 @@ curl \ "behavior.on.null.values": "DELETE", - "transforms": "debezExtract,tsconvert01,tsSelect", + "transforms": "debezExtract,tsconvert01,tsconvert02,tsSelect", "transforms.debezExtract.type": "io.debezium.transforms.ExtractNewRecordState", "transforms.debezExtract.delete.handling.mode": "drop", @@ -45,8 +45,11 @@ curl \ "transforms.debezExtract.add.fields.prefix": "source_", "transforms.tsconvert01.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", - "transforms.tsconvert01.field": "cr_dtimes", - "transforms.tsconvert01.input.type": "micro_sec", + "transforms.tsconvert01.field": "source_ts_ms" + + "transforms.tsconvert02.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", + "transforms.tsconvert02.field": "cr_dtimes", + "transforms.tsconvert02.input.type": "micro_sec", "transforms.tsSelect.type": "io.mosip.kafka.connect.transforms.TimestampSelector$Value", "transforms.tsSelect.ts.order": "cr_dtimes", From 0bb7a8f66c85c5807f8e110521859ba9ff3a744a Mon Sep 17 00:00:00 2001 From: Mayura Deshmukh Date: Thu, 26 Oct 2023 21:57:14 +0530 Subject: [PATCH 2/3] MOSIP-29932 added missing comma Signed-off-by: Mayura Deshmukh --- .../ref_connector_api_calls/70.toolkit_biometric_scores.api | 2 +- .../ref_connector_api_calls/71.toolkit_biometric_scores_new.api | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka-connect/ref_connector_api_calls/70.toolkit_biometric_scores.api b/kafka-connect/ref_connector_api_calls/70.toolkit_biometric_scores.api index d161454..7db9420 100644 --- a/kafka-connect/ref_connector_api_calls/70.toolkit_biometric_scores.api +++ b/kafka-connect/ref_connector_api_calls/70.toolkit_biometric_scores.api @@ -44,7 +44,7 @@ curl \ "transforms.ext2.field": "after", "transforms.tsconvert01.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", - "transforms.tsconvert01.field": "source_ts_ms" + "transforms.tsconvert01.field": "source_ts_ms", "transforms.tsconvert02.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", "transforms.tsconvert02.field": "cr_dtimes", diff --git a/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api b/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api index 6db9f5e..100e2d7 100644 --- a/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api +++ b/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api @@ -45,7 +45,7 @@ curl \ "transforms.debezExtract.add.fields.prefix": "source_", "transforms.tsconvert01.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", - "transforms.tsconvert01.field": "source_ts_ms" + "transforms.tsconvert01.field": "source_ts_ms", "transforms.tsconvert02.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", "transforms.tsconvert02.field": "cr_dtimes", From 0a02927e906f2c82f2bf8c60d10a4bb740e511c1 Mon Sep 17 00:00:00 2001 From: Mayura Deshmukh Date: Fri, 27 Oct 2023 13:41:16 +0530 Subject: [PATCH 3/3] MOSIP-29932 removed test file Signed-off-by: Mayura Deshmukh --- .../71.toolkit_biometric_scores_new.api | 59 ------------------- scripts/values-init.yaml | 2 +- 2 files changed, 1 insertion(+), 60 deletions(-) delete mode 100644 kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api diff --git a/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api b/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api deleted file mode 100644 index 100e2d7..0000000 --- a/kafka-connect/ref_connector_api_calls/71.toolkit_biometric_scores_new.api +++ /dev/null @@ -1,59 +0,0 @@ -# these following will be taken from env -#DB_USER= -#DB_PORT= -#DB_HOSTNAME= -#DB_PASS= -#DB_PREFIX_INDEX= -#ES_URL= - -CONN_NAME="toolkit_biometric_scores_new_$DB_PREFIX_INDEX"; # change this.. give unique name for each db/table - -ES_CONN_URL='es-connect:8083'; # needn't change .. this is the ES-connector service name - -DB_NAME='mosip_toolkit'; # change this -DB_TABLES='toolkit.biometric_scores_new' -ES_INDICES=$(echo $DB_TABLES | sed -E "s/([^,]+)/$DB_PREFIX_INDEX.\1/g") - -curl \ - -X POST \ - http://$ES_CONN_URL/connectors \ - -H 'Content-Type: application/json' \ - -d \ - '{ - "name": '\"$CONN_NAME\"', - "config": { - "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector", - "name": '\"$CONN_NAME\"', - "connection.url": '\"$ES_URL\"', - "tasks.max": "1", - "topics": '\"$ES_INDICES\"', - "key.ignore": "false", - "schema.ignore": "true", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "key.converter.schemas.enable": "false", - "value.converter.schemas.enable": "true", - - "behavior.on.null.values": "DELETE", - - "transforms": "debezExtract,tsconvert01,tsconvert02,tsSelect", - - "transforms.debezExtract.type": "io.debezium.transforms.ExtractNewRecordState", - "transforms.debezExtract.delete.handling.mode": "drop", - "transforms.debezExtract.drop.tombstones": "false", - "transforms.debezExtract.add.fields": "source.ts_ms:ts_ms,table,lsn", - "transforms.debezExtract.add.fields.prefix": "source_", - - "transforms.tsconvert01.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", - "transforms.tsconvert01.field": "source_ts_ms", - - "transforms.tsconvert02.type": "io.mosip.kafka.connect.transforms.TimestampConverterAdv$Value", - "transforms.tsconvert02.field": "cr_dtimes", - "transforms.tsconvert02.input.type": "micro_sec", - - "transforms.tsSelect.type": "io.mosip.kafka.connect.transforms.TimestampSelector$Value", - "transforms.tsSelect.ts.order": "cr_dtimes", - "transforms.tsSelect.output.field": "@timestamp_gen" - - } -}'; diff --git a/scripts/values-init.yaml b/scripts/values-init.yaml index a305f0a..1f551cb 100644 --- a/scripts/values-init.yaml +++ b/scripts/values-init.yaml @@ -26,7 +26,7 @@ debezium_connectors: - name: "mosip_idrepo" tables: "idrepo.anonymous_profile,idrepo.channel_info" - name: "mosip_toolkit" - tables: "toolkit.sbi_projects,toolkit.sdk_projects,toolkit.abis_projects,toolkit.collections,toolkit.collection_testcase_mapping,toolkit.testcase,toolkit.test_run,toolkit.test_run_details,toolkit.biometric_scores,toolkit.biometric_scores_new" + tables: "toolkit.sbi_projects,toolkit.sdk_projects,toolkit.abis_projects,toolkit.collections,toolkit.collection_testcase_mapping,toolkit.testcase,toolkit.test_run,toolkit.test_run_details,toolkit.biometric_scores" es_kafka_connectors: enabled: true