Data Transfer API, REST: Endpoint methods
JSON Representation
{
"id": "string",
"folderId": "string",
"name": "string",
"description": "string",
"labels": "object",
"settings": {
// `settings` includes only one of the fields `mysqlSource`, `postgresSource`, `kafkaSource`, `mongoSource`, `clickhouseSource`, `mysqlTarget`, `postgresTarget`, `clickhouseTarget`, `kafkaTarget`, `mongoTarget`
"mysqlSource": {
"connection": {
// `settings.mysqlSource.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"hosts": [
"string"
],
"port": "string",
"tlsMode": {
// `settings.mysqlSource.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`settings.mysqlSource.connection.onPremise.tlsMode`
},
"subnetId": "string"
},
// end of the list of possible fields`settings.mysqlSource.connection`
},
"securityGroups": [
"string"
],
"database": "string",
"serviceDatabase": "string",
"user": "string",
"password": {
"raw": "string"
},
"includeTablesRegex": [
"string"
],
"excludeTablesRegex": [
"string"
],
"timezone": "string",
"objectTransferSettings": {
"view": "string",
"routine": "string",
"trigger": "string",
"tables": "string"
}
},
"postgresSource": {
"connection": {
// `settings.postgresSource.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"hosts": [
"string"
],
"port": "string",
"tlsMode": {
// `settings.postgresSource.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`settings.postgresSource.connection.onPremise.tlsMode`
},
"subnetId": "string"
},
// end of the list of possible fields`settings.postgresSource.connection`
},
"securityGroups": [
"string"
],
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
"includeTables": [
"string"
],
"excludeTables": [
"string"
],
"slotByteLagLimit": "string",
"serviceSchema": "string",
"objectTransferSettings": {
"sequence": "string",
"sequenceOwnedBy": "string",
"sequenceSet": "string",
"table": "string",
"primaryKey": "string",
"fkConstraint": "string",
"defaultValues": "string",
"constraint": "string",
"index": "string",
"view": "string",
"materializedView": "string",
"function": "string",
"trigger": "string",
"type": "string",
"rule": "string",
"collation": "string",
"policy": "string",
"cast": "string"
}
},
"kafkaSource": {
"connection": {
// `settings.kafkaSource.connection` includes only one of the fields `clusterId`, `onPremise`
"clusterId": "string",
"onPremise": {
"brokerUrls": [
"string"
],
"tlsMode": {
// `settings.kafkaSource.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`settings.kafkaSource.connection.onPremise.tlsMode`
},
"subnetId": "string"
},
// end of the list of possible fields`settings.kafkaSource.connection`
},
"auth": {
// `settings.kafkaSource.auth` includes only one of the fields `sasl`, `noAuth`
"sasl": {
"user": "string",
"password": {
"raw": "string"
},
"mechanism": "string"
},
"noAuth": {},
// end of the list of possible fields`settings.kafkaSource.auth`
},
"securityGroups": [
"string"
],
"topicName": "string",
"transformer": {
"cloudFunction": "string",
"serviceAccountId": "string",
"numberOfRetries": "string",
"bufferSize": "string",
"bufferFlushInterval": "string",
"invocationTimeout": "string"
},
"parser": {
// `settings.kafkaSource.parser` includes only one of the fields `jsonParser`, `auditTrailsV1Parser`, `cloudLoggingParser`, `tskvParser`
"jsonParser": {
"dataSchema": {
// `settings.kafkaSource.parser.jsonParser.dataSchema` includes only one of the fields `fields`, `jsonFields`
"fields": {
"fields": [
{
"name": "string",
"type": "string",
"key": true,
"required": true,
"path": "string"
}
]
},
"jsonFields": "string",
// end of the list of possible fields`settings.kafkaSource.parser.jsonParser.dataSchema`
},
"nullKeysAllowed": true,
"addRestColumn": true
},
"auditTrailsV1Parser": {},
"cloudLoggingParser": {},
"tskvParser": {
"dataSchema": {
// `settings.kafkaSource.parser.tskvParser.dataSchema` includes only one of the fields `fields`, `jsonFields`
"fields": {
"fields": [
{
"name": "string",
"type": "string",
"key": true,
"required": true,
"path": "string"
}
]
},
"jsonFields": "string",
// end of the list of possible fields`settings.kafkaSource.parser.tskvParser.dataSchema`
},
"nullKeysAllowed": true,
"addRestColumn": true
},
// end of the list of possible fields`settings.kafkaSource.parser`
}
},
"mongoSource": {
"connection": {
"connectionOptions": {
"user": "string",
"password": {
"raw": "string"
},
"authSource": "string",
// `settings.mongoSource.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"hosts": [
"string"
],
"port": "string",
"tlsMode": {
// `settings.mongoSource.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`settings.mongoSource.connection.connectionOptions.onPremise.tlsMode`
},
"replicaSet": "string"
},
// end of the list of possible fields`settings.mongoSource.connection.connectionOptions`
}
},
"subnetId": "string",
"securityGroups": [
"string"
],
"collections": [
{
"databaseName": "string",
"collectionName": "string"
}
],
"excludedCollections": [
{
"databaseName": "string",
"collectionName": "string"
}
],
"secondaryPreferredMode": true
},
"clickhouseSource": {
"connection": {
"connectionOptions": {
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
// `settings.clickhouseSource.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"shards": [
{
"name": "string",
"hosts": [
"string"
]
}
],
"httpPort": "string",
"nativePort": "string",
"tlsMode": {
// `settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode`
}
},
// end of the list of possible fields`settings.clickhouseSource.connection.connectionOptions`
}
},
"subnetId": "string",
"securityGroups": [
"string"
],
"includeTables": [
"string"
],
"excludeTables": [
"string"
]
},
"mysqlTarget": {
"connection": {
// `settings.mysqlTarget.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"hosts": [
"string"
],
"port": "string",
"tlsMode": {
// `settings.mysqlTarget.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`settings.mysqlTarget.connection.onPremise.tlsMode`
},
"subnetId": "string"
},
// end of the list of possible fields`settings.mysqlTarget.connection`
},
"securityGroups": [
"string"
],
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
"sqlMode": "string",
"skipConstraintChecks": true,
"timezone": "string",
"cleanupPolicy": "string",
"serviceDatabase": "string"
},
"postgresTarget": {
"connection": {
// `settings.postgresTarget.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"hosts": [
"string"
],
"port": "string",
"tlsMode": {
// `settings.postgresTarget.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`settings.postgresTarget.connection.onPremise.tlsMode`
},
"subnetId": "string"
},
// end of the list of possible fields`settings.postgresTarget.connection`
},
"securityGroups": [
"string"
],
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
"cleanupPolicy": "string"
},
"clickhouseTarget": {
"connection": {
"connectionOptions": {
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
// `settings.clickhouseTarget.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"shards": [
{
"name": "string",
"hosts": [
"string"
]
}
],
"httpPort": "string",
"nativePort": "string",
"tlsMode": {
// `settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode`
}
},
// end of the list of possible fields`settings.clickhouseTarget.connection.connectionOptions`
}
},
"subnetId": "string",
"securityGroups": [
"string"
],
"clickhouseClusterName": "string",
"altNames": [
{
"fromName": "string",
"toName": "string"
}
],
"sharding": {
// `settings.clickhouseTarget.sharding` includes only one of the fields `columnValueHash`, `customMapping`, `transferId`
"columnValueHash": {
"columnName": "string"
},
"customMapping": {
"columnName": "string",
"mapping": [
{
"columnValue": {
"stringValue": "string"
},
"shardName": "string"
}
]
},
"transferId": "object",
// end of the list of possible fields`settings.clickhouseTarget.sharding`
},
"cleanupPolicy": "string"
},
"kafkaTarget": {
"connection": {
// `settings.kafkaTarget.connection` includes only one of the fields `clusterId`, `onPremise`
"clusterId": "string",
"onPremise": {
"brokerUrls": [
"string"
],
"tlsMode": {
// `settings.kafkaTarget.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`settings.kafkaTarget.connection.onPremise.tlsMode`
},
"subnetId": "string"
},
// end of the list of possible fields`settings.kafkaTarget.connection`
},
"auth": {
// `settings.kafkaTarget.auth` includes only one of the fields `sasl`, `noAuth`
"sasl": {
"user": "string",
"password": {
"raw": "string"
},
"mechanism": "string"
},
"noAuth": {},
// end of the list of possible fields`settings.kafkaTarget.auth`
},
"securityGroups": [
"string"
],
"topicSettings": {
// `settings.kafkaTarget.topicSettings` includes only one of the fields `topic`, `topicPrefix`
"topic": {
"topicName": "string",
"saveTxOrder": true
},
"topicPrefix": "string",
// end of the list of possible fields`settings.kafkaTarget.topicSettings`
}
},
"mongoTarget": {
"connection": {
"connectionOptions": {
"user": "string",
"password": {
"raw": "string"
},
"authSource": "string",
// `settings.mongoTarget.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"hosts": [
"string"
],
"port": "string",
"tlsMode": {
// `settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode`
},
"replicaSet": "string"
},
// end of the list of possible fields`settings.mongoTarget.connection.connectionOptions`
}
},
"subnetId": "string",
"securityGroups": [
"string"
],
"database": "string",
"cleanupPolicy": "string"
},
// end of the list of possible fields`settings`
}
}
Field | Description |
---|---|
id | string |
folderId | string |
name | string |
description | string |
labels | object |
settings | object |
settings. mysqlSource |
object settings includes only one of the fields mysqlSource , postgresSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , kafkaTarget , mongoTarget |
settings. mysqlSource. connection |
object Database connection settings |
settings. mysqlSource. connection. mdbClusterId |
string settings.mysqlSource.connection includes only one of the fields mdbClusterId , onPremise Managed Service for MySQL cluster ID |
settings. mysqlSource. connection. onPremise |
object settings.mysqlSource.connection includes only one of the fields mdbClusterId , onPremise |
settings. mysqlSource. connection. onPremise. hosts[] |
string |
settings. mysqlSource. connection. onPremise. port |
string (int64) Database port |
settings. mysqlSource. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
settings. mysqlSource. connection. onPremise. tlsMode. disabled |
object settings.mysqlSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. mysqlSource. connection. onPremise. tlsMode. disabled. disabled |
object settings.mysqlSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. mysqlSource. connection. onPremise. tlsMode. enabled |
object settings.mysqlSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
settings. mysqlSource. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
settings. mysqlSource. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
settings. mysqlSource. securityGroups[] |
string Security groups |
settings. mysqlSource. database |
string Database name You can leave it empty, then it will be possible to transfer tables from several databases at the same time from this source. |
settings. mysqlSource. serviceDatabase |
string Database for service tables Default: data source database. Here created technical tables (__tm_keeper, __tm_gtid_keeper). |
settings. mysqlSource. user |
string User for database access. |
settings. mysqlSource. password |
object Password for database access. |
settings. mysqlSource. password. raw |
string Raw secret value |
settings. mysqlSource. includeTablesRegex[] |
string |
settings. mysqlSource. excludeTablesRegex[] |
string |
settings. mysqlSource. timezone |
string Database timezone Is used for parsing timestamps for saving source timezones. Accepts values from IANA timezone database. Default: local timezone. |
settings. mysqlSource. objectTransferSettings |
object Schema migration Select database objects to be transferred during activation or deactivation. |
settings. mysqlSource. objectTransferSettings. view |
string Views CREATE VIEW ...
|
settings. mysqlSource. objectTransferSettings. routine |
string Routines CREATE PROCEDURE ... ; CREATE FUNCTION ... ;
|
settings. mysqlSource. objectTransferSettings. trigger |
string Triggers CREATE TRIGGER ...
|
settings. mysqlSource. objectTransferSettings. tables |
string
|
settings. postgresSource |
object settings includes only one of the fields mysqlSource , postgresSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , kafkaTarget , mongoTarget |
settings. postgresSource. connection |
object Database connection settings |
settings. postgresSource. connection. mdbClusterId |
string settings.postgresSource.connection includes only one of the fields mdbClusterId , onPremise Managed Service for PostgreSQL cluster ID |
settings. postgresSource. connection. onPremise |
object settings.postgresSource.connection includes only one of the fields mdbClusterId , onPremise |
settings. postgresSource. connection. onPremise. hosts[] |
string |
settings. postgresSource. connection. onPremise. port |
string (int64) Will be used if the cluster ID is not specified. |
settings. postgresSource. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
settings. postgresSource. connection. onPremise. tlsMode. disabled |
object settings.postgresSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. postgresSource. connection. onPremise. tlsMode. disabled. disabled |
object settings.postgresSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. postgresSource. connection. onPremise. tlsMode. enabled |
object settings.postgresSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
settings. postgresSource. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
settings. postgresSource. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
settings. postgresSource. securityGroups[] |
string Security groups |
settings. postgresSource. database |
string Database name |
settings. postgresSource. user |
string User for database access. |
settings. postgresSource. password |
object Password for database access. |
settings. postgresSource. password. raw |
string Raw secret value |
settings. postgresSource. includeTables[] |
string Included tables If none or empty list is presented, all tables are replicated. Full table name with schema. Can contain schema_name.* patterns. |
settings. postgresSource. excludeTables[] |
string Excluded tables If none or empty list is presented, all tables are replicated. Full table name with schema. Can contain schema_name.* patterns. |
settings. postgresSource. slotByteLagLimit |
string (int64) Maximum lag of replication slot (in bytes); after exceeding this limit replication will be aborted. |
settings. postgresSource. serviceSchema |
string Database schema for service tables (__consumer_keeper, __data_transfer_mole_finder). Default is public |
settings. postgresSource. objectTransferSettings |
object Select database objects to be transferred during activation or deactivation. |
settings. postgresSource. objectTransferSettings. sequence |
string Sequences CREATE SEQUENCE ...
|
settings. postgresSource. objectTransferSettings. sequenceOwnedBy |
string Owned sequences CREATE SEQUENCE ... OWNED BY ...
|
settings. postgresSource. objectTransferSettings. sequenceSet |
string
|
settings. postgresSource. objectTransferSettings. table |
string Tables CREATE TABLE ...
|
settings. postgresSource. objectTransferSettings. primaryKey |
string Primary keys ALTER TABLE ... ADD PRIMARY KEY ...
|
settings. postgresSource. objectTransferSettings. fkConstraint |
string Foreign keys ALTER TABLE ... ADD FOREIGN KEY ...
|
settings. postgresSource. objectTransferSettings. defaultValues |
string Default values ALTER TABLE ... ALTER COLUMN ... SET DEFAULT ...
|
settings. postgresSource. objectTransferSettings. constraint |
string Constraints ALTER TABLE ... ADD CONSTRAINT ...
|
settings. postgresSource. objectTransferSettings. index |
string Indexes CREATE INDEX ...
|
settings. postgresSource. objectTransferSettings. view |
string Views CREATE VIEW ...
|
settings. postgresSource. objectTransferSettings. materializedView |
string Materialized views CREATE MATERIALIZED VIEW ...
|
settings. postgresSource. objectTransferSettings. function |
string Functions CREATE FUNCTION ...
|
settings. postgresSource. objectTransferSettings. trigger |
string Triggers CREATE TRIGGER ...
|
settings. postgresSource. objectTransferSettings. type |
string Types CREATE TYPE ...
|
settings. postgresSource. objectTransferSettings. rule |
string Rules CREATE RULE ...
|
settings. postgresSource. objectTransferSettings. collation |
string Collations CREATE COLLATION ...
|
settings. postgresSource. objectTransferSettings. policy |
string Policies CREATE POLICY ...
|
settings. postgresSource. objectTransferSettings. cast |
string Casts CREATE CAST ...
|
settings. kafkaSource |
object settings includes only one of the fields mysqlSource , postgresSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , kafkaTarget , mongoTarget |
settings. kafkaSource. connection |
object Connection settings |
settings. kafkaSource. connection. clusterId |
string settings.kafkaSource.connection includes only one of the fields clusterId , onPremise Managed Service for Kafka cluster ID |
settings. kafkaSource. connection. onPremise |
object settings.kafkaSource.connection includes only one of the fields clusterId , onPremise |
settings. kafkaSource. connection. onPremise. brokerUrls[] |
string Kafka broker URLs |
settings. kafkaSource. connection. onPremise. tlsMode |
object TLS settings for broker connection. Disabled by default. |
settings. kafkaSource. connection. onPremise. tlsMode. disabled |
object settings.kafkaSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. kafkaSource. connection. onPremise. tlsMode. disabled. disabled |
object settings.kafkaSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. kafkaSource. connection. onPremise. tlsMode. enabled |
object settings.kafkaSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
settings. kafkaSource. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
settings. kafkaSource. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
settings. kafkaSource. auth |
object Authentication settings |
settings. kafkaSource. auth. sasl |
object settings.kafkaSource.auth includes only one of the fields sasl , noAuth |
settings. kafkaSource. auth. sasl. user |
string User name |
settings. kafkaSource. auth. sasl. password |
object Password for user |
settings. kafkaSource. auth. sasl. password. raw |
string Raw secret value |
settings. kafkaSource. auth. sasl. mechanism |
string SASL mechanism for authentication |
settings. kafkaSource. auth. noAuth |
object settings.kafkaSource.auth includes only one of the fields sasl , noAuth No authentication |
settings. kafkaSource. securityGroups[] |
string Security groups |
settings. kafkaSource. topicName |
string Full source topic name |
settings. kafkaSource. transformer |
object Data transformation rules |
settings. kafkaSource. transformer. cloudFunction |
string Cloud function |
settings. kafkaSource. transformer. serviceAccountId |
string Service account |
settings. kafkaSource. transformer. numberOfRetries |
string (int64) Number of retries |
settings. kafkaSource. transformer. bufferSize |
string Buffer size for function |
settings. kafkaSource. transformer. bufferFlushInterval |
string Flush interval |
settings. kafkaSource. transformer. invocationTimeout |
string Invocation timeout |
settings. kafkaSource. parser |
object Data parsing rules |
settings. kafkaSource. parser. jsonParser |
object settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
settings. kafkaSource. parser. jsonParser. dataSchema |
object |
settings. kafkaSource. parser. jsonParser. dataSchema. fields |
object settings.kafkaSource.parser.jsonParser.dataSchema includes only one of the fields fields , jsonFields |
settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[] |
object Column schema |
settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. name |
string |
settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. type |
string |
settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. key |
boolean (boolean) |
settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. required |
boolean (boolean) |
settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. path |
string |
settings. kafkaSource. parser. jsonParser. dataSchema. jsonFields |
string settings.kafkaSource.parser.jsonParser.dataSchema includes only one of the fields fields , jsonFields |
settings. kafkaSource. parser. jsonParser. nullKeysAllowed |
boolean (boolean) Allow null keys, if no - null keys will be putted to unparsed data |
settings. kafkaSource. parser. jsonParser. addRestColumn |
boolean (boolean) Will add _rest column for all unknown fields |
settings. kafkaSource. parser. auditTrailsV1Parser |
object settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
settings. kafkaSource. parser. cloudLoggingParser |
object settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
settings. kafkaSource. parser. tskvParser |
object settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
settings. kafkaSource. parser. tskvParser. dataSchema |
object |
settings. kafkaSource. parser. tskvParser. dataSchema. fields |
object settings.kafkaSource.parser.tskvParser.dataSchema includes only one of the fields fields , jsonFields |
settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[] |
object Column schema |
settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. name |
string |
settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. type |
string |
settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. key |
boolean (boolean) |
settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. required |
boolean (boolean) |
settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. path |
string |
settings. kafkaSource. parser. tskvParser. dataSchema. jsonFields |
string settings.kafkaSource.parser.tskvParser.dataSchema includes only one of the fields fields , jsonFields |
settings. kafkaSource. parser. tskvParser. nullKeysAllowed |
boolean (boolean) Allow null keys, if no - null keys will be putted to unparsed data |
settings. kafkaSource. parser. tskvParser. addRestColumn |
boolean (boolean) Will add _rest column for all unknown fields |
settings. mongoSource |
object settings includes only one of the fields mysqlSource , postgresSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , kafkaTarget , mongoTarget |
settings. mongoSource. connection |
object |
settings. mongoSource. connection. connectionOptions |
object |
settings. mongoSource. connection. connectionOptions. user |
string User name |
settings. mongoSource. connection. connectionOptions. password |
object |
settings. mongoSource. connection. connectionOptions. password. raw |
string Raw secret value |
settings. mongoSource. connection. connectionOptions. authSource |
string Database name associated with the credentials |
settings. mongoSource. connection. connectionOptions. mdbClusterId |
string settings.mongoSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
settings. mongoSource. connection. connectionOptions. onPremise |
object settings.mongoSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
settings. mongoSource. connection. connectionOptions. onPremise. hosts[] |
string |
settings. mongoSource. connection. connectionOptions. onPremise. port |
string (int64) |
settings. mongoSource. connection. connectionOptions. onPremise. tlsMode |
object |
settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. disabled |
object settings.mongoSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object settings.mongoSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. enabled |
object settings.mongoSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
settings. mongoSource. connection. connectionOptions. onPremise. replicaSet |
string |
settings. mongoSource. subnetId |
string |
settings. mongoSource. securityGroups[] |
string Security groups |
settings. mongoSource. collections[] |
object List of collections for replication. Empty list implies replication of all tables on the deployment. Allowed to use * as collection name. |
settings. mongoSource. collections[]. databaseName |
string |
settings. mongoSource. collections[]. collectionName |
string |
settings. mongoSource. excludedCollections[] |
object List of forbidden collections for replication. Allowed to use * as collection name for forbid all collections of concrete schema. |
settings. mongoSource. excludedCollections[]. databaseName |
string |
settings. mongoSource. excludedCollections[]. collectionName |
string |
settings. mongoSource. secondaryPreferredMode |
boolean (boolean) Read mode for mongo client |
settings. clickhouseSource |
object settings includes only one of the fields mysqlSource , postgresSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , kafkaTarget , mongoTarget |
settings. clickhouseSource. connection |
object |
settings. clickhouseSource. connection. connectionOptions |
object |
settings. clickhouseSource. connection. connectionOptions. database |
string Database |
settings. clickhouseSource. connection. connectionOptions. user |
string |
settings. clickhouseSource. connection. connectionOptions. password |
object Password for user |
settings. clickhouseSource. connection. connectionOptions. password. raw |
string Raw secret value |
settings. clickhouseSource. connection. connectionOptions. mdbClusterId |
string settings.clickhouseSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
settings. clickhouseSource. connection. connectionOptions. onPremise |
object settings.clickhouseSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
settings. clickhouseSource. connection. connectionOptions. onPremise. shards[] |
object |
settings. clickhouseSource. connection. connectionOptions. onPremise. shards[]. name |
string |
settings. clickhouseSource. connection. connectionOptions. onPremise. shards[]. hosts[] |
string |
settings. clickhouseSource. connection. connectionOptions. onPremise. httpPort |
string (int64) |
settings. clickhouseSource. connection. connectionOptions. onPremise. nativePort |
string (int64) |
settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode |
object |
settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. disabled |
object settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. enabled |
object settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
settings. clickhouseSource. subnetId |
string |
settings. clickhouseSource. securityGroups[] |
string |
settings. clickhouseSource. includeTables[] |
string While list of tables for replication. If none or empty list is presented - will replicate all tables. Can contain * patterns. |
settings. clickhouseSource. excludeTables[] |
string Exclude list of tables for replication. If none or empty list is presented - will replicate all tables. Can contain * patterns. |
settings. mysqlTarget |
object settings includes only one of the fields mysqlSource , postgresSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , kafkaTarget , mongoTarget |
settings. mysqlTarget. connection |
object Database connection settings |
settings. mysqlTarget. connection. mdbClusterId |
string settings.mysqlTarget.connection includes only one of the fields mdbClusterId , onPremise Managed Service for MySQL cluster ID |
settings. mysqlTarget. connection. onPremise |
object Connection options for on-premise MySQL settings.mysqlTarget.connection includes only one of the fields mdbClusterId , onPremise |
settings. mysqlTarget. connection. onPremise. hosts[] |
string |
settings. mysqlTarget. connection. onPremise. port |
string (int64) Database port |
settings. mysqlTarget. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
settings. mysqlTarget. connection. onPremise. tlsMode. disabled |
object settings.mysqlTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. mysqlTarget. connection. onPremise. tlsMode. disabled. disabled |
object settings.mysqlTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. mysqlTarget. connection. onPremise. tlsMode. enabled |
object settings.mysqlTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
settings. mysqlTarget. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
settings. mysqlTarget. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
settings. mysqlTarget. securityGroups[] |
string Security groups |
settings. mysqlTarget. database |
string Database name Allowed to leave it empty, then the tables will be created in databases with the same names as on the source. If this field is empty, then you must fill below db schema for service table. |
settings. mysqlTarget. user |
string User for database access. |
settings. mysqlTarget. password |
object Password for database access. |
settings. mysqlTarget. password. raw |
string Raw secret value |
settings. mysqlTarget. sqlMode |
string Default: NO_AUTO_VALUE_ON_ZERO,NO_DIR_IN_CREATE,NO_ENGINE_SUBSTITUTION. |
settings. mysqlTarget. skipConstraintChecks |
boolean (boolean) Disable constraints checks Recommend to disable for increase replication speed, but if schema contain cascading operations we don't recommend to disable. This option set FOREIGN_KEY_CHECKS=0 and UNIQUE_CHECKS=0. |
settings. mysqlTarget. timezone |
string Database timezone Is used for parsing timestamps for saving source timezones. Accepts values from IANA timezone database. Default: local timezone. |
settings. mysqlTarget. cleanupPolicy |
string Cleanup policy Cleanup policy for activate, reactivate and reupload processes. Default is DISABLED.
|
settings. mysqlTarget. serviceDatabase |
string Database schema for service table Default: db name. Here created technical tables (__tm_keeper, __tm_gtid_keeper). |
settings. postgresTarget |
object settings includes only one of the fields mysqlSource , postgresSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , kafkaTarget , mongoTarget |
settings. postgresTarget. connection |
object Database connection settings |
settings. postgresTarget. connection. mdbClusterId |
string settings.postgresTarget.connection includes only one of the fields mdbClusterId , onPremise Managed Service for PostgreSQL cluster ID |
settings. postgresTarget. connection. onPremise |
object Connection options for on-premise PostgreSQL settings.postgresTarget.connection includes only one of the fields mdbClusterId , onPremise |
settings. postgresTarget. connection. onPremise. hosts[] |
string |
settings. postgresTarget. connection. onPremise. port |
string (int64) Will be used if the cluster ID is not specified. |
settings. postgresTarget. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
settings. postgresTarget. connection. onPremise. tlsMode. disabled |
object settings.postgresTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. postgresTarget. connection. onPremise. tlsMode. disabled. disabled |
object settings.postgresTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. postgresTarget. connection. onPremise. tlsMode. enabled |
object settings.postgresTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
settings. postgresTarget. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
settings. postgresTarget. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
settings. postgresTarget. securityGroups[] |
string Security groups |
settings. postgresTarget. database |
string Database name |
settings. postgresTarget. user |
string User for database access. |
settings. postgresTarget. password |
object Password for database access. |
settings. postgresTarget. password. raw |
string Raw secret value |
settings. postgresTarget. cleanupPolicy |
string Cleanup policy for activate, reactivate and reupload processes. Default is truncate.
|
settings. clickhouseTarget |
object settings includes only one of the fields mysqlSource , postgresSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , kafkaTarget , mongoTarget |
settings. clickhouseTarget. connection |
object |
settings. clickhouseTarget. connection. connectionOptions |
object |
settings. clickhouseTarget. connection. connectionOptions. database |
string Database |
settings. clickhouseTarget. connection. connectionOptions. user |
string |
settings. clickhouseTarget. connection. connectionOptions. password |
object Password for user |
settings. clickhouseTarget. connection. connectionOptions. password. raw |
string Raw secret value |
settings. clickhouseTarget. connection. connectionOptions. mdbClusterId |
string settings.clickhouseTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
settings. clickhouseTarget. connection. connectionOptions. onPremise |
object settings.clickhouseTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
settings. clickhouseTarget. connection. connectionOptions. onPremise. shards[] |
object |
settings. clickhouseTarget. connection. connectionOptions. onPremise. shards[]. name |
string |
settings. clickhouseTarget. connection. connectionOptions. onPremise. shards[]. hosts[] |
string |
settings. clickhouseTarget. connection. connectionOptions. onPremise. httpPort |
string (int64) |
settings. clickhouseTarget. connection. connectionOptions. onPremise. nativePort |
string (int64) |
settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode |
object |
settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. disabled |
object settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. enabled |
object settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
settings. clickhouseTarget. subnetId |
string |
settings. clickhouseTarget. securityGroups[] |
string |
settings. clickhouseTarget. clickhouseClusterName |
string |
settings. clickhouseTarget. altNames[] |
object Alternative table names in target |
settings. clickhouseTarget. altNames[]. fromName |
string Source table name |
settings. clickhouseTarget. altNames[]. toName |
string Target table name |
settings. clickhouseTarget. sharding |
object |
settings. clickhouseTarget. sharding. columnValueHash |
object settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId |
settings. clickhouseTarget. sharding. columnValueHash. columnName |
string |
settings. clickhouseTarget. sharding. customMapping |
object settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId |
settings. clickhouseTarget. sharding. customMapping. columnName |
string |
settings. clickhouseTarget. sharding. customMapping. mapping[] |
object |
settings. clickhouseTarget. sharding. customMapping. mapping[]. columnValue |
object |
settings. clickhouseTarget. sharding. customMapping. mapping[]. columnValue. stringValue |
string |
settings. clickhouseTarget. sharding. customMapping. mapping[]. shardName |
string |
settings. clickhouseTarget. sharding. transferId |
object settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId Empty JSON object |
settings. clickhouseTarget. sharding. transferId. transferId |
object settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId Empty JSON object |
settings. clickhouseTarget. cleanupPolicy |
string |
settings. kafkaTarget |
object settings includes only one of the fields mysqlSource , postgresSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , kafkaTarget , mongoTarget |
settings. kafkaTarget. connection |
object Connection settings |
settings. kafkaTarget. connection. clusterId |
string settings.kafkaTarget.connection includes only one of the fields clusterId , onPremise Managed Service for Kafka cluster ID |
settings. kafkaTarget. connection. onPremise |
object Connection options for on-premise Kafka settings.kafkaTarget.connection includes only one of the fields clusterId , onPremise |
settings. kafkaTarget. connection. onPremise. brokerUrls[] |
string Kafka broker URLs |
settings. kafkaTarget. connection. onPremise. tlsMode |
object TLS settings for broker connection. Disabled by default. |
settings. kafkaTarget. connection. onPremise. tlsMode. disabled |
object settings.kafkaTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. kafkaTarget. connection. onPremise. tlsMode. disabled. disabled |
object settings.kafkaTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. kafkaTarget. connection. onPremise. tlsMode. enabled |
object settings.kafkaTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
settings. kafkaTarget. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
settings. kafkaTarget. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
settings. kafkaTarget. auth |
object Authentication settings |
settings. kafkaTarget. auth. sasl |
object Authentication with SASL settings.kafkaTarget.auth includes only one of the fields sasl , noAuth |
settings. kafkaTarget. auth. sasl. user |
string User name |
settings. kafkaTarget. auth. sasl. password |
object Password for user |
settings. kafkaTarget. auth. sasl. password. raw |
string Raw secret value |
settings. kafkaTarget. auth. sasl. mechanism |
string SASL mechanism for authentication |
settings. kafkaTarget. auth. noAuth |
object No authentication settings.kafkaTarget.auth includes only one of the fields sasl , noAuth |
settings. kafkaTarget. securityGroups[] |
string Security groups |
settings. kafkaTarget. topicSettings |
object Target topic settings |
settings. kafkaTarget. topicSettings. topic |
object settings.kafkaTarget.topicSettings includes only one of the fields topic , topicPrefix |
settings. kafkaTarget. topicSettings. topic. topicName |
string Topic name |
settings. kafkaTarget. topicSettings. topic. saveTxOrder |
boolean (boolean) Save transactions order Not to split events queue into separate per-table queues. |
settings. kafkaTarget. topicSettings. topicPrefix |
string settings.kafkaTarget.topicSettings includes only one of the fields topic , topicPrefix Topic prefix Analogue of the Debezium setting database.server.name. Messages will be sent to topic with name <topic_prefix>. |
settings. mongoTarget |
object settings includes only one of the fields mysqlSource , postgresSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , kafkaTarget , mongoTarget |
settings. mongoTarget. connection |
object |
settings. mongoTarget. connection. connectionOptions |
object |
settings. mongoTarget. connection. connectionOptions. user |
string User name |
settings. mongoTarget. connection. connectionOptions. password |
object Password for user |
settings. mongoTarget. connection. connectionOptions. password. raw |
string Raw secret value |
settings. mongoTarget. connection. connectionOptions. authSource |
string Database name associated with the credentials |
settings. mongoTarget. connection. connectionOptions. mdbClusterId |
string settings.mongoTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
settings. mongoTarget. connection. connectionOptions. onPremise |
object settings.mongoTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
settings. mongoTarget. connection. connectionOptions. onPremise. hosts[] |
string |
settings. mongoTarget. connection. connectionOptions. onPremise. port |
string (int64) |
settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode |
object |
settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. disabled |
object settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. enabled |
object settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
settings. mongoTarget. connection. connectionOptions. onPremise. replicaSet |
string |
settings. mongoTarget. subnetId |
string |
settings. mongoTarget. securityGroups[] |
string Security groups |
settings. mongoTarget. database |
string Database name |
settings. mongoTarget. cleanupPolicy |
string
|
Methods
Method | Description |
---|---|
create | |
delete | |
get | |
list | |
update |