Data Transfer API, REST: Transfer methods
JSON Representation
{
"id": "string",
"folderId": "string",
"name": "string",
"description": "string",
"labels": "object",
"source": {
"id": "string",
"folderId": "string",
"name": "string",
"description": "string",
"labels": "object",
"settings": {
// `source.settings` includes only one of the fields `mysqlSource`, `postgresSource`, `ydbSource`, `ydsSource`, `kafkaSource`, `mongoSource`, `clickhouseSource`, `mysqlTarget`, `postgresTarget`, `clickhouseTarget`, `ydbTarget`, `kafkaTarget`, `mongoTarget`, `metrikaSource`, `ydsTarget`
"mysqlSource": {
"connection": {
// `source.settings.mysqlSource.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"port": "string",
"subnetId": "string",
"hosts": [
"string"
],
"tlsMode": {
// `source.settings.mysqlSource.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`source.settings.mysqlSource.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`source.settings.mysqlSource.connection`
},
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
"timezone": "string",
"objectTransferSettings": {
"view": "string",
"routine": "string",
"trigger": "string",
"tables": "string"
},
"includeTablesRegex": [
"string"
],
"excludeTablesRegex": [
"string"
],
"securityGroups": [
"string"
],
"serviceDatabase": "string"
},
"postgresSource": {
"connection": {
// `source.settings.postgresSource.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"port": "string",
"subnetId": "string",
"hosts": [
"string"
],
"tlsMode": {
// `source.settings.postgresSource.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`source.settings.postgresSource.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`source.settings.postgresSource.connection`
},
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
"includeTables": [
"string"
],
"excludeTables": [
"string"
],
"slotByteLagLimit": "string",
"serviceSchema": "string",
"objectTransferSettings": {
"sequence": "string",
"sequenceOwnedBy": "string",
"table": "string",
"primaryKey": "string",
"fkConstraint": "string",
"defaultValues": "string",
"constraint": "string",
"index": "string",
"view": "string",
"function": "string",
"trigger": "string",
"type": "string",
"rule": "string",
"collation": "string",
"policy": "string",
"cast": "string",
"materializedView": "string",
"sequenceSet": "string"
},
"securityGroups": [
"string"
]
},
"ydbSource": {
"database": "string",
"instance": "string",
"paths": [
"string"
],
"serviceAccountId": "string",
"subnetId": "string",
"saKeyContent": "string",
"securityGroups": [
"string"
],
"changefeedCustomName": "string"
},
"ydsSource": {
"database": "string",
"stream": "string",
"serviceAccountId": "string",
"supportedCodecs": [
"string"
],
"parser": {
// `source.settings.ydsSource.parser` includes only one of the fields `jsonParser`, `auditTrailsV1Parser`, `cloudLoggingParser`, `tskvParser`
"jsonParser": {
"dataSchema": {
// `source.settings.ydsSource.parser.jsonParser.dataSchema` includes only one of the fields `fields`, `jsonFields`
"fields": {
"fields": [
{
"name": "string",
"type": "string",
"key": true,
"required": true,
"path": "string"
}
]
},
"jsonFields": "string",
// end of the list of possible fields`source.settings.ydsSource.parser.jsonParser.dataSchema`
},
"nullKeysAllowed": true,
"addRestColumn": true,
"unescapeStringValues": true
},
"auditTrailsV1Parser": {},
"cloudLoggingParser": {},
"tskvParser": {
"dataSchema": {
// `source.settings.ydsSource.parser.tskvParser.dataSchema` includes only one of the fields `fields`, `jsonFields`
"fields": {
"fields": [
{
"name": "string",
"type": "string",
"key": true,
"required": true,
"path": "string"
}
]
},
"jsonFields": "string",
// end of the list of possible fields`source.settings.ydsSource.parser.tskvParser.dataSchema`
},
"nullKeysAllowed": true,
"addRestColumn": true,
"unescapeStringValues": true
},
// end of the list of possible fields`source.settings.ydsSource.parser`
},
"allowTtlRewind": true,
"endpoint": "string",
"subnetId": "string",
"securityGroups": [
"string"
],
"consumer": "string"
},
"kafkaSource": {
"connection": {
// `source.settings.kafkaSource.connection` includes only one of the fields `clusterId`, `onPremise`
"clusterId": "string",
"onPremise": {
"brokerUrls": [
"string"
],
"subnetId": "string",
"tlsMode": {
// `source.settings.kafkaSource.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`source.settings.kafkaSource.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`source.settings.kafkaSource.connection`
},
"auth": {
// `source.settings.kafkaSource.auth` includes only one of the fields `sasl`, `noAuth`
"sasl": {
"user": "string",
"mechanism": "string",
"password": {
"raw": "string"
}
},
"noAuth": {},
// end of the list of possible fields`source.settings.kafkaSource.auth`
},
"securityGroups": [
"string"
],
"topicName": "string",
"transformer": {
"cloudFunction": "string",
"numberOfRetries": "string",
"bufferSize": "string",
"bufferFlushInterval": "string",
"invocationTimeout": "string",
"serviceAccountId": "string"
},
"parser": {
// `source.settings.kafkaSource.parser` includes only one of the fields `jsonParser`, `auditTrailsV1Parser`, `cloudLoggingParser`, `tskvParser`
"jsonParser": {
"dataSchema": {
// `source.settings.kafkaSource.parser.jsonParser.dataSchema` includes only one of the fields `fields`, `jsonFields`
"fields": {
"fields": [
{
"name": "string",
"type": "string",
"key": true,
"required": true,
"path": "string"
}
]
},
"jsonFields": "string",
// end of the list of possible fields`source.settings.kafkaSource.parser.jsonParser.dataSchema`
},
"nullKeysAllowed": true,
"addRestColumn": true,
"unescapeStringValues": true
},
"auditTrailsV1Parser": {},
"cloudLoggingParser": {},
"tskvParser": {
"dataSchema": {
// `source.settings.kafkaSource.parser.tskvParser.dataSchema` includes only one of the fields `fields`, `jsonFields`
"fields": {
"fields": [
{
"name": "string",
"type": "string",
"key": true,
"required": true,
"path": "string"
}
]
},
"jsonFields": "string",
// end of the list of possible fields`source.settings.kafkaSource.parser.tskvParser.dataSchema`
},
"nullKeysAllowed": true,
"addRestColumn": true,
"unescapeStringValues": true
},
// end of the list of possible fields`source.settings.kafkaSource.parser`
},
"topicNames": [
"string"
]
},
"mongoSource": {
"connection": {
"connectionOptions": {
"user": "string",
"password": {
"raw": "string"
},
"authSource": "string",
// `source.settings.mongoSource.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"hosts": [
"string"
],
"port": "string",
"replicaSet": "string",
"tlsMode": {
// `source.settings.mongoSource.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`source.settings.mongoSource.connection.connectionOptions.onPremise.tlsMode`
}
},
// end of the list of possible fields`source.settings.mongoSource.connection.connectionOptions`
}
},
"subnetId": "string",
"collections": [
{
"databaseName": "string",
"collectionName": "string"
}
],
"excludedCollections": [
{
"databaseName": "string",
"collectionName": "string"
}
],
"secondaryPreferredMode": true,
"securityGroups": [
"string"
]
},
"clickhouseSource": {
"connection": {
"connectionOptions": {
"user": "string",
"password": {
"raw": "string"
},
"database": "string",
// `source.settings.clickhouseSource.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"shards": [
{
"name": "string",
"hosts": [
"string"
]
}
],
"httpPort": "string",
"nativePort": "string",
"tlsMode": {
// `source.settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`source.settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode`
}
},
// end of the list of possible fields`source.settings.clickhouseSource.connection.connectionOptions`
}
},
"includeTables": [
"string"
],
"excludeTables": [
"string"
],
"subnetId": "string",
"securityGroups": [
"string"
]
},
"mysqlTarget": {
"connection": {
// `source.settings.mysqlTarget.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"port": "string",
"subnetId": "string",
"hosts": [
"string"
],
"tlsMode": {
// `source.settings.mysqlTarget.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`source.settings.mysqlTarget.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`source.settings.mysqlTarget.connection`
},
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
"sqlMode": "string",
"skipConstraintChecks": true,
"timezone": "string",
"cleanupPolicy": "string",
"serviceDatabase": "string",
"securityGroups": [
"string"
]
},
"postgresTarget": {
"connection": {
// `source.settings.postgresTarget.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"port": "string",
"subnetId": "string",
"hosts": [
"string"
],
"tlsMode": {
// `source.settings.postgresTarget.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`source.settings.postgresTarget.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`source.settings.postgresTarget.connection`
},
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
"cleanupPolicy": "string",
"securityGroups": [
"string"
]
},
"clickhouseTarget": {
"connection": {
"connectionOptions": {
"user": "string",
"password": {
"raw": "string"
},
"database": "string",
// `source.settings.clickhouseTarget.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"shards": [
{
"name": "string",
"hosts": [
"string"
]
}
],
"httpPort": "string",
"nativePort": "string",
"tlsMode": {
// `source.settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`source.settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode`
}
},
// end of the list of possible fields`source.settings.clickhouseTarget.connection.connectionOptions`
}
},
"subnetId": "string",
"altNames": [
{
"fromName": "string",
"toName": "string"
}
],
"cleanupPolicy": "string",
"sharding": {
// `source.settings.clickhouseTarget.sharding` includes only one of the fields `columnValueHash`, `customMapping`, `transferId`, `roundRobin`
"columnValueHash": {
"columnName": "string"
},
"customMapping": {
"columnName": "string",
"mapping": [
{
"columnValue": {
"stringValue": "string"
},
"shardName": "string"
}
]
},
"transferId": "object",
"roundRobin": "object",
// end of the list of possible fields`source.settings.clickhouseTarget.sharding`
},
"clickhouseClusterName": "string",
"securityGroups": [
"string"
]
},
"ydbTarget": {
"database": "string",
"instance": "string",
"path": "string",
"serviceAccountId": "string",
"cleanupPolicy": "string",
"subnetId": "string",
"saKeyContent": "string",
"securityGroups": [
"string"
],
"isTableColumnOriented": true,
"defaultCompression": "string"
},
"kafkaTarget": {
"connection": {
// `source.settings.kafkaTarget.connection` includes only one of the fields `clusterId`, `onPremise`
"clusterId": "string",
"onPremise": {
"brokerUrls": [
"string"
],
"subnetId": "string",
"tlsMode": {
// `source.settings.kafkaTarget.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`source.settings.kafkaTarget.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`source.settings.kafkaTarget.connection`
},
"auth": {
// `source.settings.kafkaTarget.auth` includes only one of the fields `sasl`, `noAuth`
"sasl": {
"user": "string",
"mechanism": "string",
"password": {
"raw": "string"
}
},
"noAuth": {},
// end of the list of possible fields`source.settings.kafkaTarget.auth`
},
"securityGroups": [
"string"
],
"topicSettings": {
// `source.settings.kafkaTarget.topicSettings` includes only one of the fields `topic`, `topicPrefix`
"topic": {
"topicName": "string",
"saveTxOrder": true
},
"topicPrefix": "string",
// end of the list of possible fields`source.settings.kafkaTarget.topicSettings`
},
"serializer": {
// `source.settings.kafkaTarget.serializer` includes only one of the fields `serializerAuto`, `serializerJson`, `serializerDebezium`
"serializerAuto": {},
"serializerJson": {},
"serializerDebezium": {
"serializerParameters": [
{
"key": "string",
"value": "string"
}
]
},
// end of the list of possible fields`source.settings.kafkaTarget.serializer`
}
},
"mongoTarget": {
"connection": {
"connectionOptions": {
"user": "string",
"password": {
"raw": "string"
},
"authSource": "string",
// `source.settings.mongoTarget.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"hosts": [
"string"
],
"port": "string",
"replicaSet": "string",
"tlsMode": {
// `source.settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`source.settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode`
}
},
// end of the list of possible fields`source.settings.mongoTarget.connection.connectionOptions`
}
},
"database": "string",
"cleanupPolicy": "string",
"subnetId": "string",
"securityGroups": [
"string"
]
},
"metrikaSource": {
"counterIds": [
"string"
],
"token": {
"raw": "string"
},
"streams": [
{
"type": "string",
"columns": [
"string"
]
}
]
},
"ydsTarget": {
"database": "string",
"stream": "string",
"serviceAccountId": "string",
"saveTxOrder": true,
"serializer": {
// `source.settings.ydsTarget.serializer` includes only one of the fields `serializerAuto`, `serializerJson`, `serializerDebezium`
"serializerAuto": {},
"serializerJson": {},
"serializerDebezium": {
"serializerParameters": [
{
"key": "string",
"value": "string"
}
]
},
// end of the list of possible fields`source.settings.ydsTarget.serializer`
},
"endpoint": "string",
"subnetId": "string",
"securityGroups": [
"string"
]
},
// end of the list of possible fields`source.settings`
}
},
"target": {
"id": "string",
"folderId": "string",
"name": "string",
"description": "string",
"labels": "object",
"settings": {
// `target.settings` includes only one of the fields `mysqlSource`, `postgresSource`, `ydbSource`, `ydsSource`, `kafkaSource`, `mongoSource`, `clickhouseSource`, `mysqlTarget`, `postgresTarget`, `clickhouseTarget`, `ydbTarget`, `kafkaTarget`, `mongoTarget`, `metrikaSource`, `ydsTarget`
"mysqlSource": {
"connection": {
// `target.settings.mysqlSource.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"port": "string",
"subnetId": "string",
"hosts": [
"string"
],
"tlsMode": {
// `target.settings.mysqlSource.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`target.settings.mysqlSource.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`target.settings.mysqlSource.connection`
},
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
"timezone": "string",
"objectTransferSettings": {
"view": "string",
"routine": "string",
"trigger": "string",
"tables": "string"
},
"includeTablesRegex": [
"string"
],
"excludeTablesRegex": [
"string"
],
"securityGroups": [
"string"
],
"serviceDatabase": "string"
},
"postgresSource": {
"connection": {
// `target.settings.postgresSource.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"port": "string",
"subnetId": "string",
"hosts": [
"string"
],
"tlsMode": {
// `target.settings.postgresSource.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`target.settings.postgresSource.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`target.settings.postgresSource.connection`
},
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
"includeTables": [
"string"
],
"excludeTables": [
"string"
],
"slotByteLagLimit": "string",
"serviceSchema": "string",
"objectTransferSettings": {
"sequence": "string",
"sequenceOwnedBy": "string",
"table": "string",
"primaryKey": "string",
"fkConstraint": "string",
"defaultValues": "string",
"constraint": "string",
"index": "string",
"view": "string",
"function": "string",
"trigger": "string",
"type": "string",
"rule": "string",
"collation": "string",
"policy": "string",
"cast": "string",
"materializedView": "string",
"sequenceSet": "string"
},
"securityGroups": [
"string"
]
},
"ydbSource": {
"database": "string",
"instance": "string",
"paths": [
"string"
],
"serviceAccountId": "string",
"subnetId": "string",
"saKeyContent": "string",
"securityGroups": [
"string"
],
"changefeedCustomName": "string"
},
"ydsSource": {
"database": "string",
"stream": "string",
"serviceAccountId": "string",
"supportedCodecs": [
"string"
],
"parser": {
// `target.settings.ydsSource.parser` includes only one of the fields `jsonParser`, `auditTrailsV1Parser`, `cloudLoggingParser`, `tskvParser`
"jsonParser": {
"dataSchema": {
// `target.settings.ydsSource.parser.jsonParser.dataSchema` includes only one of the fields `fields`, `jsonFields`
"fields": {
"fields": [
{
"name": "string",
"type": "string",
"key": true,
"required": true,
"path": "string"
}
]
},
"jsonFields": "string",
// end of the list of possible fields`target.settings.ydsSource.parser.jsonParser.dataSchema`
},
"nullKeysAllowed": true,
"addRestColumn": true,
"unescapeStringValues": true
},
"auditTrailsV1Parser": {},
"cloudLoggingParser": {},
"tskvParser": {
"dataSchema": {
// `target.settings.ydsSource.parser.tskvParser.dataSchema` includes only one of the fields `fields`, `jsonFields`
"fields": {
"fields": [
{
"name": "string",
"type": "string",
"key": true,
"required": true,
"path": "string"
}
]
},
"jsonFields": "string",
// end of the list of possible fields`target.settings.ydsSource.parser.tskvParser.dataSchema`
},
"nullKeysAllowed": true,
"addRestColumn": true,
"unescapeStringValues": true
},
// end of the list of possible fields`target.settings.ydsSource.parser`
},
"allowTtlRewind": true,
"endpoint": "string",
"subnetId": "string",
"securityGroups": [
"string"
],
"consumer": "string"
},
"kafkaSource": {
"connection": {
// `target.settings.kafkaSource.connection` includes only one of the fields `clusterId`, `onPremise`
"clusterId": "string",
"onPremise": {
"brokerUrls": [
"string"
],
"subnetId": "string",
"tlsMode": {
// `target.settings.kafkaSource.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`target.settings.kafkaSource.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`target.settings.kafkaSource.connection`
},
"auth": {
// `target.settings.kafkaSource.auth` includes only one of the fields `sasl`, `noAuth`
"sasl": {
"user": "string",
"mechanism": "string",
"password": {
"raw": "string"
}
},
"noAuth": {},
// end of the list of possible fields`target.settings.kafkaSource.auth`
},
"securityGroups": [
"string"
],
"topicName": "string",
"transformer": {
"cloudFunction": "string",
"numberOfRetries": "string",
"bufferSize": "string",
"bufferFlushInterval": "string",
"invocationTimeout": "string",
"serviceAccountId": "string"
},
"parser": {
// `target.settings.kafkaSource.parser` includes only one of the fields `jsonParser`, `auditTrailsV1Parser`, `cloudLoggingParser`, `tskvParser`
"jsonParser": {
"dataSchema": {
// `target.settings.kafkaSource.parser.jsonParser.dataSchema` includes only one of the fields `fields`, `jsonFields`
"fields": {
"fields": [
{
"name": "string",
"type": "string",
"key": true,
"required": true,
"path": "string"
}
]
},
"jsonFields": "string",
// end of the list of possible fields`target.settings.kafkaSource.parser.jsonParser.dataSchema`
},
"nullKeysAllowed": true,
"addRestColumn": true,
"unescapeStringValues": true
},
"auditTrailsV1Parser": {},
"cloudLoggingParser": {},
"tskvParser": {
"dataSchema": {
// `target.settings.kafkaSource.parser.tskvParser.dataSchema` includes only one of the fields `fields`, `jsonFields`
"fields": {
"fields": [
{
"name": "string",
"type": "string",
"key": true,
"required": true,
"path": "string"
}
]
},
"jsonFields": "string",
// end of the list of possible fields`target.settings.kafkaSource.parser.tskvParser.dataSchema`
},
"nullKeysAllowed": true,
"addRestColumn": true,
"unescapeStringValues": true
},
// end of the list of possible fields`target.settings.kafkaSource.parser`
},
"topicNames": [
"string"
]
},
"mongoSource": {
"connection": {
"connectionOptions": {
"user": "string",
"password": {
"raw": "string"
},
"authSource": "string",
// `target.settings.mongoSource.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"hosts": [
"string"
],
"port": "string",
"replicaSet": "string",
"tlsMode": {
// `target.settings.mongoSource.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`target.settings.mongoSource.connection.connectionOptions.onPremise.tlsMode`
}
},
// end of the list of possible fields`target.settings.mongoSource.connection.connectionOptions`
}
},
"subnetId": "string",
"collections": [
{
"databaseName": "string",
"collectionName": "string"
}
],
"excludedCollections": [
{
"databaseName": "string",
"collectionName": "string"
}
],
"secondaryPreferredMode": true,
"securityGroups": [
"string"
]
},
"clickhouseSource": {
"connection": {
"connectionOptions": {
"user": "string",
"password": {
"raw": "string"
},
"database": "string",
// `target.settings.clickhouseSource.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"shards": [
{
"name": "string",
"hosts": [
"string"
]
}
],
"httpPort": "string",
"nativePort": "string",
"tlsMode": {
// `target.settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`target.settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode`
}
},
// end of the list of possible fields`target.settings.clickhouseSource.connection.connectionOptions`
}
},
"includeTables": [
"string"
],
"excludeTables": [
"string"
],
"subnetId": "string",
"securityGroups": [
"string"
]
},
"mysqlTarget": {
"connection": {
// `target.settings.mysqlTarget.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"port": "string",
"subnetId": "string",
"hosts": [
"string"
],
"tlsMode": {
// `target.settings.mysqlTarget.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`target.settings.mysqlTarget.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`target.settings.mysqlTarget.connection`
},
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
"sqlMode": "string",
"skipConstraintChecks": true,
"timezone": "string",
"cleanupPolicy": "string",
"serviceDatabase": "string",
"securityGroups": [
"string"
]
},
"postgresTarget": {
"connection": {
// `target.settings.postgresTarget.connection` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"port": "string",
"subnetId": "string",
"hosts": [
"string"
],
"tlsMode": {
// `target.settings.postgresTarget.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`target.settings.postgresTarget.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`target.settings.postgresTarget.connection`
},
"database": "string",
"user": "string",
"password": {
"raw": "string"
},
"cleanupPolicy": "string",
"securityGroups": [
"string"
]
},
"clickhouseTarget": {
"connection": {
"connectionOptions": {
"user": "string",
"password": {
"raw": "string"
},
"database": "string",
// `target.settings.clickhouseTarget.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"shards": [
{
"name": "string",
"hosts": [
"string"
]
}
],
"httpPort": "string",
"nativePort": "string",
"tlsMode": {
// `target.settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`target.settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode`
}
},
// end of the list of possible fields`target.settings.clickhouseTarget.connection.connectionOptions`
}
},
"subnetId": "string",
"altNames": [
{
"fromName": "string",
"toName": "string"
}
],
"cleanupPolicy": "string",
"sharding": {
// `target.settings.clickhouseTarget.sharding` includes only one of the fields `columnValueHash`, `customMapping`, `transferId`, `roundRobin`
"columnValueHash": {
"columnName": "string"
},
"customMapping": {
"columnName": "string",
"mapping": [
{
"columnValue": {
"stringValue": "string"
},
"shardName": "string"
}
]
},
"transferId": "object",
"roundRobin": "object",
// end of the list of possible fields`target.settings.clickhouseTarget.sharding`
},
"clickhouseClusterName": "string",
"securityGroups": [
"string"
]
},
"ydbTarget": {
"database": "string",
"instance": "string",
"path": "string",
"serviceAccountId": "string",
"cleanupPolicy": "string",
"subnetId": "string",
"saKeyContent": "string",
"securityGroups": [
"string"
],
"isTableColumnOriented": true,
"defaultCompression": "string"
},
"kafkaTarget": {
"connection": {
// `target.settings.kafkaTarget.connection` includes only one of the fields `clusterId`, `onPremise`
"clusterId": "string",
"onPremise": {
"brokerUrls": [
"string"
],
"subnetId": "string",
"tlsMode": {
// `target.settings.kafkaTarget.connection.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`target.settings.kafkaTarget.connection.onPremise.tlsMode`
}
},
// end of the list of possible fields`target.settings.kafkaTarget.connection`
},
"auth": {
// `target.settings.kafkaTarget.auth` includes only one of the fields `sasl`, `noAuth`
"sasl": {
"user": "string",
"mechanism": "string",
"password": {
"raw": "string"
}
},
"noAuth": {},
// end of the list of possible fields`target.settings.kafkaTarget.auth`
},
"securityGroups": [
"string"
],
"topicSettings": {
// `target.settings.kafkaTarget.topicSettings` includes only one of the fields `topic`, `topicPrefix`
"topic": {
"topicName": "string",
"saveTxOrder": true
},
"topicPrefix": "string",
// end of the list of possible fields`target.settings.kafkaTarget.topicSettings`
},
"serializer": {
// `target.settings.kafkaTarget.serializer` includes only one of the fields `serializerAuto`, `serializerJson`, `serializerDebezium`
"serializerAuto": {},
"serializerJson": {},
"serializerDebezium": {
"serializerParameters": [
{
"key": "string",
"value": "string"
}
]
},
// end of the list of possible fields`target.settings.kafkaTarget.serializer`
}
},
"mongoTarget": {
"connection": {
"connectionOptions": {
"user": "string",
"password": {
"raw": "string"
},
"authSource": "string",
// `target.settings.mongoTarget.connection.connectionOptions` includes only one of the fields `mdbClusterId`, `onPremise`
"mdbClusterId": "string",
"onPremise": {
"hosts": [
"string"
],
"port": "string",
"replicaSet": "string",
"tlsMode": {
// `target.settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode` includes only one of the fields `disabled`, `enabled`
"disabled": "object",
"enabled": {
"caCertificate": "string"
},
// end of the list of possible fields`target.settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode`
}
},
// end of the list of possible fields`target.settings.mongoTarget.connection.connectionOptions`
}
},
"database": "string",
"cleanupPolicy": "string",
"subnetId": "string",
"securityGroups": [
"string"
]
},
"metrikaSource": {
"counterIds": [
"string"
],
"token": {
"raw": "string"
},
"streams": [
{
"type": "string",
"columns": [
"string"
]
}
]
},
"ydsTarget": {
"database": "string",
"stream": "string",
"serviceAccountId": "string",
"saveTxOrder": true,
"serializer": {
// `target.settings.ydsTarget.serializer` includes only one of the fields `serializerAuto`, `serializerJson`, `serializerDebezium`
"serializerAuto": {},
"serializerJson": {},
"serializerDebezium": {
"serializerParameters": [
{
"key": "string",
"value": "string"
}
]
},
// end of the list of possible fields`target.settings.ydsTarget.serializer`
},
"endpoint": "string",
"subnetId": "string",
"securityGroups": [
"string"
]
},
// end of the list of possible fields`target.settings`
}
},
"runtime": {
"ycRuntime": {
"jobCount": "string",
"uploadShardParams": {
"jobCount": "string",
"processCount": "string"
}
}
},
"status": "string",
"type": "string",
"warning": "string",
"transformation": {
"transformers": [
{
// `transformation.transformers[]` includes only one of the fields `maskField`, `filterColumns`, `renameTables`, `replacePrimaryKey`, `convertToString`, `sharderTransformer`, `tableSplitterTransformer`, `filterRows`
"maskField": {
"tables": {
"includeTables": [
"string"
],
"excludeTables": [
"string"
]
},
"columns": [
"string"
],
"function": {
"maskFunctionHash": {
"userDefinedSalt": "string"
}
}
},
"filterColumns": {
"tables": {
"includeTables": [
"string"
],
"excludeTables": [
"string"
]
},
"columns": {
"includeColumns": [
"string"
],
"excludeColumns": [
"string"
]
}
},
"renameTables": {
"renameTables": [
{
"originalName": {
"nameSpace": "string",
"name": "string"
},
"newName": {
"nameSpace": "string",
"name": "string"
}
}
]
},
"replacePrimaryKey": {
"tables": {
"includeTables": [
"string"
],
"excludeTables": [
"string"
]
},
"keys": [
"string"
]
},
"convertToString": {
"tables": {
"includeTables": [
"string"
],
"excludeTables": [
"string"
]
},
"columns": {
"includeColumns": [
"string"
],
"excludeColumns": [
"string"
]
}
},
"sharderTransformer": {
"tables": {
"includeTables": [
"string"
],
"excludeTables": [
"string"
]
},
"columns": {
"includeColumns": [
"string"
],
"excludeColumns": [
"string"
]
},
"shardsCount": "string"
},
"tableSplitterTransformer": {
"tables": {
"includeTables": [
"string"
],
"excludeTables": [
"string"
]
},
"columns": [
"string"
],
"splitter": "string"
},
"filterRows": {
"tables": {
"includeTables": [
"string"
],
"excludeTables": [
"string"
]
},
"filter": "string"
},
// end of the list of possible fields`transformation.transformers[]`
}
]
},
"prestable": true
}
Field | Description |
---|---|
id | string |
folderId | string |
name | string |
description | string |
labels | object |
source | object |
source. id |
string |
source. folderId |
string |
source. name |
string |
source. description |
string |
source. labels |
object |
source. settings |
object |
source. settings. mysqlSource |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. mysqlSource. connection |
object Database connection settings |
source. settings. mysqlSource. connection. mdbClusterId |
string source.settings.mysqlSource.connection includes only one of the fields mdbClusterId , onPremise Managed Service for MySQL cluster ID |
source. settings. mysqlSource. connection. onPremise |
object Connection options for on-premise MySQL source.settings.mysqlSource.connection includes only one of the fields mdbClusterId , onPremise |
source. settings. mysqlSource. connection. onPremise. port |
string (int64) Database port |
source. settings. mysqlSource. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
source. settings. mysqlSource. connection. onPremise. hosts[] |
string |
source. settings. mysqlSource. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
source. settings. mysqlSource. connection. onPremise. tlsMode. disabled |
object source.settings.mysqlSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. mysqlSource. connection. onPremise. tlsMode. disabled. disabled |
object source.settings.mysqlSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. mysqlSource. connection. onPremise. tlsMode. enabled |
object source.settings.mysqlSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
source. settings. mysqlSource. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
source. settings. mysqlSource. database |
string Database name You can leave it empty, then it will be possible to transfer tables from several databases at the same time from this source. |
source. settings. mysqlSource. user |
string User for database access. |
source. settings. mysqlSource. password |
object Password for database access. |
source. settings. mysqlSource. password. raw |
string Raw secret value |
source. settings. mysqlSource. timezone |
string Database timezone Is used for parsing timestamps for saving source timezones. Accepts values from IANA timezone database. Default: local timezone. |
source. settings. mysqlSource. objectTransferSettings |
object Schema migration Select database objects to be transferred during activation or deactivation. |
source. settings. mysqlSource. objectTransferSettings. view |
string Views CREATE VIEW ...
|
source. settings. mysqlSource. objectTransferSettings. routine |
string Routines CREATE PROCEDURE ... ; CREATE FUNCTION ... ;
|
source. settings. mysqlSource. objectTransferSettings. trigger |
string Triggers CREATE TRIGGER ...
|
source. settings. mysqlSource. objectTransferSettings. tables |
string
|
source. settings. mysqlSource. includeTablesRegex[] |
string |
source. settings. mysqlSource. excludeTablesRegex[] |
string |
source. settings. mysqlSource. securityGroups[] |
string Security groups |
source. settings. mysqlSource. serviceDatabase |
string Database for service tables Default: data source database. Here created technical tables (__tm_keeper, __tm_gtid_keeper). |
source. settings. postgresSource |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. postgresSource. connection |
object Database connection settings |
source. settings. postgresSource. connection. mdbClusterId |
string source.settings.postgresSource.connection includes only one of the fields mdbClusterId , onPremise Managed Service for PostgreSQL cluster ID |
source. settings. postgresSource. connection. onPremise |
object Connection options for on-premise PostgreSQL source.settings.postgresSource.connection includes only one of the fields mdbClusterId , onPremise |
source. settings. postgresSource. connection. onPremise. port |
string (int64) Will be used if the cluster ID is not specified. |
source. settings. postgresSource. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
source. settings. postgresSource. connection. onPremise. hosts[] |
string |
source. settings. postgresSource. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
source. settings. postgresSource. connection. onPremise. tlsMode. disabled |
object source.settings.postgresSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. postgresSource. connection. onPremise. tlsMode. disabled. disabled |
object source.settings.postgresSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. postgresSource. connection. onPremise. tlsMode. enabled |
object source.settings.postgresSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
source. settings. postgresSource. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
source. settings. postgresSource. database |
string Database name |
source. settings. postgresSource. user |
string User for database access. |
source. settings. postgresSource. password |
object Password for database access. |
source. settings. postgresSource. password. raw |
string Raw secret value |
source. settings. postgresSource. includeTables[] |
string Included tables If none or empty list is presented, all tables are replicated. Full table name with schema. Can contain schema_name.* patterns. |
source. settings. postgresSource. excludeTables[] |
string Excluded tables If none or empty list is presented, all tables are replicated. Full table name with schema. Can contain schema_name.* patterns. |
source. settings. postgresSource. slotByteLagLimit |
string (int64) Maximum lag of replication slot (in bytes); after exceeding this limit replication will be aborted. |
source. settings. postgresSource. serviceSchema |
string Database schema for service tables (__consumer_keeper, __data_transfer_mole_finder). Default is public |
source. settings. postgresSource. objectTransferSettings |
object Select database objects to be transferred during activation or deactivation. |
source. settings. postgresSource. objectTransferSettings. sequence |
string Sequences CREATE SEQUENCE ...
|
source. settings. postgresSource. objectTransferSettings. sequenceOwnedBy |
string Owned sequences CREATE SEQUENCE ... OWNED BY ...
|
source. settings. postgresSource. objectTransferSettings. table |
string Tables CREATE TABLE ...
|
source. settings. postgresSource. objectTransferSettings. primaryKey |
string Primary keys ALTER TABLE ... ADD PRIMARY KEY ...
|
source. settings. postgresSource. objectTransferSettings. fkConstraint |
string Foreign keys ALTER TABLE ... ADD FOREIGN KEY ...
|
source. settings. postgresSource. objectTransferSettings. defaultValues |
string Default values ALTER TABLE ... ALTER COLUMN ... SET DEFAULT ...
|
source. settings. postgresSource. objectTransferSettings. constraint |
string Constraints ALTER TABLE ... ADD CONSTRAINT ...
|
source. settings. postgresSource. objectTransferSettings. index |
string Indexes CREATE INDEX ...
|
source. settings. postgresSource. objectTransferSettings. view |
string Views CREATE VIEW ...
|
source. settings. postgresSource. objectTransferSettings. function |
string Functions CREATE FUNCTION ...
|
source. settings. postgresSource. objectTransferSettings. trigger |
string Triggers CREATE TRIGGER ...
|
source. settings. postgresSource. objectTransferSettings. type |
string Types CREATE TYPE ...
|
source. settings. postgresSource. objectTransferSettings. rule |
string Rules CREATE RULE ...
|
source. settings. postgresSource. objectTransferSettings. collation |
string Collations CREATE COLLATION ...
|
source. settings. postgresSource. objectTransferSettings. policy |
string Policies CREATE POLICY ...
|
source. settings. postgresSource. objectTransferSettings. cast |
string Casts CREATE CAST ...
|
source. settings. postgresSource. objectTransferSettings. materializedView |
string Materialized views CREATE MATERIALIZED VIEW ...
|
source. settings. postgresSource. objectTransferSettings. sequenceSet |
string
|
source. settings. postgresSource. securityGroups[] |
string Security groups |
source. settings. ydbSource |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. ydbSource. database |
string Path in YDB where to store tables |
source. settings. ydbSource. instance |
string Instance of YDB. example: ydb-ru-prestable.yandex.net:2135 |
source. settings. ydbSource. paths[] |
string |
source. settings. ydbSource. serviceAccountId |
string |
source. settings. ydbSource. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
source. settings. ydbSource. saKeyContent |
string Authorization Key |
source. settings. ydbSource. securityGroups[] |
string Security groups |
source. settings. ydbSource. changefeedCustomName |
string Pre-created change feed |
source. settings. ydsSource |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. ydsSource. database |
string Database |
source. settings. ydsSource. stream |
string Stream |
source. settings. ydsSource. serviceAccountId |
string SA which has read access to the stream. |
source. settings. ydsSource. supportedCodecs[] |
string Compression codec |
source. settings. ydsSource. parser |
object Data parsing rules |
source. settings. ydsSource. parser. jsonParser |
object source.settings.ydsSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
source. settings. ydsSource. parser. jsonParser. dataSchema |
object |
source. settings. ydsSource. parser. jsonParser. dataSchema. fields |
object source.settings.ydsSource.parser.jsonParser.dataSchema includes only one of the fields fields , jsonFields |
source. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[] |
object Column schema |
source. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[]. name |
string |
source. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[]. type |
string |
source. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[]. key |
boolean (boolean) |
source. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[]. required |
boolean (boolean) |
source. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[]. path |
string |
source. settings. ydsSource. parser. jsonParser. dataSchema. jsonFields |
string source.settings.ydsSource.parser.jsonParser.dataSchema includes only one of the fields fields , jsonFields |
source. settings. ydsSource. parser. jsonParser. nullKeysAllowed |
boolean (boolean) Allow null keys, if no - null keys will be putted to unparsed data |
source. settings. ydsSource. parser. jsonParser. addRestColumn |
boolean (boolean) Will add _rest column for all unknown fields |
source. settings. ydsSource. parser. jsonParser. unescapeStringValues |
boolean (boolean) Unescape string values |
source. settings. ydsSource. parser. auditTrailsV1Parser |
object source.settings.ydsSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
source. settings. ydsSource. parser. cloudLoggingParser |
object source.settings.ydsSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
source. settings. ydsSource. parser. tskvParser |
object source.settings.ydsSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
source. settings. ydsSource. parser. tskvParser. dataSchema |
object |
source. settings. ydsSource. parser. tskvParser. dataSchema. fields |
object source.settings.ydsSource.parser.tskvParser.dataSchema includes only one of the fields fields , jsonFields |
source. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[] |
object Column schema |
source. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[]. name |
string |
source. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[]. type |
string |
source. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[]. key |
boolean (boolean) |
source. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[]. required |
boolean (boolean) |
source. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[]. path |
string |
source. settings. ydsSource. parser. tskvParser. dataSchema. jsonFields |
string source.settings.ydsSource.parser.tskvParser.dataSchema includes only one of the fields fields , jsonFields |
source. settings. ydsSource. parser. tskvParser. nullKeysAllowed |
boolean (boolean) Allow null keys, if no - null keys will be putted to unparsed data |
source. settings. ydsSource. parser. tskvParser. addRestColumn |
boolean (boolean) Will add _rest column for all unknown fields |
source. settings. ydsSource. parser. tskvParser. unescapeStringValues |
boolean (boolean) Unescape string values |
source. settings. ydsSource. allowTtlRewind |
boolean (boolean) Should continue working, if consumer read lag exceed TTL of topic False: stop the transfer in error state, if detected lost data. True: continue working with losing part of data |
source. settings. ydsSource. endpoint |
string for dedicated db |
source. settings. ydsSource. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
source. settings. ydsSource. securityGroups[] |
string Security groups |
source. settings. ydsSource. consumer |
string for important streams |
source. settings. kafkaSource |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. kafkaSource. connection |
object Connection settings |
source. settings. kafkaSource. connection. clusterId |
string source.settings.kafkaSource.connection includes only one of the fields clusterId , onPremise Managed Service for Kafka cluster ID |
source. settings. kafkaSource. connection. onPremise |
object Connection options for on-premise Kafka source.settings.kafkaSource.connection includes only one of the fields clusterId , onPremise |
source. settings. kafkaSource. connection. onPremise. brokerUrls[] |
string Kafka broker URLs |
source. settings. kafkaSource. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
source. settings. kafkaSource. connection. onPremise. tlsMode |
object TLS settings for broker connection. Disabled by default. |
source. settings. kafkaSource. connection. onPremise. tlsMode. disabled |
object source.settings.kafkaSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. kafkaSource. connection. onPremise. tlsMode. disabled. disabled |
object source.settings.kafkaSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. kafkaSource. connection. onPremise. tlsMode. enabled |
object source.settings.kafkaSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
source. settings. kafkaSource. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
source. settings. kafkaSource. auth |
object Authentication settings |
source. settings. kafkaSource. auth. sasl |
object Authentication with SASL source.settings.kafkaSource.auth includes only one of the fields sasl , noAuth |
source. settings. kafkaSource. auth. sasl. user |
string User name |
source. settings. kafkaSource. auth. sasl. mechanism |
string SASL mechanism for authentication |
source. settings. kafkaSource. auth. sasl. password |
object Password for user |
source. settings. kafkaSource. auth. sasl. password. raw |
string Raw secret value |
source. settings. kafkaSource. auth. noAuth |
object No authentication source.settings.kafkaSource.auth includes only one of the fields sasl , noAuth |
source. settings. kafkaSource. securityGroups[] |
string Security groups |
source. settings. kafkaSource. topicName |
string Full source topic name Deprecated in favor of topic names |
source. settings. kafkaSource. transformer |
object Data transformation rules |
source. settings. kafkaSource. transformer. cloudFunction |
string Cloud function |
source. settings. kafkaSource. transformer. numberOfRetries |
string (int64) Number of retries |
source. settings. kafkaSource. transformer. bufferSize |
string Buffer size for function |
source. settings. kafkaSource. transformer. bufferFlushInterval |
string Flush interval |
source. settings. kafkaSource. transformer. invocationTimeout |
string Invocation timeout |
source. settings. kafkaSource. transformer. serviceAccountId |
string Service account |
source. settings. kafkaSource. parser |
object Data parsing rules |
source. settings. kafkaSource. parser. jsonParser |
object source.settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
source. settings. kafkaSource. parser. jsonParser. dataSchema |
object |
source. settings. kafkaSource. parser. jsonParser. dataSchema. fields |
object source.settings.kafkaSource.parser.jsonParser.dataSchema includes only one of the fields fields , jsonFields |
source. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[] |
object Column schema |
source. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. name |
string |
source. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. type |
string |
source. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. key |
boolean (boolean) |
source. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. required |
boolean (boolean) |
source. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. path |
string |
source. settings. kafkaSource. parser. jsonParser. dataSchema. jsonFields |
string source.settings.kafkaSource.parser.jsonParser.dataSchema includes only one of the fields fields , jsonFields |
source. settings. kafkaSource. parser. jsonParser. nullKeysAllowed |
boolean (boolean) Allow null keys, if no - null keys will be putted to unparsed data |
source. settings. kafkaSource. parser. jsonParser. addRestColumn |
boolean (boolean) Will add _rest column for all unknown fields |
source. settings. kafkaSource. parser. jsonParser. unescapeStringValues |
boolean (boolean) Unescape string values |
source. settings. kafkaSource. parser. auditTrailsV1Parser |
object source.settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
source. settings. kafkaSource. parser. cloudLoggingParser |
object source.settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
source. settings. kafkaSource. parser. tskvParser |
object source.settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
source. settings. kafkaSource. parser. tskvParser. dataSchema |
object |
source. settings. kafkaSource. parser. tskvParser. dataSchema. fields |
object source.settings.kafkaSource.parser.tskvParser.dataSchema includes only one of the fields fields , jsonFields |
source. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[] |
object Column schema |
source. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. name |
string |
source. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. type |
string |
source. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. key |
boolean (boolean) |
source. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. required |
boolean (boolean) |
source. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. path |
string |
source. settings. kafkaSource. parser. tskvParser. dataSchema. jsonFields |
string source.settings.kafkaSource.parser.tskvParser.dataSchema includes only one of the fields fields , jsonFields |
source. settings. kafkaSource. parser. tskvParser. nullKeysAllowed |
boolean (boolean) Allow null keys, if no - null keys will be putted to unparsed data |
source. settings. kafkaSource. parser. tskvParser. addRestColumn |
boolean (boolean) Will add _rest column for all unknown fields |
source. settings. kafkaSource. parser. tskvParser. unescapeStringValues |
boolean (boolean) Unescape string values |
source. settings. kafkaSource. topicNames[] |
string List of topic names to read |
source. settings. mongoSource |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. mongoSource. connection |
object |
source. settings. mongoSource. connection. connectionOptions |
object |
source. settings. mongoSource. connection. connectionOptions. user |
string User name |
source. settings. mongoSource. connection. connectionOptions. password |
object Password for user |
source. settings. mongoSource. connection. connectionOptions. password. raw |
string Raw secret value |
source. settings. mongoSource. connection. connectionOptions. authSource |
string Database name associated with the credentials |
source. settings. mongoSource. connection. connectionOptions. mdbClusterId |
string source.settings.mongoSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
source. settings. mongoSource. connection. connectionOptions. onPremise |
object source.settings.mongoSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
source. settings. mongoSource. connection. connectionOptions. onPremise. hosts[] |
string |
source. settings. mongoSource. connection. connectionOptions. onPremise. port |
string (int64) |
source. settings. mongoSource. connection. connectionOptions. onPremise. replicaSet |
string |
source. settings. mongoSource. connection. connectionOptions. onPremise. tlsMode |
object |
source. settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. disabled |
object source.settings.mongoSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object source.settings.mongoSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. enabled |
object source.settings.mongoSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
source. settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
source. settings. mongoSource. subnetId |
string |
source. settings. mongoSource. collections[] |
object List of collections for replication. Empty list implies replication of all tables on the deployment. Allowed to use * as collection name. |
source. settings. mongoSource. collections[]. databaseName |
string |
source. settings. mongoSource. collections[]. collectionName |
string |
source. settings. mongoSource. excludedCollections[] |
object List of forbidden collections for replication. Allowed to use * as collection name for forbid all collections of concrete schema. |
source. settings. mongoSource. excludedCollections[]. databaseName |
string |
source. settings. mongoSource. excludedCollections[]. collectionName |
string |
source. settings. mongoSource. secondaryPreferredMode |
boolean (boolean) Read mode for mongo client |
source. settings. mongoSource. securityGroups[] |
string Security groups |
source. settings. clickhouseSource |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. clickhouseSource. connection |
object |
source. settings. clickhouseSource. connection. connectionOptions |
object |
source. settings. clickhouseSource. connection. connectionOptions. user |
string |
source. settings. clickhouseSource. connection. connectionOptions. password |
object Password for user |
source. settings. clickhouseSource. connection. connectionOptions. password. raw |
string Raw secret value |
source. settings. clickhouseSource. connection. connectionOptions. database |
string Database |
source. settings. clickhouseSource. connection. connectionOptions. mdbClusterId |
string source.settings.clickhouseSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
source. settings. clickhouseSource. connection. connectionOptions. onPremise |
object source.settings.clickhouseSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
source. settings. clickhouseSource. connection. connectionOptions. onPremise. shards[] |
object |
source. settings. clickhouseSource. connection. connectionOptions. onPremise. shards[]. name |
string |
source. settings. clickhouseSource. connection. connectionOptions. onPremise. shards[]. hosts[] |
string |
source. settings. clickhouseSource. connection. connectionOptions. onPremise. httpPort |
string (int64) |
source. settings. clickhouseSource. connection. connectionOptions. onPremise. nativePort |
string (int64) |
source. settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode |
object |
source. settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. disabled |
object source.settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object source.settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. enabled |
object source.settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
source. settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
source. settings. clickhouseSource. includeTables[] |
string While list of tables for replication. If none or empty list is presented - will replicate all tables. Can contain * patterns. |
source. settings. clickhouseSource. excludeTables[] |
string Exclude list of tables for replication. If none or empty list is presented - will replicate all tables. Can contain * patterns. |
source. settings. clickhouseSource. subnetId |
string |
source. settings. clickhouseSource. securityGroups[] |
string |
source. settings. mysqlTarget |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. mysqlTarget. connection |
object Database connection settings |
source. settings. mysqlTarget. connection. mdbClusterId |
string source.settings.mysqlTarget.connection includes only one of the fields mdbClusterId , onPremise Managed Service for MySQL cluster ID |
source. settings. mysqlTarget. connection. onPremise |
object Connection options for on-premise MySQL source.settings.mysqlTarget.connection includes only one of the fields mdbClusterId , onPremise |
source. settings. mysqlTarget. connection. onPremise. port |
string (int64) Database port |
source. settings. mysqlTarget. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
source. settings. mysqlTarget. connection. onPremise. hosts[] |
string |
source. settings. mysqlTarget. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
source. settings. mysqlTarget. connection. onPremise. tlsMode. disabled |
object source.settings.mysqlTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. mysqlTarget. connection. onPremise. tlsMode. disabled. disabled |
object source.settings.mysqlTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. mysqlTarget. connection. onPremise. tlsMode. enabled |
object source.settings.mysqlTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
source. settings. mysqlTarget. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
source. settings. mysqlTarget. database |
string Database name Allowed to leave it empty, then the tables will be created in databases with the same names as on the source. If this field is empty, then you must fill below db schema for service table. |
source. settings. mysqlTarget. user |
string User for database access. |
source. settings. mysqlTarget. password |
object Password for database access. |
source. settings. mysqlTarget. password. raw |
string Raw secret value |
source. settings. mysqlTarget. sqlMode |
string Default: NO_AUTO_VALUE_ON_ZERO,NO_DIR_IN_CREATE,NO_ENGINE_SUBSTITUTION. |
source. settings. mysqlTarget. skipConstraintChecks |
boolean (boolean) Disable constraints checks Recommend to disable for increase replication speed, but if schema contain cascading operations we don't recommend to disable. This option set FOREIGN_KEY_CHECKS=0 and UNIQUE_CHECKS=0. |
source. settings. mysqlTarget. timezone |
string Database timezone Is used for parsing timestamps for saving source timezones. Accepts values from IANA timezone database. Default: local timezone. |
source. settings. mysqlTarget. cleanupPolicy |
string Cleanup policy Cleanup policy for activate, reactivate and reupload processes. Default is DISABLED.
|
source. settings. mysqlTarget. serviceDatabase |
string Database schema for service table Default: db name. Here created technical tables (__tm_keeper, __tm_gtid_keeper). |
source. settings. mysqlTarget. securityGroups[] |
string Security groups |
source. settings. postgresTarget |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. postgresTarget. connection |
object Database connection settings |
source. settings. postgresTarget. connection. mdbClusterId |
string source.settings.postgresTarget.connection includes only one of the fields mdbClusterId , onPremise Managed Service for PostgreSQL cluster ID |
source. settings. postgresTarget. connection. onPremise |
object Connection options for on-premise PostgreSQL source.settings.postgresTarget.connection includes only one of the fields mdbClusterId , onPremise |
source. settings. postgresTarget. connection. onPremise. port |
string (int64) Will be used if the cluster ID is not specified. |
source. settings. postgresTarget. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
source. settings. postgresTarget. connection. onPremise. hosts[] |
string |
source. settings. postgresTarget. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
source. settings. postgresTarget. connection. onPremise. tlsMode. disabled |
object source.settings.postgresTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. postgresTarget. connection. onPremise. tlsMode. disabled. disabled |
object source.settings.postgresTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. postgresTarget. connection. onPremise. tlsMode. enabled |
object source.settings.postgresTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
source. settings. postgresTarget. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
source. settings. postgresTarget. database |
string Database name |
source. settings. postgresTarget. user |
string User for database access. |
source. settings. postgresTarget. password |
object Password for database access. |
source. settings. postgresTarget. password. raw |
string Raw secret value |
source. settings. postgresTarget. cleanupPolicy |
string Cleanup policy for activate, reactivate and reupload processes. Default is truncate.
|
source. settings. postgresTarget. securityGroups[] |
string Security groups |
source. settings. clickhouseTarget |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. clickhouseTarget. connection |
object |
source. settings. clickhouseTarget. connection. connectionOptions |
object |
source. settings. clickhouseTarget. connection. connectionOptions. user |
string |
source. settings. clickhouseTarget. connection. connectionOptions. password |
object Password for user |
source. settings. clickhouseTarget. connection. connectionOptions. password. raw |
string Raw secret value |
source. settings. clickhouseTarget. connection. connectionOptions. database |
string Database |
source. settings. clickhouseTarget. connection. connectionOptions. mdbClusterId |
string source.settings.clickhouseTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
source. settings. clickhouseTarget. connection. connectionOptions. onPremise |
object source.settings.clickhouseTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
source. settings. clickhouseTarget. connection. connectionOptions. onPremise. shards[] |
object |
source. settings. clickhouseTarget. connection. connectionOptions. onPremise. shards[]. name |
string |
source. settings. clickhouseTarget. connection. connectionOptions. onPremise. shards[]. hosts[] |
string |
source. settings. clickhouseTarget. connection. connectionOptions. onPremise. httpPort |
string (int64) |
source. settings. clickhouseTarget. connection. connectionOptions. onPremise. nativePort |
string (int64) |
source. settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode |
object |
source. settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. disabled |
object source.settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object source.settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. enabled |
object source.settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
source. settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
source. settings. clickhouseTarget. subnetId |
string |
source. settings. clickhouseTarget. altNames[] |
object Alternative table names in target |
source. settings. clickhouseTarget. altNames[]. fromName |
string Source table name |
source. settings. clickhouseTarget. altNames[]. toName |
string Target table name |
source. settings. clickhouseTarget. cleanupPolicy |
string |
source. settings. clickhouseTarget. sharding |
object |
source. settings. clickhouseTarget. sharding. columnValueHash |
object source.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin |
source. settings. clickhouseTarget. sharding. columnValueHash. columnName |
string |
source. settings. clickhouseTarget. sharding. customMapping |
object source.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin |
source. settings. clickhouseTarget. sharding. customMapping. columnName |
string |
source. settings. clickhouseTarget. sharding. customMapping. mapping[] |
object |
source. settings. clickhouseTarget. sharding. customMapping. mapping[]. columnValue |
object |
source. settings. clickhouseTarget. sharding. customMapping. mapping[]. columnValue. stringValue |
string |
source. settings. clickhouseTarget. sharding. customMapping. mapping[]. shardName |
string |
source. settings. clickhouseTarget. sharding. transferId |
object source.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin Empty JSON object |
source. settings. clickhouseTarget. sharding. transferId. transferId |
object source.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin Empty JSON object |
source. settings. clickhouseTarget. sharding. roundRobin |
object source.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin Empty JSON object |
source. settings. clickhouseTarget. sharding. roundRobin. roundRobin |
object source.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin Empty JSON object |
source. settings. clickhouseTarget. clickhouseClusterName |
string |
source. settings. clickhouseTarget. securityGroups[] |
string |
source. settings. ydbTarget |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. ydbTarget. database |
string Path in YDB where to store tables |
source. settings. ydbTarget. instance |
string Instance of YDB. example: ydb-ru-prestable.yandex.net:2135 |
source. settings. ydbTarget. path |
string Path extension for database, each table will be layouted into this path |
source. settings. ydbTarget. serviceAccountId |
string |
source. settings. ydbTarget. cleanupPolicy |
string Cleanup policy |
source. settings. ydbTarget. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
source. settings. ydbTarget. saKeyContent |
string SA content |
source. settings. ydbTarget. securityGroups[] |
string Security groups |
source. settings. ydbTarget. isTableColumnOriented |
boolean (boolean) Should create column-oriented table (OLAP). By default it creates row-oriented (OLTP) |
source. settings. ydbTarget. defaultCompression |
string Compression that will be used for default columns family on YDB table creation |
source. settings. kafkaTarget |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. kafkaTarget. connection |
object Connection settings |
source. settings. kafkaTarget. connection. clusterId |
string source.settings.kafkaTarget.connection includes only one of the fields clusterId , onPremise Managed Service for Kafka cluster ID |
source. settings. kafkaTarget. connection. onPremise |
object Connection options for on-premise Kafka source.settings.kafkaTarget.connection includes only one of the fields clusterId , onPremise |
source. settings. kafkaTarget. connection. onPremise. brokerUrls[] |
string Kafka broker URLs |
source. settings. kafkaTarget. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
source. settings. kafkaTarget. connection. onPremise. tlsMode |
object TLS settings for broker connection. Disabled by default. |
source. settings. kafkaTarget. connection. onPremise. tlsMode. disabled |
object source.settings.kafkaTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. kafkaTarget. connection. onPremise. tlsMode. disabled. disabled |
object source.settings.kafkaTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. kafkaTarget. connection. onPremise. tlsMode. enabled |
object source.settings.kafkaTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
source. settings. kafkaTarget. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
source. settings. kafkaTarget. auth |
object Authentication settings |
source. settings. kafkaTarget. auth. sasl |
object Authentication with SASL source.settings.kafkaTarget.auth includes only one of the fields sasl , noAuth |
source. settings. kafkaTarget. auth. sasl. user |
string User name |
source. settings. kafkaTarget. auth. sasl. mechanism |
string SASL mechanism for authentication |
source. settings. kafkaTarget. auth. sasl. password |
object Password for user |
source. settings. kafkaTarget. auth. sasl. password. raw |
string Raw secret value |
source. settings. kafkaTarget. auth. noAuth |
object No authentication source.settings.kafkaTarget.auth includes only one of the fields sasl , noAuth |
source. settings. kafkaTarget. securityGroups[] |
string Security groups |
source. settings. kafkaTarget. topicSettings |
object Target topic settings |
source. settings. kafkaTarget. topicSettings. topic |
object Full topic name source.settings.kafkaTarget.topicSettings includes only one of the fields topic , topicPrefix |
source. settings. kafkaTarget. topicSettings. topic. topicName |
string Topic name |
source. settings. kafkaTarget. topicSettings. topic. saveTxOrder |
boolean (boolean) Save transactions order Not to split events queue into separate per-table queues. |
source. settings. kafkaTarget. topicSettings. topicPrefix |
string source.settings.kafkaTarget.topicSettings includes only one of the fields topic , topicPrefix Topic prefix Analogue of the Debezium setting database.server.name. Messages will be sent to topic with name <topic_prefix>. |
source. settings. kafkaTarget. serializer |
object Data serialization format settings Data serialization format |
source. settings. kafkaTarget. serializer. serializerAuto |
object Select the serialization format automatically source.settings.kafkaTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
source. settings. kafkaTarget. serializer. serializerJson |
object Serialize data in json format source.settings.kafkaTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
source. settings. kafkaTarget. serializer. serializerDebezium |
object Serialize data in debezium format source.settings.kafkaTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
source. settings. kafkaTarget. serializer. serializerDebezium. serializerParameters[] |
object Settings of sterilization parameters as key-value pairs |
source. settings. kafkaTarget. serializer. serializerDebezium. serializerParameters[]. key |
string Name of the serializer parameter |
source. settings. kafkaTarget. serializer. serializerDebezium. serializerParameters[]. value |
string Value of the serializer parameter |
source. settings. mongoTarget |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. mongoTarget. connection |
object |
source. settings. mongoTarget. connection. connectionOptions |
object |
source. settings. mongoTarget. connection. connectionOptions. user |
string User name |
source. settings. mongoTarget. connection. connectionOptions. password |
object Password for user |
source. settings. mongoTarget. connection. connectionOptions. password. raw |
string Raw secret value |
source. settings. mongoTarget. connection. connectionOptions. authSource |
string Database name associated with the credentials |
source. settings. mongoTarget. connection. connectionOptions. mdbClusterId |
string source.settings.mongoTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
source. settings. mongoTarget. connection. connectionOptions. onPremise |
object source.settings.mongoTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
source. settings. mongoTarget. connection. connectionOptions. onPremise. hosts[] |
string |
source. settings. mongoTarget. connection. connectionOptions. onPremise. port |
string (int64) |
source. settings. mongoTarget. connection. connectionOptions. onPremise. replicaSet |
string |
source. settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode |
object |
source. settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. disabled |
object source.settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object source.settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
source. settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. enabled |
object source.settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
source. settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
source. settings. mongoTarget. database |
string Database name |
source. settings. mongoTarget. cleanupPolicy |
string
|
source. settings. mongoTarget. subnetId |
string |
source. settings. mongoTarget. securityGroups[] |
string Security groups |
source. settings. metrikaSource |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. metrikaSource. counterIds[] |
string (int64) |
source. settings. metrikaSource. token |
object |
source. settings. metrikaSource. token. raw |
string Raw secret value |
source. settings. metrikaSource. streams[] |
object |
source. settings. metrikaSource. streams[]. type |
string |
source. settings. metrikaSource. streams[]. columns[] |
string |
source. settings. ydsTarget |
object source.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
source. settings. ydsTarget. database |
string Database |
source. settings. ydsTarget. stream |
string Stream |
source. settings. ydsTarget. serviceAccountId |
string SA which has read access to the stream. |
source. settings. ydsTarget. saveTxOrder |
boolean (boolean) Save transaction order Not to split events queue into separate per-table queues. Incompatible with setting Topic prefix, only with Topic full name. |
source. settings. ydsTarget. serializer |
object Data serialization format Data serialization format |
source. settings. ydsTarget. serializer. serializerAuto |
object Select the serialization format automatically source.settings.ydsTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
source. settings. ydsTarget. serializer. serializerJson |
object Serialize data in json format source.settings.ydsTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
source. settings. ydsTarget. serializer. serializerDebezium |
object Serialize data in debezium format source.settings.ydsTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
source. settings. ydsTarget. serializer. serializerDebezium. serializerParameters[] |
object Settings of sterilization parameters as key-value pairs |
source. settings. ydsTarget. serializer. serializerDebezium. serializerParameters[]. key |
string Name of the serializer parameter |
source. settings. ydsTarget. serializer. serializerDebezium. serializerParameters[]. value |
string Value of the serializer parameter |
source. settings. ydsTarget. endpoint |
string for dedicated db |
source. settings. ydsTarget. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
source. settings. ydsTarget. securityGroups[] |
string Security groups |
target | object |
target. id |
string |
target. folderId |
string |
target. name |
string |
target. description |
string |
target. labels |
object |
target. settings |
object |
target. settings. mysqlSource |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. mysqlSource. connection |
object Database connection settings |
target. settings. mysqlSource. connection. mdbClusterId |
string target.settings.mysqlSource.connection includes only one of the fields mdbClusterId , onPremise Managed Service for MySQL cluster ID |
target. settings. mysqlSource. connection. onPremise |
object Connection options for on-premise MySQL target.settings.mysqlSource.connection includes only one of the fields mdbClusterId , onPremise |
target. settings. mysqlSource. connection. onPremise. port |
string (int64) Database port |
target. settings. mysqlSource. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
target. settings. mysqlSource. connection. onPremise. hosts[] |
string |
target. settings. mysqlSource. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
target. settings. mysqlSource. connection. onPremise. tlsMode. disabled |
object target.settings.mysqlSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. mysqlSource. connection. onPremise. tlsMode. disabled. disabled |
object target.settings.mysqlSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. mysqlSource. connection. onPremise. tlsMode. enabled |
object target.settings.mysqlSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
target. settings. mysqlSource. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
target. settings. mysqlSource. database |
string Database name You can leave it empty, then it will be possible to transfer tables from several databases at the same time from this source. |
target. settings. mysqlSource. user |
string User for database access. |
target. settings. mysqlSource. password |
object Password for database access. |
target. settings. mysqlSource. password. raw |
string Raw secret value |
target. settings. mysqlSource. timezone |
string Database timezone Is used for parsing timestamps for saving source timezones. Accepts values from IANA timezone database. Default: local timezone. |
target. settings. mysqlSource. objectTransferSettings |
object Schema migration Select database objects to be transferred during activation or deactivation. |
target. settings. mysqlSource. objectTransferSettings. view |
string Views CREATE VIEW ...
|
target. settings. mysqlSource. objectTransferSettings. routine |
string Routines CREATE PROCEDURE ... ; CREATE FUNCTION ... ;
|
target. settings. mysqlSource. objectTransferSettings. trigger |
string Triggers CREATE TRIGGER ...
|
target. settings. mysqlSource. objectTransferSettings. tables |
string
|
target. settings. mysqlSource. includeTablesRegex[] |
string |
target. settings. mysqlSource. excludeTablesRegex[] |
string |
target. settings. mysqlSource. securityGroups[] |
string Security groups |
target. settings. mysqlSource. serviceDatabase |
string Database for service tables Default: data source database. Here created technical tables (__tm_keeper, __tm_gtid_keeper). |
target. settings. postgresSource |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. postgresSource. connection |
object Database connection settings |
target. settings. postgresSource. connection. mdbClusterId |
string target.settings.postgresSource.connection includes only one of the fields mdbClusterId , onPremise Managed Service for PostgreSQL cluster ID |
target. settings. postgresSource. connection. onPremise |
object Connection options for on-premise PostgreSQL target.settings.postgresSource.connection includes only one of the fields mdbClusterId , onPremise |
target. settings. postgresSource. connection. onPremise. port |
string (int64) Will be used if the cluster ID is not specified. |
target. settings. postgresSource. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
target. settings. postgresSource. connection. onPremise. hosts[] |
string |
target. settings. postgresSource. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
target. settings. postgresSource. connection. onPremise. tlsMode. disabled |
object target.settings.postgresSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. postgresSource. connection. onPremise. tlsMode. disabled. disabled |
object target.settings.postgresSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. postgresSource. connection. onPremise. tlsMode. enabled |
object target.settings.postgresSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
target. settings. postgresSource. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
target. settings. postgresSource. database |
string Database name |
target. settings. postgresSource. user |
string User for database access. |
target. settings. postgresSource. password |
object Password for database access. |
target. settings. postgresSource. password. raw |
string Raw secret value |
target. settings. postgresSource. includeTables[] |
string Included tables If none or empty list is presented, all tables are replicated. Full table name with schema. Can contain schema_name.* patterns. |
target. settings. postgresSource. excludeTables[] |
string Excluded tables If none or empty list is presented, all tables are replicated. Full table name with schema. Can contain schema_name.* patterns. |
target. settings. postgresSource. slotByteLagLimit |
string (int64) Maximum lag of replication slot (in bytes); after exceeding this limit replication will be aborted. |
target. settings. postgresSource. serviceSchema |
string Database schema for service tables (__consumer_keeper, __data_transfer_mole_finder). Default is public |
target. settings. postgresSource. objectTransferSettings |
object Select database objects to be transferred during activation or deactivation. |
target. settings. postgresSource. objectTransferSettings. sequence |
string Sequences CREATE SEQUENCE ...
|
target. settings. postgresSource. objectTransferSettings. sequenceOwnedBy |
string Owned sequences CREATE SEQUENCE ... OWNED BY ...
|
target. settings. postgresSource. objectTransferSettings. table |
string Tables CREATE TABLE ...
|
target. settings. postgresSource. objectTransferSettings. primaryKey |
string Primary keys ALTER TABLE ... ADD PRIMARY KEY ...
|
target. settings. postgresSource. objectTransferSettings. fkConstraint |
string Foreign keys ALTER TABLE ... ADD FOREIGN KEY ...
|
target. settings. postgresSource. objectTransferSettings. defaultValues |
string Default values ALTER TABLE ... ALTER COLUMN ... SET DEFAULT ...
|
target. settings. postgresSource. objectTransferSettings. constraint |
string Constraints ALTER TABLE ... ADD CONSTRAINT ...
|
target. settings. postgresSource. objectTransferSettings. index |
string Indexes CREATE INDEX ...
|
target. settings. postgresSource. objectTransferSettings. view |
string Views CREATE VIEW ...
|
target. settings. postgresSource. objectTransferSettings. function |
string Functions CREATE FUNCTION ...
|
target. settings. postgresSource. objectTransferSettings. trigger |
string Triggers CREATE TRIGGER ...
|
target. settings. postgresSource. objectTransferSettings. type |
string Types CREATE TYPE ...
|
target. settings. postgresSource. objectTransferSettings. rule |
string Rules CREATE RULE ...
|
target. settings. postgresSource. objectTransferSettings. collation |
string Collations CREATE COLLATION ...
|
target. settings. postgresSource. objectTransferSettings. policy |
string Policies CREATE POLICY ...
|
target. settings. postgresSource. objectTransferSettings. cast |
string Casts CREATE CAST ...
|
target. settings. postgresSource. objectTransferSettings. materializedView |
string Materialized views CREATE MATERIALIZED VIEW ...
|
target. settings. postgresSource. objectTransferSettings. sequenceSet |
string
|
target. settings. postgresSource. securityGroups[] |
string Security groups |
target. settings. ydbSource |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. ydbSource. database |
string Path in YDB where to store tables |
target. settings. ydbSource. instance |
string Instance of YDB. example: ydb-ru-prestable.yandex.net:2135 |
target. settings. ydbSource. paths[] |
string |
target. settings. ydbSource. serviceAccountId |
string |
target. settings. ydbSource. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
target. settings. ydbSource. saKeyContent |
string Authorization Key |
target. settings. ydbSource. securityGroups[] |
string Security groups |
target. settings. ydbSource. changefeedCustomName |
string Pre-created change feed |
target. settings. ydsSource |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. ydsSource. database |
string Database |
target. settings. ydsSource. stream |
string Stream |
target. settings. ydsSource. serviceAccountId |
string SA which has read access to the stream. |
target. settings. ydsSource. supportedCodecs[] |
string Compression codec |
target. settings. ydsSource. parser |
object Data parsing rules |
target. settings. ydsSource. parser. jsonParser |
object target.settings.ydsSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
target. settings. ydsSource. parser. jsonParser. dataSchema |
object |
target. settings. ydsSource. parser. jsonParser. dataSchema. fields |
object target.settings.ydsSource.parser.jsonParser.dataSchema includes only one of the fields fields , jsonFields |
target. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[] |
object Column schema |
target. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[]. name |
string |
target. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[]. type |
string |
target. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[]. key |
boolean (boolean) |
target. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[]. required |
boolean (boolean) |
target. settings. ydsSource. parser. jsonParser. dataSchema. fields. fields[]. path |
string |
target. settings. ydsSource. parser. jsonParser. dataSchema. jsonFields |
string target.settings.ydsSource.parser.jsonParser.dataSchema includes only one of the fields fields , jsonFields |
target. settings. ydsSource. parser. jsonParser. nullKeysAllowed |
boolean (boolean) Allow null keys, if no - null keys will be putted to unparsed data |
target. settings. ydsSource. parser. jsonParser. addRestColumn |
boolean (boolean) Will add _rest column for all unknown fields |
target. settings. ydsSource. parser. jsonParser. unescapeStringValues |
boolean (boolean) Unescape string values |
target. settings. ydsSource. parser. auditTrailsV1Parser |
object target.settings.ydsSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
target. settings. ydsSource. parser. cloudLoggingParser |
object target.settings.ydsSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
target. settings. ydsSource. parser. tskvParser |
object target.settings.ydsSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
target. settings. ydsSource. parser. tskvParser. dataSchema |
object |
target. settings. ydsSource. parser. tskvParser. dataSchema. fields |
object target.settings.ydsSource.parser.tskvParser.dataSchema includes only one of the fields fields , jsonFields |
target. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[] |
object Column schema |
target. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[]. name |
string |
target. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[]. type |
string |
target. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[]. key |
boolean (boolean) |
target. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[]. required |
boolean (boolean) |
target. settings. ydsSource. parser. tskvParser. dataSchema. fields. fields[]. path |
string |
target. settings. ydsSource. parser. tskvParser. dataSchema. jsonFields |
string target.settings.ydsSource.parser.tskvParser.dataSchema includes only one of the fields fields , jsonFields |
target. settings. ydsSource. parser. tskvParser. nullKeysAllowed |
boolean (boolean) Allow null keys, if no - null keys will be putted to unparsed data |
target. settings. ydsSource. parser. tskvParser. addRestColumn |
boolean (boolean) Will add _rest column for all unknown fields |
target. settings. ydsSource. parser. tskvParser. unescapeStringValues |
boolean (boolean) Unescape string values |
target. settings. ydsSource. allowTtlRewind |
boolean (boolean) Should continue working, if consumer read lag exceed TTL of topic False: stop the transfer in error state, if detected lost data. True: continue working with losing part of data |
target. settings. ydsSource. endpoint |
string for dedicated db |
target. settings. ydsSource. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
target. settings. ydsSource. securityGroups[] |
string Security groups |
target. settings. ydsSource. consumer |
string for important streams |
target. settings. kafkaSource |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. kafkaSource. connection |
object Connection settings |
target. settings. kafkaSource. connection. clusterId |
string target.settings.kafkaSource.connection includes only one of the fields clusterId , onPremise Managed Service for Kafka cluster ID |
target. settings. kafkaSource. connection. onPremise |
object Connection options for on-premise Kafka target.settings.kafkaSource.connection includes only one of the fields clusterId , onPremise |
target. settings. kafkaSource. connection. onPremise. brokerUrls[] |
string Kafka broker URLs |
target. settings. kafkaSource. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
target. settings. kafkaSource. connection. onPremise. tlsMode |
object TLS settings for broker connection. Disabled by default. |
target. settings. kafkaSource. connection. onPremise. tlsMode. disabled |
object target.settings.kafkaSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. kafkaSource. connection. onPremise. tlsMode. disabled. disabled |
object target.settings.kafkaSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. kafkaSource. connection. onPremise. tlsMode. enabled |
object target.settings.kafkaSource.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
target. settings. kafkaSource. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
target. settings. kafkaSource. auth |
object Authentication settings |
target. settings. kafkaSource. auth. sasl |
object Authentication with SASL target.settings.kafkaSource.auth includes only one of the fields sasl , noAuth |
target. settings. kafkaSource. auth. sasl. user |
string User name |
target. settings. kafkaSource. auth. sasl. mechanism |
string SASL mechanism for authentication |
target. settings. kafkaSource. auth. sasl. password |
object Password for user |
target. settings. kafkaSource. auth. sasl. password. raw |
string Raw secret value |
target. settings. kafkaSource. auth. noAuth |
object No authentication target.settings.kafkaSource.auth includes only one of the fields sasl , noAuth |
target. settings. kafkaSource. securityGroups[] |
string Security groups |
target. settings. kafkaSource. topicName |
string Full source topic name Deprecated in favor of topic names |
target. settings. kafkaSource. transformer |
object Data transformation rules |
target. settings. kafkaSource. transformer. cloudFunction |
string Cloud function |
target. settings. kafkaSource. transformer. numberOfRetries |
string (int64) Number of retries |
target. settings. kafkaSource. transformer. bufferSize |
string Buffer size for function |
target. settings. kafkaSource. transformer. bufferFlushInterval |
string Flush interval |
target. settings. kafkaSource. transformer. invocationTimeout |
string Invocation timeout |
target. settings. kafkaSource. transformer. serviceAccountId |
string Service account |
target. settings. kafkaSource. parser |
object Data parsing rules |
target. settings. kafkaSource. parser. jsonParser |
object target.settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
target. settings. kafkaSource. parser. jsonParser. dataSchema |
object |
target. settings. kafkaSource. parser. jsonParser. dataSchema. fields |
object target.settings.kafkaSource.parser.jsonParser.dataSchema includes only one of the fields fields , jsonFields |
target. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[] |
object Column schema |
target. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. name |
string |
target. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. type |
string |
target. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. key |
boolean (boolean) |
target. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. required |
boolean (boolean) |
target. settings. kafkaSource. parser. jsonParser. dataSchema. fields. fields[]. path |
string |
target. settings. kafkaSource. parser. jsonParser. dataSchema. jsonFields |
string target.settings.kafkaSource.parser.jsonParser.dataSchema includes only one of the fields fields , jsonFields |
target. settings. kafkaSource. parser. jsonParser. nullKeysAllowed |
boolean (boolean) Allow null keys, if no - null keys will be putted to unparsed data |
target. settings. kafkaSource. parser. jsonParser. addRestColumn |
boolean (boolean) Will add _rest column for all unknown fields |
target. settings. kafkaSource. parser. jsonParser. unescapeStringValues |
boolean (boolean) Unescape string values |
target. settings. kafkaSource. parser. auditTrailsV1Parser |
object target.settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
target. settings. kafkaSource. parser. cloudLoggingParser |
object target.settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
target. settings. kafkaSource. parser. tskvParser |
object target.settings.kafkaSource.parser includes only one of the fields jsonParser , auditTrailsV1Parser , cloudLoggingParser , tskvParser |
target. settings. kafkaSource. parser. tskvParser. dataSchema |
object |
target. settings. kafkaSource. parser. tskvParser. dataSchema. fields |
object target.settings.kafkaSource.parser.tskvParser.dataSchema includes only one of the fields fields , jsonFields |
target. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[] |
object Column schema |
target. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. name |
string |
target. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. type |
string |
target. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. key |
boolean (boolean) |
target. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. required |
boolean (boolean) |
target. settings. kafkaSource. parser. tskvParser. dataSchema. fields. fields[]. path |
string |
target. settings. kafkaSource. parser. tskvParser. dataSchema. jsonFields |
string target.settings.kafkaSource.parser.tskvParser.dataSchema includes only one of the fields fields , jsonFields |
target. settings. kafkaSource. parser. tskvParser. nullKeysAllowed |
boolean (boolean) Allow null keys, if no - null keys will be putted to unparsed data |
target. settings. kafkaSource. parser. tskvParser. addRestColumn |
boolean (boolean) Will add _rest column for all unknown fields |
target. settings. kafkaSource. parser. tskvParser. unescapeStringValues |
boolean (boolean) Unescape string values |
target. settings. kafkaSource. topicNames[] |
string List of topic names to read |
target. settings. mongoSource |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. mongoSource. connection |
object |
target. settings. mongoSource. connection. connectionOptions |
object |
target. settings. mongoSource. connection. connectionOptions. user |
string User name |
target. settings. mongoSource. connection. connectionOptions. password |
object Password for user |
target. settings. mongoSource. connection. connectionOptions. password. raw |
string Raw secret value |
target. settings. mongoSource. connection. connectionOptions. authSource |
string Database name associated with the credentials |
target. settings. mongoSource. connection. connectionOptions. mdbClusterId |
string target.settings.mongoSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
target. settings. mongoSource. connection. connectionOptions. onPremise |
object target.settings.mongoSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
target. settings. mongoSource. connection. connectionOptions. onPremise. hosts[] |
string |
target. settings. mongoSource. connection. connectionOptions. onPremise. port |
string (int64) |
target. settings. mongoSource. connection. connectionOptions. onPremise. replicaSet |
string |
target. settings. mongoSource. connection. connectionOptions. onPremise. tlsMode |
object |
target. settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. disabled |
object target.settings.mongoSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object target.settings.mongoSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. enabled |
object target.settings.mongoSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
target. settings. mongoSource. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
target. settings. mongoSource. subnetId |
string |
target. settings. mongoSource. collections[] |
object List of collections for replication. Empty list implies replication of all tables on the deployment. Allowed to use * as collection name. |
target. settings. mongoSource. collections[]. databaseName |
string |
target. settings. mongoSource. collections[]. collectionName |
string |
target. settings. mongoSource. excludedCollections[] |
object List of forbidden collections for replication. Allowed to use * as collection name for forbid all collections of concrete schema. |
target. settings. mongoSource. excludedCollections[]. databaseName |
string |
target. settings. mongoSource. excludedCollections[]. collectionName |
string |
target. settings. mongoSource. secondaryPreferredMode |
boolean (boolean) Read mode for mongo client |
target. settings. mongoSource. securityGroups[] |
string Security groups |
target. settings. clickhouseSource |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. clickhouseSource. connection |
object |
target. settings. clickhouseSource. connection. connectionOptions |
object |
target. settings. clickhouseSource. connection. connectionOptions. user |
string |
target. settings. clickhouseSource. connection. connectionOptions. password |
object Password for user |
target. settings. clickhouseSource. connection. connectionOptions. password. raw |
string Raw secret value |
target. settings. clickhouseSource. connection. connectionOptions. database |
string Database |
target. settings. clickhouseSource. connection. connectionOptions. mdbClusterId |
string target.settings.clickhouseSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
target. settings. clickhouseSource. connection. connectionOptions. onPremise |
object target.settings.clickhouseSource.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
target. settings. clickhouseSource. connection. connectionOptions. onPremise. shards[] |
object |
target. settings. clickhouseSource. connection. connectionOptions. onPremise. shards[]. name |
string |
target. settings. clickhouseSource. connection. connectionOptions. onPremise. shards[]. hosts[] |
string |
target. settings. clickhouseSource. connection. connectionOptions. onPremise. httpPort |
string (int64) |
target. settings. clickhouseSource. connection. connectionOptions. onPremise. nativePort |
string (int64) |
target. settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode |
object |
target. settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. disabled |
object target.settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object target.settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. enabled |
object target.settings.clickhouseSource.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
target. settings. clickhouseSource. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
target. settings. clickhouseSource. includeTables[] |
string While list of tables for replication. If none or empty list is presented - will replicate all tables. Can contain * patterns. |
target. settings. clickhouseSource. excludeTables[] |
string Exclude list of tables for replication. If none or empty list is presented - will replicate all tables. Can contain * patterns. |
target. settings. clickhouseSource. subnetId |
string |
target. settings. clickhouseSource. securityGroups[] |
string |
target. settings. mysqlTarget |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. mysqlTarget. connection |
object Database connection settings |
target. settings. mysqlTarget. connection. mdbClusterId |
string target.settings.mysqlTarget.connection includes only one of the fields mdbClusterId , onPremise Managed Service for MySQL cluster ID |
target. settings. mysqlTarget. connection. onPremise |
object Connection options for on-premise MySQL target.settings.mysqlTarget.connection includes only one of the fields mdbClusterId , onPremise |
target. settings. mysqlTarget. connection. onPremise. port |
string (int64) Database port |
target. settings. mysqlTarget. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
target. settings. mysqlTarget. connection. onPremise. hosts[] |
string |
target. settings. mysqlTarget. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
target. settings. mysqlTarget. connection. onPremise. tlsMode. disabled |
object target.settings.mysqlTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. mysqlTarget. connection. onPremise. tlsMode. disabled. disabled |
object target.settings.mysqlTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. mysqlTarget. connection. onPremise. tlsMode. enabled |
object target.settings.mysqlTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
target. settings. mysqlTarget. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
target. settings. mysqlTarget. database |
string Database name Allowed to leave it empty, then the tables will be created in databases with the same names as on the source. If this field is empty, then you must fill below db schema for service table. |
target. settings. mysqlTarget. user |
string User for database access. |
target. settings. mysqlTarget. password |
object Password for database access. |
target. settings. mysqlTarget. password. raw |
string Raw secret value |
target. settings. mysqlTarget. sqlMode |
string Default: NO_AUTO_VALUE_ON_ZERO,NO_DIR_IN_CREATE,NO_ENGINE_SUBSTITUTION. |
target. settings. mysqlTarget. skipConstraintChecks |
boolean (boolean) Disable constraints checks Recommend to disable for increase replication speed, but if schema contain cascading operations we don't recommend to disable. This option set FOREIGN_KEY_CHECKS=0 and UNIQUE_CHECKS=0. |
target. settings. mysqlTarget. timezone |
string Database timezone Is used for parsing timestamps for saving source timezones. Accepts values from IANA timezone database. Default: local timezone. |
target. settings. mysqlTarget. cleanupPolicy |
string Cleanup policy Cleanup policy for activate, reactivate and reupload processes. Default is DISABLED.
|
target. settings. mysqlTarget. serviceDatabase |
string Database schema for service table Default: db name. Here created technical tables (__tm_keeper, __tm_gtid_keeper). |
target. settings. mysqlTarget. securityGroups[] |
string Security groups |
target. settings. postgresTarget |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. postgresTarget. connection |
object Database connection settings |
target. settings. postgresTarget. connection. mdbClusterId |
string target.settings.postgresTarget.connection includes only one of the fields mdbClusterId , onPremise Managed Service for PostgreSQL cluster ID |
target. settings. postgresTarget. connection. onPremise |
object Connection options for on-premise PostgreSQL target.settings.postgresTarget.connection includes only one of the fields mdbClusterId , onPremise |
target. settings. postgresTarget. connection. onPremise. port |
string (int64) Will be used if the cluster ID is not specified. |
target. settings. postgresTarget. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
target. settings. postgresTarget. connection. onPremise. hosts[] |
string |
target. settings. postgresTarget. connection. onPremise. tlsMode |
object TLS settings for server connection. Disabled by default. |
target. settings. postgresTarget. connection. onPremise. tlsMode. disabled |
object target.settings.postgresTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. postgresTarget. connection. onPremise. tlsMode. disabled. disabled |
object target.settings.postgresTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. postgresTarget. connection. onPremise. tlsMode. enabled |
object target.settings.postgresTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
target. settings. postgresTarget. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
target. settings. postgresTarget. database |
string Database name |
target. settings. postgresTarget. user |
string User for database access. |
target. settings. postgresTarget. password |
object Password for database access. |
target. settings. postgresTarget. password. raw |
string Raw secret value |
target. settings. postgresTarget. cleanupPolicy |
string Cleanup policy for activate, reactivate and reupload processes. Default is truncate.
|
target. settings. postgresTarget. securityGroups[] |
string Security groups |
target. settings. clickhouseTarget |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. clickhouseTarget. connection |
object |
target. settings. clickhouseTarget. connection. connectionOptions |
object |
target. settings. clickhouseTarget. connection. connectionOptions. user |
string |
target. settings. clickhouseTarget. connection. connectionOptions. password |
object Password for user |
target. settings. clickhouseTarget. connection. connectionOptions. password. raw |
string Raw secret value |
target. settings. clickhouseTarget. connection. connectionOptions. database |
string Database |
target. settings. clickhouseTarget. connection. connectionOptions. mdbClusterId |
string target.settings.clickhouseTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
target. settings. clickhouseTarget. connection. connectionOptions. onPremise |
object target.settings.clickhouseTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
target. settings. clickhouseTarget. connection. connectionOptions. onPremise. shards[] |
object |
target. settings. clickhouseTarget. connection. connectionOptions. onPremise. shards[]. name |
string |
target. settings. clickhouseTarget. connection. connectionOptions. onPremise. shards[]. hosts[] |
string |
target. settings. clickhouseTarget. connection. connectionOptions. onPremise. httpPort |
string (int64) |
target. settings. clickhouseTarget. connection. connectionOptions. onPremise. nativePort |
string (int64) |
target. settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode |
object |
target. settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. disabled |
object target.settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object target.settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. enabled |
object target.settings.clickhouseTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
target. settings. clickhouseTarget. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
target. settings. clickhouseTarget. subnetId |
string |
target. settings. clickhouseTarget. altNames[] |
object Alternative table names in target |
target. settings. clickhouseTarget. altNames[]. fromName |
string Source table name |
target. settings. clickhouseTarget. altNames[]. toName |
string Target table name |
target. settings. clickhouseTarget. cleanupPolicy |
string |
target. settings. clickhouseTarget. sharding |
object |
target. settings. clickhouseTarget. sharding. columnValueHash |
object target.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin |
target. settings. clickhouseTarget. sharding. columnValueHash. columnName |
string |
target. settings. clickhouseTarget. sharding. customMapping |
object target.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin |
target. settings. clickhouseTarget. sharding. customMapping. columnName |
string |
target. settings. clickhouseTarget. sharding. customMapping. mapping[] |
object |
target. settings. clickhouseTarget. sharding. customMapping. mapping[]. columnValue |
object |
target. settings. clickhouseTarget. sharding. customMapping. mapping[]. columnValue. stringValue |
string |
target. settings. clickhouseTarget. sharding. customMapping. mapping[]. shardName |
string |
target. settings. clickhouseTarget. sharding. transferId |
object target.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin Empty JSON object |
target. settings. clickhouseTarget. sharding. transferId. transferId |
object target.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin Empty JSON object |
target. settings. clickhouseTarget. sharding. roundRobin |
object target.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin Empty JSON object |
target. settings. clickhouseTarget. sharding. roundRobin. roundRobin |
object target.settings.clickhouseTarget.sharding includes only one of the fields columnValueHash , customMapping , transferId , roundRobin Empty JSON object |
target. settings. clickhouseTarget. clickhouseClusterName |
string |
target. settings. clickhouseTarget. securityGroups[] |
string |
target. settings. ydbTarget |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. ydbTarget. database |
string Path in YDB where to store tables |
target. settings. ydbTarget. instance |
string Instance of YDB. example: ydb-ru-prestable.yandex.net:2135 |
target. settings. ydbTarget. path |
string Path extension for database, each table will be layouted into this path |
target. settings. ydbTarget. serviceAccountId |
string |
target. settings. ydbTarget. cleanupPolicy |
string Cleanup policy |
target. settings. ydbTarget. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
target. settings. ydbTarget. saKeyContent |
string SA content |
target. settings. ydbTarget. securityGroups[] |
string Security groups |
target. settings. ydbTarget. isTableColumnOriented |
boolean (boolean) Should create column-oriented table (OLAP). By default it creates row-oriented (OLTP) |
target. settings. ydbTarget. defaultCompression |
string Compression that will be used for default columns family on YDB table creation |
target. settings. kafkaTarget |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. kafkaTarget. connection |
object Connection settings |
target. settings. kafkaTarget. connection. clusterId |
string target.settings.kafkaTarget.connection includes only one of the fields clusterId , onPremise Managed Service for Kafka cluster ID |
target. settings. kafkaTarget. connection. onPremise |
object Connection options for on-premise Kafka target.settings.kafkaTarget.connection includes only one of the fields clusterId , onPremise |
target. settings. kafkaTarget. connection. onPremise. brokerUrls[] |
string Kafka broker URLs |
target. settings. kafkaTarget. connection. onPremise. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
target. settings. kafkaTarget. connection. onPremise. tlsMode |
object TLS settings for broker connection. Disabled by default. |
target. settings. kafkaTarget. connection. onPremise. tlsMode. disabled |
object target.settings.kafkaTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. kafkaTarget. connection. onPremise. tlsMode. disabled. disabled |
object target.settings.kafkaTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. kafkaTarget. connection. onPremise. tlsMode. enabled |
object target.settings.kafkaTarget.connection.onPremise.tlsMode includes only one of the fields disabled , enabled |
target. settings. kafkaTarget. connection. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
target. settings. kafkaTarget. auth |
object Authentication settings |
target. settings. kafkaTarget. auth. sasl |
object Authentication with SASL target.settings.kafkaTarget.auth includes only one of the fields sasl , noAuth |
target. settings. kafkaTarget. auth. sasl. user |
string User name |
target. settings. kafkaTarget. auth. sasl. mechanism |
string SASL mechanism for authentication |
target. settings. kafkaTarget. auth. sasl. password |
object Password for user |
target. settings. kafkaTarget. auth. sasl. password. raw |
string Raw secret value |
target. settings. kafkaTarget. auth. noAuth |
object No authentication target.settings.kafkaTarget.auth includes only one of the fields sasl , noAuth |
target. settings. kafkaTarget. securityGroups[] |
string Security groups |
target. settings. kafkaTarget. topicSettings |
object Target topic settings |
target. settings. kafkaTarget. topicSettings. topic |
object Full topic name target.settings.kafkaTarget.topicSettings includes only one of the fields topic , topicPrefix |
target. settings. kafkaTarget. topicSettings. topic. topicName |
string Topic name |
target. settings. kafkaTarget. topicSettings. topic. saveTxOrder |
boolean (boolean) Save transactions order Not to split events queue into separate per-table queues. |
target. settings. kafkaTarget. topicSettings. topicPrefix |
string target.settings.kafkaTarget.topicSettings includes only one of the fields topic , topicPrefix Topic prefix Analogue of the Debezium setting database.server.name. Messages will be sent to topic with name <topic_prefix>. |
target. settings. kafkaTarget. serializer |
object Data serialization format settings Data serialization format |
target. settings. kafkaTarget. serializer. serializerAuto |
object Select the serialization format automatically target.settings.kafkaTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
target. settings. kafkaTarget. serializer. serializerJson |
object Serialize data in json format target.settings.kafkaTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
target. settings. kafkaTarget. serializer. serializerDebezium |
object Serialize data in debezium format target.settings.kafkaTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
target. settings. kafkaTarget. serializer. serializerDebezium. serializerParameters[] |
object Settings of sterilization parameters as key-value pairs |
target. settings. kafkaTarget. serializer. serializerDebezium. serializerParameters[]. key |
string Name of the serializer parameter |
target. settings. kafkaTarget. serializer. serializerDebezium. serializerParameters[]. value |
string Value of the serializer parameter |
target. settings. mongoTarget |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. mongoTarget. connection |
object |
target. settings. mongoTarget. connection. connectionOptions |
object |
target. settings. mongoTarget. connection. connectionOptions. user |
string User name |
target. settings. mongoTarget. connection. connectionOptions. password |
object Password for user |
target. settings. mongoTarget. connection. connectionOptions. password. raw |
string Raw secret value |
target. settings. mongoTarget. connection. connectionOptions. authSource |
string Database name associated with the credentials |
target. settings. mongoTarget. connection. connectionOptions. mdbClusterId |
string target.settings.mongoTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
target. settings. mongoTarget. connection. connectionOptions. onPremise |
object target.settings.mongoTarget.connection.connectionOptions includes only one of the fields mdbClusterId , onPremise |
target. settings. mongoTarget. connection. connectionOptions. onPremise. hosts[] |
string |
target. settings. mongoTarget. connection. connectionOptions. onPremise. port |
string (int64) |
target. settings. mongoTarget. connection. connectionOptions. onPremise. replicaSet |
string |
target. settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode |
object |
target. settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. disabled |
object target.settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. disabled. disabled |
object target.settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled Empty JSON object |
target. settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. enabled |
object target.settings.mongoTarget.connection.connectionOptions.onPremise.tlsMode includes only one of the fields disabled , enabled |
target. settings. mongoTarget. connection. connectionOptions. onPremise. tlsMode. enabled. caCertificate |
string CA certificate X.509 certificate of the certificate authority which issued the server's certificate, in PEM format. When CA certificate is specified TLS is used to connect to the server. |
target. settings. mongoTarget. database |
string Database name |
target. settings. mongoTarget. cleanupPolicy |
string
|
target. settings. mongoTarget. subnetId |
string |
target. settings. mongoTarget. securityGroups[] |
string Security groups |
target. settings. metrikaSource |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. metrikaSource. counterIds[] |
string (int64) |
target. settings. metrikaSource. token |
object |
target. settings. metrikaSource. token. raw |
string Raw secret value |
target. settings. metrikaSource. streams[] |
object |
target. settings. metrikaSource. streams[]. type |
string |
target. settings. metrikaSource. streams[]. columns[] |
string |
target. settings. ydsTarget |
object target.settings includes only one of the fields mysqlSource , postgresSource , ydbSource , ydsSource , kafkaSource , mongoSource , clickhouseSource , mysqlTarget , postgresTarget , clickhouseTarget , ydbTarget , kafkaTarget , mongoTarget , metrikaSource , ydsTarget |
target. settings. ydsTarget. database |
string Database |
target. settings. ydsTarget. stream |
string Stream |
target. settings. ydsTarget. serviceAccountId |
string SA which has read access to the stream. |
target. settings. ydsTarget. saveTxOrder |
boolean (boolean) Save transaction order Not to split events queue into separate per-table queues. Incompatible with setting Topic prefix, only with Topic full name. |
target. settings. ydsTarget. serializer |
object Data serialization format Data serialization format |
target. settings. ydsTarget. serializer. serializerAuto |
object Select the serialization format automatically target.settings.ydsTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
target. settings. ydsTarget. serializer. serializerJson |
object Serialize data in json format target.settings.ydsTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
target. settings. ydsTarget. serializer. serializerDebezium |
object Serialize data in debezium format target.settings.ydsTarget.serializer includes only one of the fields serializerAuto , serializerJson , serializerDebezium |
target. settings. ydsTarget. serializer. serializerDebezium. serializerParameters[] |
object Settings of sterilization parameters as key-value pairs |
target. settings. ydsTarget. serializer. serializerDebezium. serializerParameters[]. key |
string Name of the serializer parameter |
target. settings. ydsTarget. serializer. serializerDebezium. serializerParameters[]. value |
string Value of the serializer parameter |
target. settings. ydsTarget. endpoint |
string for dedicated db |
target. settings. ydsTarget. subnetId |
string Network interface for endpoint. If none will assume public ipv4 |
target. settings. ydsTarget. securityGroups[] |
string Security groups |
runtime | object |
runtime. ycRuntime |
object |
runtime. ycRuntime. jobCount |
string (int64) |
runtime. ycRuntime. uploadShardParams |
object |
runtime. ycRuntime. uploadShardParams. jobCount |
string (int64) |
runtime. ycRuntime. uploadShardParams. processCount |
string (int64) |
status | string
|
type | string
|
warning | string |
transformation | object Transformation is converting data using special transformer functions. These functions are executed on a data stream, applied to each data change item, and transform them. A transformer can be run at both the metadata and data levels. Data can only be transformed if the source and target are of different types. |
transformation. transformers[] |
object Transformers are set as a list. When activating a transfer, a transformation plan is made for the tables that match the specified criteria. Transformers are applied to the tables in the sequence specified in the list. |
transformation. transformers[]. maskField |
object transformation.transformers[] includes only one of the fields maskField , filterColumns , renameTables , replacePrimaryKey , convertToString , sharderTransformer , tableSplitterTransformer , filterRows Mask field transformer allows you to hash data |
transformation. transformers[]. maskField. tables |
object List of included and excluded tables Filter tables using lists of included and excluded tables. |
transformation. transformers[]. maskField. tables. includeTables[] |
string List of tables that will be included to transfer |
transformation. transformers[]. maskField. tables. excludeTables[] |
string List of tables that will be excluded to transfer |
transformation. transformers[]. maskField. columns[] |
string Specify the name of the column for data masking (a regular expression). |
transformation. transformers[]. maskField. function |
object Mask function Mask function |
transformation. transformers[]. maskField. function. maskFunctionHash |
object Hash data using HMAC |
transformation. transformers[]. maskField. function. maskFunctionHash. userDefinedSalt |
string This string will be used in the HMAC(sha256, salt) function applied to the column data. |
transformation. transformers[]. filterColumns |
object transformation.transformers[] includes only one of the fields maskField , filterColumns , renameTables , replacePrimaryKey , convertToString , sharderTransformer , tableSplitterTransformer , filterRows Set up a list of table columns to transfer |
transformation. transformers[]. filterColumns. tables |
object List of the tables to filter using lists of included and excluded tables. Filter tables using lists of included and excluded tables. |
transformation. transformers[]. filterColumns. tables. includeTables[] |
string List of tables that will be included to transfer |
transformation. transformers[]. filterColumns. tables. excludeTables[] |
string List of tables that will be excluded to transfer |
transformation. transformers[]. filterColumns. columns |
object List of the columns to transfer to the target tables using lists of included and excluded columns. Filter columns using lists of included and excluded columns. |
transformation. transformers[]. filterColumns. columns. includeColumns[] |
string List of columns that will be included to transfer |
transformation. transformers[]. filterColumns. columns. excludeColumns[] |
string List of columns that will be excluded to transfer |
transformation. transformers[]. renameTables |
object transformation.transformers[] includes only one of the fields maskField , filterColumns , renameTables , replacePrimaryKey , convertToString , sharderTransformer , tableSplitterTransformer , filterRows Set rules for renaming tables by specifying the current names of the tables in the source and new names for these tables in the target. |
transformation. transformers[]. renameTables. renameTables[] |
object List of renaming rules |
transformation. transformers[]. renameTables. renameTables[]. originalName |
object Specify the current names of the table in the source |
transformation. transformers[]. renameTables. renameTables[]. originalName. nameSpace |
string |
transformation. transformers[]. renameTables. renameTables[]. originalName. name |
string |
transformation. transformers[]. renameTables. renameTables[]. newName |
object Specify the new names for this table in the target |
transformation. transformers[]. renameTables. renameTables[]. newName. nameSpace |
string |
transformation. transformers[]. renameTables. renameTables[]. newName. name |
string |
transformation. transformers[]. replacePrimaryKey |
object transformation.transformers[] includes only one of the fields maskField , filterColumns , renameTables , replacePrimaryKey , convertToString , sharderTransformer , tableSplitterTransformer , filterRows Override primary keys |
transformation. transformers[]. replacePrimaryKey. tables |
object List of included and excluded tables Filter tables using lists of included and excluded tables. |
transformation. transformers[]. replacePrimaryKey. tables. includeTables[] |
string List of tables that will be included to transfer |
transformation. transformers[]. replacePrimaryKey. tables. excludeTables[] |
string List of tables that will be excluded to transfer |
transformation. transformers[]. replacePrimaryKey. keys[] |
string List of columns to be used as primary keys |
transformation. transformers[]. convertToString |
object transformation.transformers[] includes only one of the fields maskField , filterColumns , renameTables , replacePrimaryKey , convertToString , sharderTransformer , tableSplitterTransformer , filterRows Convert column values to strings The values will be converted depending on the source type Conversion rules are described here: https://cloud.yandex.com/en/docs/data-transfer/concepts/data-transformation#convert-to-string |
transformation. transformers[]. convertToString. tables |
object List of included and excluded tables Filter tables using lists of included and excluded tables. |
transformation. transformers[]. convertToString. tables. includeTables[] |
string List of tables that will be included to transfer |
transformation. transformers[]. convertToString. tables. excludeTables[] |
string List of tables that will be excluded to transfer |
transformation. transformers[]. convertToString. columns |
object List of included and excluded columns Filter columns using lists of included and excluded columns. |
transformation. transformers[]. convertToString. columns. includeColumns[] |
string List of columns that will be included to transfer |
transformation. transformers[]. convertToString. columns. excludeColumns[] |
string List of columns that will be excluded to transfer |
transformation. transformers[]. sharderTransformer |
object transformation.transformers[] includes only one of the fields maskField , filterColumns , renameTables , replacePrimaryKey , convertToString , sharderTransformer , tableSplitterTransformer , filterRows Set the number of shards for particular tables and a list of columns whose values will be used for calculating a hash to determine a shard. |
transformation. transformers[]. sharderTransformer. tables |
object List of included and excluded tables Filter tables using lists of included and excluded tables. |
transformation. transformers[]. sharderTransformer. tables. includeTables[] |
string List of tables that will be included to transfer |
transformation. transformers[]. sharderTransformer. tables. excludeTables[] |
string List of tables that will be excluded to transfer |
transformation. transformers[]. sharderTransformer. columns |
object List of included and excluded columns Filter columns using lists of included and excluded columns. |
transformation. transformers[]. sharderTransformer. columns. includeColumns[] |
string List of columns that will be included to transfer |
transformation. transformers[]. sharderTransformer. columns. excludeColumns[] |
string List of columns that will be excluded to transfer |
transformation. transformers[]. sharderTransformer. shardsCount |
string (int64) Number of shards |
transformation. transformers[]. tableSplitterTransformer |
object transformation.transformers[] includes only one of the fields maskField , filterColumns , renameTables , replacePrimaryKey , convertToString , sharderTransformer , tableSplitterTransformer , filterRows A transfer splits the X table into multiple tables (X_1, X_2, ..., X_n) based on data. If a row was located in the X table before it was split, it is now in the X_i table, where i is determined by the column list and split string parameters. Example: If the column list has two columns, month of birth and gender, specified and the split string states @, information about an employee whose name is John and who was born on February 11, 1984, from the Employees table will get to a new table named Employees@February@male. |
transformation. transformers[]. tableSplitterTransformer. tables |
object List of included and excluded tables Filter tables using lists of included and excluded tables. |
transformation. transformers[]. tableSplitterTransformer. tables. includeTables[] |
string List of tables that will be included to transfer |
transformation. transformers[]. tableSplitterTransformer. tables. excludeTables[] |
string List of tables that will be excluded to transfer |
transformation. transformers[]. tableSplitterTransformer. columns[] |
string Specify the columns in the tables to be partitioned. |
transformation. transformers[]. tableSplitterTransformer. splitter |
string Specify the split string to be used for merging components in a new table name. |
transformation. transformers[]. filterRows |
object transformation.transformers[] includes only one of the fields maskField , filterColumns , renameTables , replacePrimaryKey , convertToString , sharderTransformer , tableSplitterTransformer , filterRows This filter only applies to transfers with queues (Logbroker or Apache Kafka®) as a data source. When running a transfer, only the strings meeting the specified criteria remain in a changefeed. |
transformation. transformers[]. filterRows. tables |
object List of included and excluded tables Filter tables using lists of included and excluded tables. |
transformation. transformers[]. filterRows. tables. includeTables[] |
string List of tables that will be included to transfer |
transformation. transformers[]. filterRows. tables. excludeTables[] |
string List of tables that will be excluded to transfer |
transformation. transformers[]. filterRows. filter |
string Filtering criterion. This can be comparison operators for numeric, string, and Boolean values, comparison to NULL, and checking whether a substring is part of a string. Details here: https://cloud.yandex.com/en/docs/data-transfer/concepts/data-transformation#append-only-sources |
prestable | boolean (boolean) |
Methods
Method | Description |
---|---|
activate | |
create | |
deactivate | |
delete | |
get | |
list | |
update |