Hi,
I'm having a high latency during the CDC on the source database. I would like to now if there is a way to tuning the configuration in order to decrease the latency or if there is anything else that I can do.
Migration Task CDC metrics:

Replication Instance metrics:

My current configuration:
Source endpoint configuration: Oracle
{
"AddSupplementalLogging": true,
"ExtraArchivedLogDestIds": [],
"AllowSelectNestedTables": true,
"DatabaseName": "XXXXX",
"UseBFile": true,
"UseLogminerReader": false,
"SecretsManagerAccessRoleArn": "XXXXXX",
"SecretsManagerSecretId": "XXXXXXX"
}
Target endpoint configuration: Redshift
{
"DatabaseName": "XXXXXX",
"DateFormat": "auto",
"MaxFileSize": 250000,
"TimeFormat": "auto",
"SecretsManagerAccessRoleArn": "XXXXXXX",
"SecretsManagerSecretId": "XXXXXXX"
}
Replication Instance:
- Instance class: dms.c5.9xlarge
- Allocated storage: 1.2TB
- Multi AZ: yes
Migration Task:
{
"Logging": {
"EnableLogging": true,
"EnableLogContext": false,
"LogComponents": [
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "TRANSFORMATION"
},
{
"Severity": "LOGGER_SEVERITY_INFO",
"Id": "SOURCE_UNLOAD"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "IO"
},
{
"Severity": "LOGGER_SEVERITY_INFO",
"Id": "TARGET_LOAD"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "PERFORMANCE"
},
{
"Severity": "LOGGER_SEVERITY_INFO",
"Id": "SOURCE_CAPTURE"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "SORTER"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "REST_SERVER"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "VALIDATOR_EXT"
},
{
"Severity": "LOGGER_SEVERITY_INFO",
"Id": "TARGET_APPLY"
},
{
"Severity": "LOGGER_SEVERITY_INFO",
"Id": "TASK_MANAGER"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "TABLES_MANAGER"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "METADATA_MANAGER"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "FILE_FACTORY"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "COMMON"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "ADDONS"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "DATA_STRUCTURE"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "COMMUNICATION"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "FILE_TRANSFER"
}
],
"CloudWatchLogGroup": "XXXXXX",
"CloudWatchLogStream": "XXXXXX"
},
"StreamBufferSettings": {
"StreamBufferCount": 3,
"CtrlStreamBufferSizeInMB": 5,
"StreamBufferSizeInMB": 8
},
"ErrorBehavior": {
"FailOnNoTablesCaptured": true,
"ApplyErrorUpdatePolicy": "LOG_ERROR",
"FailOnTransactionConsistencyBreached": false,
"RecoverableErrorThrottlingMax": 1800,
"DataErrorEscalationPolicy": "SUSPEND_TABLE",
"ApplyErrorEscalationCount": 0,
"RecoverableErrorStopRetryAfterThrottlingMax": true,
"RecoverableErrorThrottling": true,
"ApplyErrorFailOnTruncationDdl": false,
"DataTruncationErrorPolicy": "LOG_ERROR",
"ApplyErrorInsertPolicy": "LOG_ERROR",
"EventErrorPolicy": "IGNORE",
"ApplyErrorEscalationPolicy": "LOG_ERROR",
"RecoverableErrorCount": -1,
"DataErrorEscalationCount": 0,
"TableErrorEscalationPolicy": "STOP_TASK",
"RecoverableErrorInterval": 5,
"ApplyErrorDeletePolicy": "IGNORE_RECORD",
"TableErrorEscalationCount": 0,
"FullLoadIgnoreConflicts": true,
"DataErrorPolicy": "LOG_ERROR",
"TableErrorPolicy": "SUSPEND_TABLE"
},
"TTSettings": {
"TTS3Settings": null,
"TTRecordSettings": null,
"EnableTT": false
},
"FullLoadSettings": {
"CommitRate": 50000,
"StopTaskCachedChangesApplied": false,
"StopTaskCachedChangesNotApplied": false,
"MaxFullLoadSubTasks": 49,
"TransactionConsistencyTimeout": 600,
"CreatePkAfterFullLoad": true,
"TargetTablePrepMode": "TRUNCATE_BEFORE_LOAD"
},
"TargetMetadata": {
"ParallelApplyBufferSize": 1000,
"ParallelApplyQueuesPerThread": 0,
"ParallelApplyThreads": 8,
"TargetSchema": "",
"InlineLobMaxSize": 0,
"ParallelLoadQueuesPerThread": 0,
"SupportLobs": true,
"LobChunkSize": 0,
"TaskRecoveryTableEnabled": false,
"ParallelLoadThreads": 32,
"LobMaxSize": 32,
"BatchApplyEnabled": true,
"FullLobMode": false,
"LimitedSizeLobMode": true,
"LoadMaxFileSize": 0,
"ParallelLoadBufferSize": 1000
},
"BeforeImageSettings": null,
"ControlTablesSettings": {
"historyTimeslotInMinutes": 5,
"HistoryTimeslotInMinutes": 5,
"StatusTableEnabled": false,
"SuspendedTablesTableEnabled": false,
"HistoryTableEnabled": false,
"ControlSchema": "",
"FullLoadExceptionTableEnabled": false
},
"LoopbackPreventionSettings": null,
"CharacterSetSettings": null,
"FailTaskWhenCleanTaskResourceFailed": false,
"ChangeProcessingTuning": {
"StatementCacheSize": 50,
"CommitTimeout": 1,
"BatchApplyPreserveTransaction": true,
"BatchApplyTimeoutMin": 60,
"BatchSplitSize": 0,
"BatchApplyTimeoutMax": 60,
"MinTransactionSize": 1000,
"MemoryKeepTime": 60,
"BatchApplyMemoryLimit": 500,
"MemoryLimitTotal": 1024
},
"ChangeProcessingDdlHandlingPolicy": {
"HandleSourceTableDropped": true,
"HandleSourceTableTruncated": true,
"HandleSourceTableAltered": true
},
"PostProcessingRules": null
}
Thanks for you response Keerthi, but most of the configuration is related to the full_load, I'm having problems with the ongoing changes (CDC). I tried the ASM endpoint configuration, but the AWS RDS doesn't support the ASM Feature for Oracle. https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Oracle.Concepts.FeatureSupport.html#Oracle.Concepts.FeatureSupport.unsupported