DMS MySQL errors when replicating to OpenSearch
Hello,
I am receiving a "no handler found for uri" error when attempting to replicate data from a MySQL RDS instance using DMS 3.4.7 to OpenSearch 2.5.
2023-03-29T18:35:07 [TARGET_LOAD ]E: Elasticsearch:FAILED SourceTable:accounts TargetIndex:accounts Operation:INSERT_ENTRY RecordPKKey:1010 RecordPKID:7A5DF5FFA0DEC2228D90B8D0A0F1B0767B748B0A41314C123075B8289E4E053FES HttpCode:400 ESErrorResponse:
{ "error": "no handler found for uri [/accounts/doc/7A5DF5FFA0DEC2228D90B8D0A0F1B0767B748B0A41314C123075B8289E4E053F] and method [PUT]" }
[1026400] (elasticsearch_utils.c:657)
I ssh'd into the OpenSearch cluster and the index does exist, so it is creating the index, but no records are being written. What is strange to me is that based on this error, it looks like DMS is attempting to write the record to /accounts/doc/id
when in the official OpenSearch documentation the operation should be using /accounts/_doc/id
as noted here:
PUT sample-index/_doc/1
{
"Description": "To be or not to be, that is the question."
}
https://opensearch.org/docs/2.5/api-reference/document-apis/index-document/
When I attempt to insert a record with the underscore (PUT accounts/_doc/1) it works. Am I missing something here? Here is my task config:
{
"Logging": {
"EnableLogging": true,
"EnableLogContext": false,
"LogComponents": [
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "TRANSFORMATION"
},
{
"Severity": "LOGGER_SEVERITY_DETAILED_DEBUG",
"Id": "SOURCE_UNLOAD"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "IO"
},
{
"Severity": "LOGGER_SEVERITY_DETAILED_DEBUG",
"Id": "TARGET_LOAD"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "PERFORMANCE"
},
{
"Severity": "LOGGER_SEVERITY_DETAILED_DEBUG",
"Id": "SOURCE_CAPTURE"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "SORTER"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "REST_SERVER"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "VALIDATOR_EXT"
},
{
"Severity": "LOGGER_SEVERITY_DETAILED_DEBUG",
"Id": "TARGET_APPLY"
},
{
"Severity": "LOGGER_SEVERITY_DETAILED_DEBUG",
"Id": "TASK_MANAGER"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "TABLES_MANAGER"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "METADATA_MANAGER"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "FILE_FACTORY"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "COMMON"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "ADDONS"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "DATA_STRUCTURE"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "COMMUNICATION"
},
{
"Severity": "LOGGER_SEVERITY_DEFAULT",
"Id": "FILE_TRANSFER"
}
],
"CloudWatchLogGroup": "hidden",
"CloudWatchLogStream": "hidden"
},
"StreamBufferSettings": {
"StreamBufferCount": 3,
"CtrlStreamBufferSizeInMB": 5,
"StreamBufferSizeInMB": 8
},
"ErrorBehavior": {
"FailOnNoTablesCaptured": false,
"ApplyErrorUpdatePolicy": "LOG_ERROR",
"FailOnTransactionConsistencyBreached": false,
"RecoverableErrorThrottlingMax": 1800,
"DataErrorEscalationPolicy": "SUSPEND_TABLE",
"ApplyErrorEscalationCount": 0,
"RecoverableErrorStopRetryAfterThrottlingMax": false,
"RecoverableErrorThrottling": true,
"ApplyErrorFailOnTruncationDdl": false,
"DataTruncationErrorPolicy": "LOG_ERROR",
"ApplyErrorInsertPolicy": "LOG_ERROR",
"EventErrorPolicy": "IGNORE",
"ApplyErrorEscalationPolicy": "LOG_ERROR",
"RecoverableErrorCount": -1,
"DataErrorEscalationCount": 0,
"TableErrorEscalationPolicy": "STOP_TASK",
"RecoverableErrorInterval": 5,
"ApplyErrorDeletePolicy": "IGNORE_RECORD",
"TableErrorEscalationCount": 0,
"FullLoadIgnoreConflicts": true,
"DataErrorPolicy": "LOG_ERROR",
"TableErrorPolicy": "SUSPEND_TABLE"
},
"TTSettings": {
"TTS3Settings": null,
"TTRecordSettings": null,
"EnableTT": false
},
"FullLoadSettings": {
"CommitRate": 10000,
"StopTaskCachedChangesApplied": false,
"StopTaskCachedChangesNotApplied": false,
"MaxFullLoadSubTasks": 8,
"TransactionConsistencyTimeout": 600,
"CreatePkAfterFullLoad": false,
"TargetTablePrepMode": "DROP_AND_CREATE"
},
"TargetMetadata": {
"ParallelApplyBufferSize": 0,
"ParallelApplyQueuesPerThread": 0,
"ParallelApplyThreads": 0,
"TargetSchema": "",
"InlineLobMaxSize": 0,
"ParallelLoadQueuesPerThread": 0,
"SupportLobs": true,
"LobChunkSize": 64,
"TaskRecoveryTableEnabled": false,
"ParallelLoadThreads": 0,
"LobMaxSize": 0,
"BatchApplyEnabled": false,
"FullLobMode": true,
"LimitedSizeLobMode": false,
"LoadMaxFileSize": 0,
"ParallelLoadBufferSize": 0
},
"BeforeImageSettings": null,
"ControlTablesSettings": {
"historyTimeslotInMinutes": 5,
"HistoryTimeslotInMinutes": 5,
"StatusTableEnabled": false,
"SuspendedTablesTableEnabled": false,
"HistoryTableEnabled": false,
"ControlSchema": "",
"FullLoadExceptionTableEnabled": false
},
"LoopbackPreventionSettings": null,
"CharacterSetSettings": null,
"FailTaskWhenCleanTaskResourceFailed": false,
"ChangeProcessingTuning": {
"StatementCacheSize": 50,
"CommitTimeout": 1,
"BatchApplyPreserveTransaction": true,
"BatchApplyTimeoutMin": 1,
"BatchSplitSize": 0,
"BatchApplyTimeoutMax": 30,
"MinTransactionSize": 1000,
"MemoryKeepTime": 60,
"BatchApplyMemoryLimit": 500,
"MemoryLimitTotal": 1024
},
"ChangeProcessingDdlHandlingPolicy": {
"HandleSourceTableDropped": true,
"HandleSourceTableTruncated": true,
"HandleSourceTableAltered": true
},
"PostProcessingRules": null
}
- 最新
- 投票最多
- 评论最多
Incase this comes up for anyone the future - the solution is to set UseNewMappingType
to true
under the Elasticsearch settings in the endpoint. It is defaulted to false.
Deep in the depths of the aws docs: https://docs.aws.amazon.com/dms/latest/APIReference/API_CreateEndpoint.html https://docs.aws.amazon.com/dms/latest/APIReference/API_ElasticsearchSettings.html
相关内容
- 已提问 1 年前lg...
- AWS 官方已更新 3 年前
- AWS 官方已更新 2 年前
- AWS 官方已更新 3 年前