By using AWS re:Post, you agree to the Terms of Use
/Multiple tables in one Data Pipeline?/

Multiple tables in one Data Pipeline?

0

I am using Data Pipelines to replicate data from RDS to Redshift with one pipeline per table they are replicating.

Is it possible to consolidate the pipelines into a single pipeline for multiple tables? And if so, is there an example JSON config that would make this work?

1 Answers
0
Accepted Answer

Yes, this is possible. Below I've included a demo JSON code for that pipeline.

Note that the JSON isn't complete as there are more attributes required for MySql and Redshift - I've included links to the full attribute requirements as well.

MySqlDataNode Requirements

RedshiftDataNode Requirements

{
  "objects": [
    {
      "output": {
        "ref": "RedshiftDataNodeId_BWynX"
      },
      "input": {
        "ref": "MySqlDataNodeId_iOnTY"
      },
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "name": "DefaultCopyActivity2",
      "id": "CopyActivityId_WMYod",
      "type": "CopyActivity"
    },
    {
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "name": "DefaultMySqlDataNode3",
      "id": "MySqlDataNodeId_9uoCY",
      "type": "MySqlDataNode",
      "table": ""
    },
    {
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "name": "DefaultRedshiftDataNode1",
      "id": "RedshiftDataNodeId_mJUEj",
      "type": "RedshiftDataNode",
      "tableName": ""
    },
    {
      "output": {
        "ref": "RedshiftDataNodeId_yz06G"
      },
      "input": {
        "ref": "MySqlDataNodeId_9uoCY"
      },
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "name": "DefaultCopyActivity3",
      "id": "CopyActivityId_nm5LG",
      "type": "CopyActivity"
    },
    {
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "name": "DefaultMySqlDataNode1",
      "id": "MySqlDataNodeId_6Xe5h",
      "type": "MySqlDataNode",
      "table": ""
    },
    {
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "name": "DefaultRedshiftDataNode3",
      "id": "RedshiftDataNodeId_yz06G",
      "type": "RedshiftDataNode",
      "tableName": ""
    },
    {
      "failureAndRerunMode": "CASCADE",
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "resourceRole": "DataPipelineDefaultResourceRole",
      "role": "DataPipelineDefaultRole",
      "pipelineLogUri": "s3://athena-ab3/",
      "scheduleType": "cron",
      "name": "Default",
      "id": "Default"
    },
    {
      "occurrences": "1",
      "period": "1 days",
      "name": "RunOnce",
      "id": "DefaultSchedule",
      "type": "Schedule",
      "startAt": "FIRST_ACTIVATION_DATE_TIME"
    },
    {
      "output": {
        "ref": "RedshiftDataNodeId_mJUEj"
      },
      "input": {
        "ref": "MySqlDataNodeId_6Xe5h"
      },
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "name": "DefaultCopyActivity1",
      "id": "CopyActivityId_G0Lps",
      "type": "CopyActivity"
    },
    {
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "name": "DefaultRedshiftDataNode2",
      "id": "RedshiftDataNodeId_BWynX",
      "type": "RedshiftDataNode",
      "tableName": ""
    },
    {
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "name": "DefaultMySqlDataNode2",
      "id": "MySqlDataNodeId_iOnTY",
      "type": "MySqlDataNode",
      "table": ""
    }
  ],
  "parameters": []
}
answered a year ago

You are not logged in. Log in to post an answer.

A good answer clearly answers the question and provides constructive feedback and encourages professional growth in the question asker.

Guidelines for Answering Questions