Skip to content

ECS task cannot start, unable to retrieve ecr registry auth

0

With terraform I've created an ECS setup. When I do an apply I get the following error on the Tasks UI: ResourceInitializationError: unable to pull secrets or registry auth: execution resource retrieval failed: unable to retrieve ecr registry auth: service call has been retried 3 time(s): RequestError: send request failed caused by: Post "https://api.ecr.eu-central-1.amazonaws.com/": dial tcp 3.122.129.122:443: i/o timeout. Please check your task network configuration.

I have private subnets, but from other resources I found about this issue indicates that with an NAT and an Internet gateway there should be no issue.

Here is my Terraform config. (I know the execution and task roles are overkill but I was looking for the solution, and this is not it)

provider "aws" {
  region = "eu-central-1"
}

resource "aws_ecr_repository" "tay_tay_ecr_repository_client" {
  name = "tay-tay-ecr-repository-client"
}

resource "aws_ecs_cluster" "tay_tay_ecs_cluster" {
  name = "tay-tay-ecs-cluster"
}

# --- START Client task definition ---
resource "aws_ecs_task_definition" "tay_tay_ecs_task_client" {
  family                   = "tay-tay-ecs-container-client"
  container_definitions    = <<DEFINITION
  [
    {
      "name": "${var.client_container_name}",
      "image": "${aws_ecr_repository.tay_tay_ecr_repository_client.repository_url}",
      "essential": true,
      "portMappings": [
        {
          "containerPort": 5173,
          "hostPort": 5173
        }
      ],
      "memory": 512,
      "cpu": 256
    }
  ]
  DEFINITION
  requires_compatibilities = ["FARGATE"]
  network_mode             = "awsvpc"
  memory                   = 512
  cpu                      = 256
  task_role_arn = aws_iam_role.ecs_task_role.arn
  execution_role_arn = aws_iam_role.ecs_execution_role.arn
}

resource "aws_iam_role" "ecs_task_role" {
  name = "ecs_task_role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17",
    Statement = [
      {
        Action = "sts:AssumeRole",
        Effect = "Allow",
        Principal = {
          Service = "ecs-tasks.amazonaws.com"
        }
      }
    ]
  })
}

resource "aws_iam_role" "ecs_execution_role" {
  name = "ecs_execution_role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17",
    Statement = [
      {
        Action = "sts:AssumeRole",
        Effect = "Allow",
        Principal = {
          Service = "ecs-tasks.amazonaws.com"
        }
      }
    ]
  })
}

resource "aws_iam_policy" "ecr_policy" {
  name        = "ecr_policy"
  description = "Policy to access ECR registry"

  policy = jsonencode({
    Version = "2012-10-17",
    Statement = [
      {
        Action = [
          "ecr:*",
          "logs:*",
          "ssm:*",
          "ssmmessages:*",
          "secretsmanager:*",
          "kms:*"
        ],
        Effect   = "Allow",
        Resource = "*"
      }
    ]
  })
}

resource "aws_iam_policy_attachment" "ecs_task_role_attachment" {
  name       = "ecs_task_role_attachment"
  roles      = [aws_iam_role.ecs_task_role.name]
  policy_arn = aws_iam_policy.ecr_policy.arn
}

resource "aws_iam_policy_attachment" "ecs_execution_role_attachment" {
  name       = "ecs_execution_role_attachment"
  roles      = [aws_iam_role.ecs_execution_role.name]
  policy_arn = aws_iam_policy.ecr_policy.arn
}

# --- END Client task definition ---

# --- START Client service ---
resource "aws_ecs_service" "tay_tay_ecs_service_client" {
  name            = "tay-tay-ecs-service-client"
  cluster         = aws_ecs_cluster.tay_tay_ecs_cluster.id
  task_definition = aws_ecs_task_definition.tay_tay_ecs_task_client.arn
  launch_type     = "FARGATE"
  desired_count   = 1

  load_balancer {
    target_group_arn = aws_lb_target_group.tay_tay_client_alb_target_group.arn
    container_name   = var.client_container_name
    container_port   = 5173
  }

  network_configuration {
    subnets          = [aws_subnet.tay_tay_subnet_a.id, aws_subnet.tay_tay_subnet_b.id, aws_subnet.tay_tay_subnet_c.id]
    assign_public_ip = true
  }
}
# --- END Client service ---

resource "aws_vpc" "tay_tay_vpc" {
  cidr_block = "10.0.0.0/16"
  enable_dns_support = true
  enable_dns_hostnames = true
}

resource "aws_internet_gateway" "tay_tay_internet_gateway" {
  vpc_id = aws_vpc.tay_tay_vpc.id
}


resource "aws_route_table" "tay_tay_route_table" {
  vpc_id = aws_vpc.tay_tay_vpc.id
}

resource "aws_route" "tay_tay_subnet_a_route_to_internet" {
  route_table_id         = aws_route_table.tay_tay_route_table.id
  destination_cidr_block = "0.0.0.0/0"
  gateway_id             = aws_internet_gateway.tay_tay_internet_gateway.id
}

resource "aws_subnet" "tay_tay_subnet_a" {
  availability_zone = "eu-central-1a"
  vpc_id            = aws_vpc.tay_tay_vpc.id
  cidr_block        = "10.0.0.0/18"
  map_public_ip_on_launch = false
}

resource "aws_subnet" "tay_tay_subnet_b" {
  availability_zone = "eu-central-1b"
  vpc_id            = aws_vpc.tay_tay_vpc.id
  cidr_block        = "10.0.64.0/18"
  map_public_ip_on_launch = false
}

resource "aws_subnet" "tay_tay_subnet_c" {
  availability_zone = "eu-central-1c"
  vpc_id            = aws_vpc.tay_tay_vpc.id
  cidr_block        = "10.0.128.0/18"
  map_public_ip_on_launch = false
}

# Create a security group for the NAT Gateway
resource "aws_security_group" "tay_tay_nat_security_group" {
  name_prefix = "tay-tay-nat-security-group"
}

resource "aws_security_group_rule" "tay_tay_nat_security_group_allow_outbound" {
  type        = "egress"
  from_port   = 0
  to_port     = 0
  protocol    = "-1"
  security_group_id = aws_security_group.tay_tay_nat_security_group.id
  cidr_blocks = ["0.0.0.0/0"]
}

resource "aws_eip" "tay_tay_nat_eip" {
  count = 3
}

# Create a NAT Gateway in each public subnet
resource "aws_nat_gateway" "tay_tay_nat" {
  count = 3
  allocation_id = aws_eip.tay_tay_nat_eip[count.index].id
  subnet_id    = element([aws_subnet.tay_tay_subnet_a.id, aws_subnet.tay_tay_subnet_b.id, aws_subnet.tay_tay_subnet_c.id], count.index)
}

# --- START ALB ---
resource "aws_alb" "tay_tay_alb" {
  name               = "tay-tay-alb"
  load_balancer_type = "application"
  subnets = [
    aws_subnet.tay_tay_subnet_a.id,
    aws_subnet.tay_tay_subnet_b.id,
    aws_subnet.tay_tay_subnet_c.id
  ]
  security_groups = [aws_security_group.tay_tay_alb_security_group.id]
}

resource "aws_security_group" "tay_tay_alb_security_group" {
  vpc_id = aws_vpc.tay_tay_vpc.id

  ingress {
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "aws_lb_target_group" "tay_tay_client_alb_target_group" {
  name        = "tay-tay-client-alb-target-group"
  port        = 80
  protocol    = "HTTP"
  target_type = "ip"
  vpc_id      = aws_vpc.tay_tay_vpc.id
  health_check {
    matcher = "200,301,302"
    path = "/"
  }
}

resource "aws_lb_listener" "tay_tay_client_alb_listener" {
  load_balancer_arn = aws_alb.tay_tay_alb.arn
  port              = "80"
  protocol          = "HTTP"
  default_action {
    type             = "forward"
    target_group_arn = aws_lb_target_group.tay_tay_client_alb_target_group.arn
  }
}
# --- END ALB ---
  • For anyone in the future running into this issue, I've resolved it by doing the following.

    Changed aws_ecs_task_definition - execution_role_arn to

    resource "aws_iam_role" "ecsTaskExecutionRole" {
      name               = "ecsTaskExecutionRole"
      assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json
    }
    
    data "aws_iam_policy_document" "assume_role_policy" {
      statement {
        actions = ["sts:AssumeRole"]
    
        principals {
          type        = "Service"
          identifiers = ["ecs-tasks.amazonaws.com"]
        }
      }
    }
    
    resource "aws_iam_role_policy_attachment" "ecsTaskExecutionRole_policy" {
      role       = aws_iam_role.ecsTaskExecutionRole.name
      policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
    }
    

    Downgraded Fargate platform version to 1.3.0 in aws_ecs_service with

    platform_version = "1.3.0"
    

    Added this security group to the aws_ecs_service

    resource "aws_security_group" "tay_tay_client_service_security_group" {
      vpc_id = aws_vpc.tay_tay_vpc.id
      ingress {
        from_port = 0
        to_port   = 0
        protocol  = "-1"
        security_groups = [aws_security_group.tay_tay_alb_security_group.id]
      }
    
      egress {
        from_port   = 0
        to_port     = 0
        protocol    = "-1"
        cidr_blocks = ["0.0.0.0/0"]
      }
    }
    

    Changed the aws_route to point to the vpc's default route table. Added manual aws_route_table_association for every subnet to the vpc's default route table.

1 Answer
0

Please use this terraform script which worked for me

resource "aws_ecs_cluster" "cluster" { name = "my-cluster" }

resource "aws_ecs_task_definition" "task" { family = "my-task" network_mode = "awsvpc" requires_compatibilities = ["FARGATE"] cpu = "256" memory = "512" execution_role_arn = aws_iam_role.ecs_execution_role.arn

container_definitions = jsonencode([{ name = "my-container" image = "your-account-id.dkr.ecr.your-region.amazonaws.com/my-repository:tag" cpu = 256 memory = 512 essential = true portMappings = [{ containerPort = 80 hostPort = 80 }] }]) }

resource "aws_iam_role" "ecs_execution_role" { name = "ecs_execution_role"

assume_role_policy = jsonencode({ Version = "2012-10-17", Statement = [{ Action = "sts:AssumeRole", Effect = "Allow", Principal = { Service = "ecs-tasks.amazonaws.com" }, }] }) }

resource "aws_iam_role_policy_attachment" "ecs_execution_role_policy" { role = aws_iam_role.ecs_execution_role.name policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" }

resource "aws_ecs_service" "service" { name = "my-service" cluster = aws_ecs_cluster.cluster.id task_definition = aws_ecs_task_definition.task.arn launch_type = "FARGATE"

network_configuration { subnets = ["subnet-xxxxxxxxxxxxxx", "subnet-yyyyyyyyyyyyyy"] security_groups = ["sg-xxxxxxxxxxxxxx"] }

desired_count = 1 }

In the network_configuration, specify the IDs of your VPC's subnets and the security group for your tasks.

answered 3 years ago

EXPERT

reviewed 2 years ago

You are not logged in. Log in to post an answer.

A good answer clearly answers the question and provides constructive feedback and encourages professional growth in the question asker.