ECS Managed Scaling keeps Tasks in "Provisioning" state

Hi everyone,

I am working on an example infrastructure for testing the ECS Managed Scaling feature for automatically scaling the underlying EC2 instances. I can't use Fargate because we require a lot of memory and possibly GPUs in our use case.

Our EC2 infrastructure is supposed to be able to scale down to 0 when there are no Tasks running or pending for ECS, which should be possible according to this article as long as the Scaling Target value is set to 100.

I am creating an EC2 Auto Scaling Group with an initial size of 0 that I register as a Capacity Provider in my ECS Cluster. I also create an ECS Service that is using the Capacity Provider to run its Tasks with an initial size of 0 as well. So far so good, everything seems to be configured correctly in AWS.

If I update the Service to have a Desired Tasks size of 1, I see that my Service has a "Pending" Task that is in the "Provisioning" state. This makes sense because the Capacity Provider is scaled to 0 and does not contain any compute infrastructure to run the task. After a while, the AWS managed CloudWatch Alarm gets triggered which causes the Auto Scaling Group to create two EC2 instances, which is exactly what I want.

Now here is the problem: I would expect that ECS notices that there are two EC2 instances present in the Capacity Provider now and that the Task that is in the "Provisioning" state is going to be placed on one of the instances and move to the "Running" state. However, this does not happen and the Task is forever stuck in the "Provisioning" state.

Can you help me, troubleshoot the issue? I am deploying my infrastructure using terraform, so you can also have a look at my terraform configuration below.

Thanks a lot!

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "5.89.0"
    }
  }

  required_version = ">= 1.2.0"
}

provider "aws" {
  region = "eu-central-1"
}

module "vpc" {
  source  = "terraform-aws-modules/vpc/aws"
  version = "5.19.0"

  name = "ECS-VPC"
  cidr = "10.0.0.0/16"

  azs = ["eu-central-1a", "eu-central-1b", "eu-central-1c"]
  private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
  public_subnets = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"]

  enable_nat_gateway = true
}

resource "aws_ecs_cluster" "main" {
  name = "ecs-cluster"
}

resource "aws_security_group" "ecs_sg" {
  name        = "ecs-sg"
  description = "Security group for ECS instances"
  vpc_id = module.vpc.vpc_id

  ingress {
    description = "Allow all traffic from within the SG"
    from_port = 0
    to_port = 0
    protocol = "-1"
    self = true
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "aws_iam_role" "ecs_instance_role" {
  name = "ecs-instance-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Sid    = ""
        Principal = {
          Service = "ec2.amazonaws.com"
        }
      },
    ]
  })
}

resource "aws_iam_role_policy_attachment" "ecs_instance_role_attachment" {
  role       = aws_iam_role.ecs_instance_role.name
  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
}

resource "aws_iam_instance_profile" "ecs_instance_profile" {
  name = "ecs-instance-profile"
  role = aws_iam_role.ecs_instance_role.name
}

data "aws_ami" "ecs_ami" {
  most_recent = true
  owners      = ["amazon"]

  filter {
    name   = "name"
    values = ["al2023-ami-ecs-hvm-*"]
  }
}

resource "aws_launch_configuration" "ecs_lc" {
  name_prefix          = "ecs-lc-"
  image_id             = data.aws_ami.ecs_ami.id
  instance_type        = "t3.micro"
  iam_instance_profile = aws_iam_instance_profile.ecs_instance_profile.name
  security_groups      = [aws_security_group.ecs_sg.id]
}

resource "aws_autoscaling_group" "ecs_asg" {
  name                      = "ecs-asg"
  launch_configuration      = aws_launch_configuration.ecs_lc.name
  min_size                  = 0
  max_size                  = 3
  desired_capacity          = 0
  vpc_zone_identifier       = module.vpc.private_subnets
  health_check_type         = "EC2"
  health_check_grace_period = 300
}

resource "aws_ecs_capacity_provider" "ecs_capacity_provider" {
  name = "EC2-Capacity-Provider"
  auto_scaling_group_provider {
    auto_scaling_group_arn = aws_autoscaling_group.ecs_asg.arn
    managed_scaling {
      maximum_scaling_step_size = 3
      minimum_scaling_step_size = 1
      target_capacity = 100
      status = "ENABLED"
    }
  }
}

resource "aws_ecs_cluster_capacity_providers" "ecs_cluster_capacity_providers" {
  cluster_name = aws_ecs_cluster.main.name
  capacity_providers = [aws_ecs_capacity_provider.ecs_capacity_provider.name]
}

resource "aws_ecs_task_definition" "nginx_task" {
  family                   = "nginx-task"
  requires_compatibilities = ["EC2"]
  network_mode             = "none"
  cpu                      = 256
  memory                   = 128
  container_definitions = jsonencode([
    {
      name      = "nginx-container"
      image     = "nginx:latest"
      essential = true
    }
  ])
}

resource "aws_ecs_service" "nginx_service" {
  name            = "nginx-service"
  cluster         = aws_ecs_cluster.main.id
  task_definition = aws_ecs_task_definition.nginx_task.arn
  desired_count   = 0
  deployment_maximum_percent = 200
  deployment_minimum_healthy_percent = 100
  capacity_provider_strategy {
    capacity_provider = aws_ecs_capacity_provider.ecs_capacity_provider.name
    weight = 100
  }
}