EC2 Instances Killed by HEX Verdandi

Hi, we have seen large amount of EC2 instaces got terminated by HEX Verdandi. And sometime batch job got killed after 3 instances was terminated in a row by Verdandi. We still don't know why. Can anyone help? Thank you!

In the past week (2025-01-30 to 2025-02-05), we saw 29 EC2 instances were terminated by Verdandi in prod and 244 EC2 instances were terminated by Verdandi in gamma.

Example CloudTrail message:

"eventVersion": "1.08",
"userIdentity": {
    "type": "AssumedRole",
    "principalId": "AROA6P7IXVPJ7GOYJPOUO:0c86c978f4464a679c715b5a4478b5e6",
    "arn": "arn:aws:sts::996383632339:assumed-role/OperationsManagementStack-VerdandiRole-1BX8MOXUWKOEF/0c86c978f4464a679c715b5a4478b5e6",
    "accountId": "996383632339",
    "accessKeyId": "...",
    "sessionContext": {
        "sessionIssuer": {
            "type": "Role",
            "principalId": "AROA6P7IXVPJ7GOYJPOUO",
            "arn": "arn:aws:iam::996383632339:role/OperationsManagementStack-VerdandiRole-1BX8MOXUWKOEF",
            "accountId": "996383632339",
            "userName": "OperationsManagementStack-VerdandiRole-1BX8MOXUWKOEF"
        },
        "webIdFederationData": {},
        "attributes": {
            "creationDate": "2025-02-05T02:23:47Z",
            "mfaAuthenticated": "false"
        }
    }
},
"eventTime": "2025-02-05T02:32:43Z",
"eventSource": "autoscaling.amazonaws.com",
"eventName": "TerminateInstanceInAutoScalingGroup",
"awsRegion": "us-east-1",
"sourceIPAddress": "52.201.122.38",
"userAgent": "Boto3/1.33.13 md/Botocore#1.33.13 ua/2.0 os/linux#4.14.355-275.572.amzn2.x86_64 md/arch#x86_64 lang/python#3.7.16 md/pyimpl#CPython exec-env/AWS_ECS_EC2 cfg/retry-mode#legacy Botocore/1.33.13",
"requestParameters": {
    "instanceId": "i-010d80ae9102457f8",
    "shouldDecrementDesiredCapacity": false
},
"responseElements": {
    "activity": {
        "activityId": "81d6534f-cf8e-a52b-370f-a1116bacf5e1",
        "autoScalingGroupName": "EcsClusterStack-CPAutoScalingGroup-Np169TuQjLaP",
        "description": "Terminating EC2 instance: i-010d80ae9102457f8",
        "cause": "At 2025-02-05T02:32:43Z instance i-010d80ae9102457f8 was taken out of service in response to a user request.",
        "startTime": "Feb 5, 2025, 2:32:43 AM",
        "statusCode": "InProgress",
        "progress": 0,
        "details": "{\"Subnet ID\":\"subnet-08490d2707bbefaaa\",\"Availability Zone\":\"us-east-1c\"}"
    }
},
"requestID": "550f8e28-06c6-4993-a9ee-19c991b02885",
"eventID": "22f3ed98-f24b-4939-b035-4ff154cb82dd",
"readOnly": false,
"eventType": "AwsApiCall",
"managementEvent": true,
"recipientAccountId": "996383632339",
"eventCategory": "Management",
"tlsDetails": {
    "tlsVersion": "TLSv1.2",
    "cipherSuite": "ECDHE-RSA-AES128-GCM-SHA256",
    "clientProvidedHostHeader": "autoscaling.us-east-1.amazonaws.com"
}

}