Cannot curl OpenSearch Service Endpoint from VPN Client Endpoint

Question

Hello,
I've problem: I can't curl OpenSearch service endpoint while being connected to VPN Client. Following Terraform code reproduces error. How can I fix timeout on curl to search service?
```hcl
terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
  }
}

locals {
  availability_zones  = ["us-east-1a"]
  vpn_server_cert_arn = "arn:aws:acm:us-east-1::certificate/"
  region              = "us-east-1"
}

variable "environment" {
  type    = string
  default = "test"
}

provider "aws" {
  region = "us-east-1"
}

resource "aws_vpc" "vpc" {
  cidr_block           = "10.0.0.0/16"
  enable_dns_hostnames = true
  enable_dns_support   = true
  tags = {
    Name = "vpc-${var.environment}"
  }
}
resource "aws_s3_bucket" "bucket" {
  bucket = "vpc-vpn-test-flow-logs"
}

resource "aws_flow_log" "flowlog" {
  log_destination          = aws_s3_bucket.bucket.arn
  log_destination_type     = "s3"
  traffic_type             = "ALL"
  vpc_id                   = aws_vpc.vpc.id
  max_aggregation_interval = 60
}

resource "aws_subnet" "public_subnet_1" {
  vpc_id                  = aws_vpc.vpc.id
  cidr_block              = "10.0.1.0/24"
  availability_zone       = local.availability_zones[0]
  map_public_ip_on_launch = true
  tags = {
    Name = "public-subnet-1-${var.environment}"
  }
}

resource "aws_subnet" "private_subnet_1" {
  vpc_id            = aws_vpc.vpc.id
  cidr_block        = "10.0.3.0/24"
  availability_zone = local.availability_zones[0]
  tags = {
    Name = "private-subnet-1-${var.environment}"
  }
}

resource "aws_internet_gateway" "igw" {
  vpc_id = aws_vpc.vpc.id
  tags = {
    Name = "igw-${var.environment}"
  }
}

resource "aws_route_table" "public" {
  vpc_id = aws_vpc.vpc.id

route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.igw.id
  }

tags = {
    Name = "public-route-table-${var.environment}"
  }
}

resource "aws_route_table_association" "public_1" {
  subnet_id      = aws_subnet.public_subnet_1.id
  route_table_id = aws_route_table.public.id
}

resource "aws_ec2_client_vpn_endpoint" "client_vpn" {
  client_cidr_block      = "10.1.0.0/22"
  server_certificate_arn = local.vpn_server_cert_arn
  vpc_id                 = aws_vpc.vpc.id
  security_group_ids     = [aws_security_group.vpn_security_group.id]

authentication_options {
    type                       = "certificate-authentication"
    root_certificate_chain_arn = local.vpn_server_cert_arn
  }

connection_log_options {
    enabled = false
  }

tags = {
    Name = "vpn-${var.environment}"
  }
}

resource "aws_ec2_client_vpn_network_association" "vpn_association_1" {
  client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client_vpn.id
  subnet_id              = aws_subnet.public_subnet_1.id
}

resource "aws_ec2_client_vpn_authorization_rule" "vpn_auth_rule" {
  client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client_vpn.id
  target_network_cidr    = "0.0.0.0/0"
  authorize_all_groups   = true
}

resource "aws_ec2_client_vpn_route" "internet_access_1" {
  client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client_vpn.id
  destination_cidr_block = "0.0.0.0/0"
  target_vpc_subnet_id   = aws_subnet.public_subnet_1.id
  description            = "Route to Internet Gateway for VPN clients"
}

resource "aws_security_group" "vpn_security_group" {
  vpc_id = aws_vpc.vpc.id

ingress {
    description = "Allow incoming VPN traffic"
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["10.1.0.0/22"]
  }

egress {
    description = "Allow all outbound traffic"
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

tags = {
    Name = "vpn-security-group-${var.environment}"
  }
}

resource "aws_opensearch_domain" "opensearch" {
  domain_name = "opensearch-${var.environment}"

cluster_config {
    instance_type          = "t3.small.search"
    instance_count         = 1
    zone_awareness_enabled = false
  }

ebs_options {
    ebs_enabled = true
    volume_size = 10
    volume_type = "gp2"
  }

vpc_options {
    security_group_ids = [aws_security_group.opensearch_sg.id]
    subnet_ids         = [aws_subnet.public_subnet_1.id]
  }

access_policies = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect    = "Allow"
        Principal = "*"
        Action    = "es:*"
        Resource  = "arn:aws:es:${local.region}:${data.aws_caller_identity.current.account_id}:domain/opensearch-${var.environment}/*"
      }
    ]
  })

tags = {
    Name = "opensearch-${var.environment}"
  }
}

resource "aws_security_group" "opensearch_sg" {
  vpc_id = aws_vpc.vpc.id

ingress {
    description = "Allow VPN clients to access OpenSearch"
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    cidr_blocks = [aws_ec2_client_vpn_endpoint.client_vpn.client_cidr_block]
  }

egress {
    description = "Allow all outbound traffic"
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

tags = {
    Name = "opensearch-sg-${var.environment}"
  }
}

data "aws_caller_identity" "current" {}
```

Gary Mclean · Accepted Answer

This will not work because VPN Clients are NAT'd. The security group rule will never see the VPN Clients CIDR Block

```
resource "aws_security_group" "opensearch_sg" {
  vpc_id = aws_vpc.vpc.id

ingress {
    description = "Allow VPN clients to access OpenSearch"
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    cidr_blocks = [aws_ec2_client_vpn_endpoint.client_vpn.client_cidr_block]
  }

egress {
    description = "Allow all outbound traffic"
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

tags = {
    Name = "opensearch-sg-${var.environment}"
  }
}
```

Try this

```
resource "aws_security_group" "opensearch_sg" {
  vpc_id = aws_vpc.vpc.id

ingress {
    description = "Allow VPN clients to access OpenSearch"
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    security_groups = [aws_security_group.vpn_security_group.id ]
  }

egress {
    description = "Allow all outbound traffic"
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

tags = {
    Name = "opensearch-sg-${var.environment}"
  }
}
```

Gary Mclean · Answer

I see several issues with your setup here and would be helpful if you can share your error. I am going to assume you cant resolve the DNS name for the VPC Endpoint

1. No NAT Gateway
2. Split TUNNEL = false
3. No DNS Resolution for VPN Clients

Currently you have split tunnel=false (By default) so ALL VPN traffic is being sent to your VPC. Your VPN Clients cant access the internet without a NAT gateway. They are also unable to resolve DNS Names because you dont have a DNS Server Set.

If you set your dns_servers = "10.0.0.2" VPC IP +2 this may fix your issue and you can resolve and connect to the private VPC Endpoint. But you have other configuration designs decisions to make.

```
resource "aws_ec2_client_vpn_endpoint" "client_vpn" {
  client_cidr_block      = "10.1.0.0/22"
  dns_servers            = [cidrhost(aws_vpc.vpc.cidr_block, 2)]
  server_certificate_arn = local.vpn_server_cert_arn
  vpc_id                 = aws_vpc.vpc.id
  security_group_ids     = [aws_security_group.vpn_security_group.id]

authentication_options {
    type                       = "certificate-authentication"
    root_certificate_chain_arn = local.vpn_server_cert_arn
  }

connection_log_options {
    enabled = false
  }

tags = {
    Name = "vpn-${var.environment}"
  }
}
```

re:Post Agent · Answer

The issue you're experiencing with not being able to curl the OpenSearch service endpoint while connected to the VPN client can be resolved by making a few adjustments to your Terraform configuration. Here are the steps to fix the timeout:

1. Ensure proper routing: Add a route in the VPC route table associated with the subnet where OpenSearch is deployed. This route should direct traffic from the VPN client CIDR to the OpenSearch subnet.

2. Adjust security group rules: Modify the OpenSearch security group to allow incoming traffic from the VPN client CIDR block on the HTTPS port (443).

3. Configure OpenSearch network settings: Ensure that OpenSearch is set up to use the VPC endpoint for access.

4. VPN client configuration: Make sure your VPN client is configured to route traffic to the OpenSearch domain through the VPN connection.

Here are the specific changes to make in your Terraform code:

1. Add a route to the public route table:

```hcl
resource "aws_route" "vpn_to_opensearch" {
  route_table_id         = aws_route_table.public.id
  destination_cidr_block = aws_ec2_client_vpn_endpoint.client_vpn.client_cidr_block
  gateway_id             = aws_internet_gateway.igw.id
}
```

2. Modify the OpenSearch security group:

```hcl
resource "aws_security_group" "opensearch_sg" {
  # ... (existing configuration)

ingress {
    description = "Allow VPN clients to access OpenSearch"
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    cidr_blocks = [aws_ec2_client_vpn_endpoint.client_vpn.client_cidr_block]
  }

# ... (rest of the configuration)
}
```

3. Ensure OpenSearch is using VPC access:

```hcl
resource "aws_opensearch_domain" "opensearch" {
  # ... (existing configuration)

vpc_options {
    subnet_ids         = [aws_subnet.public_subnet_1.id]
    security_group_ids = [aws_security_group.opensearch_sg.id]
  }

# ... (rest of the configuration)
}
```

After making these changes, apply the updated Terraform configuration. Once the changes are applied, you should be able to curl the OpenSearch service endpoint while connected to the VPN client. If you still experience issues, double-check your VPN client configuration and ensure that it's routing traffic to the OpenSearch domain correctly.  
**Sources**  
[Access Amazon OpenSearch Service using an OpenSearch Service-managed VPC endpoint (AWS PrivateLink) - Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/vpc-interface-endpoints.html)  
[Community | Goodbye Elasticsearch, Hello OpenSearch: A Golang Developer's Journey with Amazon Q (Lessons Learned)](https://community.aws/content/2fBVnCWwN5TEa1wEWl5Wah7Vb2A/migrating-golang-project-using-elasticsearch-to-opensearch-with-amazonq)

Cannot curl OpenSearch Service Endpoint from VPN Client Endpoint

Add your answer

Relevant content