Having API Gateway sign a proxied request destined for OpenSearch in CloudFormation

Hi there,

I have a CloudFormation setup that I hope to use to automatically index documents in OpenSearch. I've set up an ingestion bucket in S3; when a file's dropped in the bucket, it triggers a Lambda function, which slightly transforms the data before sending it to OpenSearch. I was having trouble applying a sigv4 signature to the indexing request when it's sent to OpenSearch (kept getting a mismatch error), so I abandoned that in favour of proxying everything through API Gateway. While the signature error is now gone, I keep getting a "User: anonymous is not authorized to perform: es:ESHttpPut because no resource-based policy allows the es:ESHttpPut action" error.

My OpenSearch policy explicitly allows Put (and the other verbs, since I also want to be able to query OpenSearch through the proxy) for API Gateway, though:

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "AllowAPIGatewayAccess",
      "Effect": "Allow",
      "Principal": {
        "AWS": "arn:aws:iam::ACCOUNT_ID:role/ingest-app-APIGatewayRole"
      },
      "Action": [
        "es:ESHttpGet",
        "es:ESHttpPost",
        "es:ESHttpPut",
        "es:ESHttpDelete"
      ],
      "Resource": "arn:aws:es:REGION:ACCOUNT_ID:domain/ingest-app-domain/*"
    }
  ]
}

Here's my (rather long, sorry) CloudFormation. OpenSearch is administered manually. I can't for the life of me figure this out. Any advice would be much appreciated.

AWSTemplateFormatVersion: '2010-09-09'
Description: >
  Full stack: S3 to Lambda to API Gateway to OpenSearch

Parameters:
  ProjectName:
    Type: String
    Default: "ingest-app"
  GitHubRepo:
    Type: String
    Description: "Format: owner/repo"
  GitHubBranch:
    Type: String
    Default: "main"
  GitHubToken:
    Type: String
    NoEcho: true
  OpenSearchDomainEndpoint:
    Type: String
    Description: "OpenSearch domain endpoint (https://...)"
  OpenSearchDomainName:
    Type: String
    Description: "OpenSearch domain name"

Resources:

  ## IAM Roles
  IngestLambdaRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub "${ProjectName}-IngestLambdaRole"
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: LambdaPolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - s3:GetObject
                Resource: "*"
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:PutLogEvents
                Resource: "*"

  APIGatewayRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub "${ProjectName}-APIGatewayRole"
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service: apigateway.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: OpenSearchPolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - es:ESHttpPut
                  - es:ESHttpPost
                  - es:ESHttpGet
                Resource: !Sub "arn:${AWS::Partition}:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}/*"

  ## Lambda
  IngestLambdaFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: !Sub "${ProjectName}-IngestLambda"
      Runtime: nodejs18.x
      Handler: index.handler
      Role: !GetAtt IngestLambdaRole.Arn
      Timeout: 30
      Environment:
        Variables:
          API_GATEWAY_ENDPOINT: !Sub "https://${Gateway}.execute-api.${AWS::Region}.amazonaws.com/prod"
      Code:
        ZipFile: |
          const https = require('https');
          const { URL } = require('url');
          const { S3Client, GetObjectCommand } = require('@aws-sdk/client-s3');

          const apiUrl = process.env.API_GATEWAY_ENDPOINT;

          exports.handler = async (event) => {
            const record = event.Records[0];
            const bucket = record.s3.bucket.name;
            const key = decodeURIComponent(record.s3.object.key.replace(/\+/g, ' '));
            const [jurisdiction, filename] = key.split('/');
            const date = filename.replace('.txt', '');

            const client = new S3Client();
            const s3Res = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
            const body = await streamToString(s3Res.Body);

            const doc = { jurisdiction, date, body };

            console.log(key);

            await sendToApiGateway(doc, key.replace(/\//g, '_').replace('.txt', '').replace(/-/g, '_'));
          };

          async function sendToApiGateway(doc, id) {
            console.log(`${apiUrl}/text-index/_create/${id}`);
            const url = new URL(`${apiUrl}/text-index/_create/${id}`);
            const json = JSON.stringify(doc);

            const options = {
              method: 'PUT',
              hostname: url.hostname,
              path: url.pathname,
              headers: {
                'Content-Type': 'application/json',
                'Content-Length': Buffer.byteLength(json),
              }
            };

            return new Promise((resolve, reject) => {
              const req = https.request(options, (res) => {
                res.setEncoding('utf8');
                let data = '';
                res.on('data', (chunk) => {
                  console.log(`OpenSearch response chunk: ${chunk}`);
                  data += chunk;
                });
                res.on('end', () => {
                  console.log(`OpenSearch response status: ${res.statusCode}`);
                  console.log(`OpenSearch response body: ${data}`);
                  if (res.statusCode >= 200 && res.statusCode < 300) {
                    resolve();
                  } else {
                    reject(new Error(`OpenSearch error: ${res.statusCode} - ${data}`));
                  }
                });
              });

              req.on('error', (err) => {
                console.error(`Request failed: ${err}`);
                reject(err);
              });

              req.write(json);
              req.end();
            });
          }

          function streamToString(stream) {
            return new Promise((resolve, reject) => {
              const chunks = [];
              stream.on('data', (chunk) => chunks.push(chunk));
              stream.on('error', reject);
              stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
            });
          }

  LambdaInvokePermission:
    Type: AWS::Lambda::Permission
    Properties:
      FunctionName: !Ref IngestLambdaFunction
      Action: lambda:InvokeFunction
      Principal: s3.amazonaws.com

  ## S3
  IngestBucket:
    Type: AWS::S3::Bucket
    Properties:
      NotificationConfiguration:
        LambdaConfigurations:
          - Event: "s3:ObjectCreated:*"
            Function: !GetAtt IngestLambdaFunction.Arn

  ## API Gateway
  Gateway:
    Type: AWS::ApiGateway::RestApi
    Properties:
      Name: !Sub "${ProjectName}-APIGateway"

  GatewayRootMethod:
    Type: AWS::ApiGateway::Method
    Properties:
      ResourceId: !GetAtt Gateway.RootResourceId
      RestApiId: !Ref Gateway
      HttpMethod: ANY
      AuthorizationType: NONE
      Integration:
        IntegrationHttpMethod: "ANY"
        Type: "HTTP_PROXY"
        Uri: !Sub "${OpenSearchDomainEndpoint}/"

  GatewayProxy:
    Type: AWS::ApiGateway::Resource
    Properties:
      ParentId: !GetAtt Gateway.RootResourceId
      RestApiId: !Ref Gateway
      PathPart: "{proxy+}"

  GatewayProxyMethod:
    Type: AWS::ApiGateway::Method
    Properties:
      ResourceId: !Ref GatewayProxy
      RestApiId: !Ref Gateway
      HttpMethod: ANY
      AuthorizationType: NONE
      RequestParameters:
          method.request.path.proxy: true
      Integration:
        IntegrationHttpMethod: "ANY"
        Type: HTTP_PROXY
        Uri: !Sub "${OpenSearchDomainEndpoint}/{proxy}"
        RequestParameters:
          integration.request.path.proxy: "method.request.path.proxy"
        Credentials: !GetAtt APIGatewayRole.Arn

  GatewayDeployment:
    Type: AWS::ApiGateway::Deployment
    DependsOn:
      - GatewayRootMethod
      - GatewayProxyMethod
    Properties:
      RestApiId: !Ref Gateway

  # Create a CloudWatch Logs Log Group for API Gateway logs
  GatewayLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub "/aws/apigateway/${ProjectName}"
      RetentionInDays: 14

  # Create an explicit API Gateway Stage with logging settings
  GatewayStage:
    Type: AWS::ApiGateway::Stage
    Properties:
      StageName: prod
      DeploymentId: !Ref GatewayDeployment
      RestApiId: !Ref Gateway
      MethodSettings:
        - DataTraceEnabled: true
          HttpMethod: "*"
          LoggingLevel: INFO
          ResourcePath: "/*"
      AccessLogSetting:
        DestinationArn: !GetAtt GatewayLogGroup.Arn
        Format: >
          { "requestId": "$context.requestId", "ip": "$context.identity.sourceIp", "caller": "$context.identity.caller", "user": "$context.identity.user", "requestTime": "$context.requestTime", "httpMethod": "$context.httpMethod", "resourcePath": "$context.resourcePath", "status": "$context.status", "protocol": "$context.protocol", "responseLength": "$context.responseLength", "integrationStatus": "$context.integration.status", "integrationError": "$context.integration.error", "integrationLatency": "$context.integration.latency", "integrationRequestId": "$context.integration.requestId", "integrationService": "$context.integration.service", "integrationPath": "$context.integration.path" }

Outputs:
  BucketName:
    Value: !Ref IngestBucket
  ApiInvokeURL:
    Value: !Sub "https://${Gateway}.execute-api.${AWS::Region}.amazonaws.com/prod"
  IngestLambdaName:
    Value: !Ref IngestLambdaFunction
  GatewayValue:
    Value: !Ref Gateway
  OpenSearchDomainEndpoint:
    Value: !Ref OpenSearchDomainEndpoint