o110.pyWriteDynamicFrame error in AWS Glue

Hi there,

I used the same script for another table and got the error in AWS Glue "An error occurred while calling o110.pyWriteDynamicFrame. Exception thrown in awaitResult:" I have checked the script a few times and re-created the logic from scratch too.

Could anyone help me to understand where the issue is?

Script is below

import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job

@params: [TempDir, JOB_NAME]

args = getResolvedOptions(sys.argv, ['TempDir','JOB_NAME'])

sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
job = Job(glueContext)
job.init(args['JOB_NAME'], args)

@type: DataSource

@args: [database = "csvdatafroms3 - customer.io - metrics", table_name = "customer_io", transformation_ctx = "datasource0"]

@return: datasource0

@inputs: []

datasource0 = glueContext.create_dynamic_frame.from_catalog(database = "csvdatafroms3 - customer.io - metrics", table_name = "customer_io", transformation_ctx = "datasource0")

@type: ApplyMapping

@args: [mapping = [("event_id", "string", "event_id", "string"), ("workspace_id", "int", "workspace_id", "int"), ("delivery_id", "string", "delivery_id", "string"), ("metric", "string", "metric", "string"), ("created_at", "timestamp", "created_at", "timestamp"), ("reason", "string", "reason", "string"), ("link_id", "int", "link_id", "int"), ("link_url", "string", "link_url", "string")], transformation_ctx = "applymapping1"]

@return: applymapping1

@inputs: [frame = datasource0]

applymapping1 = ApplyMapping.apply(frame = datasource0, mappings = [("event_id", "string", "event_id", "string"), ("workspace_id", "int", "workspace_id", "int"), ("delivery_id", "string", "delivery_id", "string"), ("metric", "string", "metric", "string"), ("created_at", "timestamp", "created_at", "timestamp"), ("reason", "string", "reason", "string"), ("link_id", "int", "link_id", "int"), ("link_url", "string", "link_url", "string")], transformation_ctx = "applymapping1")

@type: SelectFields

@args: [paths = ["workspace_id", "reason", "event_id", "metric", "delivery_id", "created_at", "link_url", "link_id"], transformation_ctx = "selectfields2"]

@return: selectfields2

@inputs: [frame = applymapping1]

selectfields2 = SelectFields.apply(frame = applymapping1, paths = ["workspace_id", "reason", "event_id", "metric", "delivery_id", "created_at", "link_url", "link_id"], transformation_ctx = "selectfields2")

@type: ResolveChoice

@args: [choice = "MATCH_CATALOG", database = "redshiftschemaall", table_name = "dev_customerionew_metrics", transformation_ctx = "resolvechoice3"]

@return: resolvechoice3

@inputs: [frame = selectfields2]

resolvechoice3 = ResolveChoice.apply(frame = selectfields2, choice = "MATCH_CATALOG", database = "redshiftschemaall", table_name = "dev_customerionew_metrics", transformation_ctx = "resolvechoice3")

@type: ResolveChoice

@args: [choice = "make_cols", transformation_ctx = "resolvechoice4"]

@return: resolvechoice4

@inputs: [frame = resolvechoice3]

resolvechoice4 = ResolveChoice.apply(frame = resolvechoice3, choice = "make_cols", transformation_ctx = "resolvechoice4")

@type: DataSink

@args: [database = "redshiftschemaall", table_name = "dev_customerionew_metrics", redshift_tmp_dir = TempDir, transformation_ctx = "datasink5"]

@return: datasink5

@inputs: [frame = resolvechoice4]

datasink5 = glueContext.write_dynamic_frame.from_catalog(frame = resolvechoice4, database = "redshiftschemaall", table_name = "dev_customerionew_metrics", redshift_tmp_dir = args["TempDir"], transformation_ctx = "datasink5")
job.commit()