o110.pyWriteDynamicFrame error in AWS Glue
Hi there,
I used the same script for another table and got the error in AWS Glue "An error occurred while calling o110.pyWriteDynamicFrame. Exception thrown in awaitResult:" I have checked the script a few times and re-created the logic from scratch too.
Could anyone help me to understand where the issue is?
Script is below
import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job
@params: [TempDir, JOB_NAME]
args = getResolvedOptions(sys.argv, ['TempDir','JOB_NAME'])
sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
job = Job(glueContext)
job.init(args['JOB_NAME'], args)
@type: DataSource
@args: [database = "csvdatafroms3 - customer.io - metrics", table_name = "customer_io", transformation_ctx = "datasource0"]
@return: datasource0
@inputs: []
datasource0 = glueContext.create_dynamic_frame.from_catalog(database = "csvdatafroms3 - customer.io - metrics", table_name = "customer_io", transformation_ctx = "datasource0")
@type: ApplyMapping
@args: [mapping = [("event_id", "string", "event_id", "string"), ("workspace_id", "int", "workspace_id", "int"), ("delivery_id", "string", "delivery_id", "string"), ("metric", "string", "metric", "string"), ("created_at", "timestamp", "created_at", "timestamp"), ("reason", "string", "reason", "string"), ("link_id", "int", "link_id", "int"), ("link_url", "string", "link_url", "string")], transformation_ctx = "applymapping1"]
@return: applymapping1
@inputs: [frame = datasource0]
applymapping1 = ApplyMapping.apply(frame = datasource0, mappings = [("event_id", "string", "event_id", "string"), ("workspace_id", "int", "workspace_id", "int"), ("delivery_id", "string", "delivery_id", "string"), ("metric", "string", "metric", "string"), ("created_at", "timestamp", "created_at", "timestamp"), ("reason", "string", "reason", "string"), ("link_id", "int", "link_id", "int"), ("link_url", "string", "link_url", "string")], transformation_ctx = "applymapping1")
@type: SelectFields
@args: [paths = ["workspace_id", "reason", "event_id", "metric", "delivery_id", "created_at", "link_url", "link_id"], transformation_ctx = "selectfields2"]
@return: selectfields2
@inputs: [frame = applymapping1]
selectfields2 = SelectFields.apply(frame = applymapping1, paths = ["workspace_id", "reason", "event_id", "metric", "delivery_id", "created_at", "link_url", "link_id"], transformation_ctx = "selectfields2")
@type: ResolveChoice
@args: [choice = "MATCH_CATALOG", database = "redshiftschemaall", table_name = "dev_customerionew_metrics", transformation_ctx = "resolvechoice3"]
@return: resolvechoice3
@inputs: [frame = selectfields2]
resolvechoice3 = ResolveChoice.apply(frame = selectfields2, choice = "MATCH_CATALOG", database = "redshiftschemaall", table_name = "dev_customerionew_metrics", transformation_ctx = "resolvechoice3")
@type: ResolveChoice
@args: [choice = "make_cols", transformation_ctx = "resolvechoice4"]
@return: resolvechoice4
@inputs: [frame = resolvechoice3]
resolvechoice4 = ResolveChoice.apply(frame = resolvechoice3, choice = "make_cols", transformation_ctx = "resolvechoice4")
@type: DataSink
@args: [database = "redshiftschemaall", table_name = "dev_customerionew_metrics", redshift_tmp_dir = TempDir, transformation_ctx = "datasink5"]
@return: datasink5
@inputs: [frame = resolvechoice4]
datasink5 = glueContext.write_dynamic_frame.from_catalog(frame = resolvechoice4, database = "redshiftschemaall", table_name = "dev_customerionew_metrics", redshift_tmp_dir = args["TempDir"], transformation_ctx = "datasink5")
job.commit()
- Mais recentes
- Mais votos
- Mais comentários
Conteúdo relevante
- AWS OFICIALAtualizada há 6 meses
- AWS OFICIALAtualizada há 2 anos
- AWS OFICIALAtualizada há 9 meses
- AWS OFICIALAtualizada há 3 anos