o110.pyWriteDynamicFrame error in AWS Glue

0

Hi there,

I used the same script for another table and got the error in AWS Glue "An error occurred while calling o110.pyWriteDynamicFrame. Exception thrown in awaitResult:" I have checked the script a few times and re-created the logic from scratch too.

Could anyone help me to understand where the issue is?

Script is below

import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job

@params: [TempDir, JOB_NAME]

args = getResolvedOptions(sys.argv, ['TempDir','JOB_NAME'])

sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
job = Job(glueContext)
job.init(args['JOB_NAME'], args)

@type: DataSource

@args: [database = "csvdatafroms3 - customer.io - metrics", table_name = "customer_io", transformation_ctx = "datasource0"]

@return: datasource0

@inputs: []

datasource0 = glueContext.create_dynamic_frame.from_catalog(database = "csvdatafroms3 - customer.io - metrics", table_name = "customer_io", transformation_ctx = "datasource0")

@type: ApplyMapping

@args: [mapping = [("event_id", "string", "event_id", "string"), ("workspace_id", "int", "workspace_id", "int"), ("delivery_id", "string", "delivery_id", "string"), ("metric", "string", "metric", "string"), ("created_at", "timestamp", "created_at", "timestamp"), ("reason", "string", "reason", "string"), ("link_id", "int", "link_id", "int"), ("link_url", "string", "link_url", "string")], transformation_ctx = "applymapping1"]

@return: applymapping1

@inputs: [frame = datasource0]

applymapping1 = ApplyMapping.apply(frame = datasource0, mappings = [("event_id", "string", "event_id", "string"), ("workspace_id", "int", "workspace_id", "int"), ("delivery_id", "string", "delivery_id", "string"), ("metric", "string", "metric", "string"), ("created_at", "timestamp", "created_at", "timestamp"), ("reason", "string", "reason", "string"), ("link_id", "int", "link_id", "int"), ("link_url", "string", "link_url", "string")], transformation_ctx = "applymapping1")

@type: SelectFields

@args: [paths = ["workspace_id", "reason", "event_id", "metric", "delivery_id", "created_at", "link_url", "link_id"], transformation_ctx = "selectfields2"]

@return: selectfields2

@inputs: [frame = applymapping1]

selectfields2 = SelectFields.apply(frame = applymapping1, paths = ["workspace_id", "reason", "event_id", "metric", "delivery_id", "created_at", "link_url", "link_id"], transformation_ctx = "selectfields2")

@type: ResolveChoice

@args: [choice = "MATCH_CATALOG", database = "redshiftschemaall", table_name = "dev_customerionew_metrics", transformation_ctx = "resolvechoice3"]

@return: resolvechoice3

@inputs: [frame = selectfields2]

resolvechoice3 = ResolveChoice.apply(frame = selectfields2, choice = "MATCH_CATALOG", database = "redshiftschemaall", table_name = "dev_customerionew_metrics", transformation_ctx = "resolvechoice3")

@type: ResolveChoice

@args: [choice = "make_cols", transformation_ctx = "resolvechoice4"]

@return: resolvechoice4

@inputs: [frame = resolvechoice3]

resolvechoice4 = ResolveChoice.apply(frame = resolvechoice3, choice = "make_cols", transformation_ctx = "resolvechoice4")

@type: DataSink

@args: [database = "redshiftschemaall", table_name = "dev_customerionew_metrics", redshift_tmp_dir = TempDir, transformation_ctx = "datasink5"]

@return: datasink5

@inputs: [frame = resolvechoice4]

datasink5 = glueContext.write_dynamic_frame.from_catalog(frame = resolvechoice4, database = "redshiftschemaall", table_name = "dev_customerionew_metrics", redshift_tmp_dir = args["TempDir"], transformation_ctx = "datasink5")
job.commit()

gefragt vor 4 Jahren2711 Aufrufe
1 Antwort
0

Did you fix this?

beantwortet vor 3 Jahren

Du bist nicht angemeldet. Anmelden um eine Antwort zu veröffentlichen.

Eine gute Antwort beantwortet die Frage klar, gibt konstruktives Feedback und fördert die berufliche Weiterentwicklung des Fragenstellers.

Richtlinien für die Beantwortung von Fragen