Modified data through custom script is updated but when joining this data, old data is coming instead of updated data

I have written a custom script to transform a column, data transformed successfully and I can verify that in data preview in aws glue studio, acft_tail_num -> 8911(earlier this was N8911 ) when I joined the data with another table, I see N8911 instead of 8911. this is happening with other fields as well. attaching script for reference

`def MyTransform (glueContext, dfc) -> DynamicFrameCollection: import pyspark.sql.functions as F from awsglue.dynamicframe import DynamicFrame

# Step 1: Get first DynamicFrame
input_key = list(dfc.keys())[0]
dynamic_frame = dfc.select(input_key)
df = dynamic_frame.toDF()

# Step 2: Original transformations (unchanged sections)
df = df.withColumn(
    "FAULT_DESC_CLEAN",
    F.when(F.col("FAULT_DESC").startswith("<"), F.regexp_replace("FAULT_DESC", "(.*)br>", ""))
     .otherwise(F.col("FAULT_DESC"))
)

df = df.withColumn(
    "FAULT_NAME",
    F.when(
        F.col("FAULT_DESC_CLEAN").isNotNull() & (F.col("FAULT_DESC_CLEAN") != F.col("FAULT_NAME")),
        F.concat(F.col("FAULT_NAME"), F.lit(" "), F.col("FAULT_DESC_CLEAN"))
    ).otherwise(F.col("FAULT_NAME"))
)

df = df.withColumn(
    "FAULT_NAME",
    F.when(
        F.col("FAULT_NAME").substr(1, 7) == F.col("LOG_PAGE_NBR"),
        F.regexp_replace(F.concat(F.col("FAULT_NAME"), F.col("FAULT_DESC_CLEAN")), F.col("LOG_PAGE_NBR"), "")
    ).otherwise(F.concat(F.col("FAULT_NAME"), F.col("FAULT_DESC_CLEAN")))
)

df = df.withColumn(
    "FACT",
    F.when(F.col("CORR_ACTION_EXTSN").isNotNull(),
           F.concat(F.col("FACT"), F.lit(" "), F.col("CORR_ACTION_EXTSN")))
     .otherwise(F.col("FACT"))
)

conditions = [
    (F.col("WO_DESC").contains("30Y"), "30Y"),
    (F.col("WO_DESC").contains("28Y"), "28Y"),
    (F.col("WO_DESC").contains("27Y"), "27Y"),
    (F.col("WO_DESC").contains("26Y"), "26Y"),
    (F.col("WO_DESC").contains("24Y"), "24Y"),
    (F.col("WO_DESC").contains("22Y"), "22Y"),
    (F.col("WO_DESC").contains("21Y"), "21Y"),
    (F.col("WO_DESC").contains("20Y"), "20Y"),
    (F.col("WO_DESC").contains("18Y"), "18Y"),
    (F.col("WO_DESC").contains("16Y"), "16Y"),
    (F.col("WO_DESC").contains("15Y"), "15Y"),
    (F.col("WO_DESC").contains("14Y"), "14Y"),
    (F.col("WO_DESC").contains("12Y"), "12Y"),
    (F.col("WO_DESC").contains("10Y"), "10Y"),
    (F.col("WO_DESC").contains("9Y"), "9Y"),
    (F.col("WO_DESC").contains("8Y"), "8Y"),
    (F.col("WO_DESC").contains("6Y"), "6Y"),
    (F.col("WO_DESC").contains("HRON"), "HRON"),
    (F.col("WO_DESC").contains("IM") & F.col("WO_DESC").contains("HC"), "IM-HC"),
    (F.col("WO_DESC").contains("PAINT"), "PAINT"),
    (F.col("WO_DESC").contains("CABIN VISIT"), "CABIN VISIT"),
    (F.col("WO_DESC").contains("PSEW"), "PSEW"),
]

check_type_col = F.lit("")
for cond, result in conditions:
    check_type_col = F.when(cond, result).otherwise(check_type_col)

df = df.withColumn("CHECK_TYPE", check_type_col)

df = df.withColumn(
    "disc_type_log",
    F.when(
        (F.col("ROW_SOURCE_CD") == 'MTX') &
        ((F.col("FAULT_SOURCE_TYPE") == 'PILOT') | (F.col("FAULT_SOURCE_TYPE") == "LOGPAGE")),
        1
    )
    .when(
        (F.col("ROW_SOURCE_CD") == 'MTX') &
        (F.col("FAULT_SOURCE_TYPE") == 'MECH') &
        ((F.col("LOG_PAGE_NBR").substr(0, 1).between("0", "9")) |
        (F.col("FAULT_TYPE_CD").isin("MEL", "CDL")) |
        F.col("CHECK_TYPE").isNull()), 1
    ).otherwise(0)
)

df = df.withColumn(
    "disc_type_formula",
    F.when(F.col("disc_type_log") == 1, F.col("FAULT_SOURCE_TYPE"))
     .when((F.col("disc_type_log") == 0) & (F.col("FAULT_SOURCE_TYPE") == "MECH"), "NRC")
     .otherwise(F.col("FAULT_SOURCE_TYPE"))
)

df = df.withColumn("unq_id", F.col("FAULT_BARCODE_NUM"))

# Create discrepancy
df = df.withColumn(
    "discrepancy",
    F.concat(F.col("FAULT_NAME"), F.lit(" "), F.col("FAULT_DESC_CLEAN"))
)

# ------------------------------------------------------------------------
# ✅ ADDITIONAL OPERATIONS AFTER DISCREPANCY CREATION
# ------------------------------------------------------------------------

# 1) REGEX_Replace([acft_tail_num], "[^\\d]", "")
df = df.withColumn("acft_tail_num", F.regexp_replace("acft_tail_num", "[^\\d]", ""))

# 2) ATA transformation logic
df = df.withColumn(
    "ata",
    F.when(F.locate(" ", F.col("ata")) == 2,
           F.concat(F.substring(F.col("ata"), 1, 2), F.lit("00")))
     .when(F.locate(" ", F.col("ata")) == 5,
           F.concat(F.substring(F.col("ata"), 1, 2), F.substring(F.col("ata"), 3, 2)))
     .otherwise(F.lit("INV"))
)

# 3) Add a new column Source = "MTX"
df = df.withColumn("Source", F.lit("MTX"))

# 4) resolved_loc_cd = left([resolved_loc_cd],3)
df = df.withColumn("resolved_loc_cd", F.substring(F.col("resolved_loc_cd"), 1, 3))

# 5) REGEX_Replace([DISCREPANCY], "[^ -~]", "$1")
df = df.withColumn("discrepancy", F.regexp_replace("discrepancy", "[^ -~]", "$1"))

# 6) Regex_Replace([DISCREPANCY], "\r*\n*", "")
df = df.withColumn("discrepancy", F.regexp_replace("discrepancy", "\r*\n*", ""))

# 7) Regex_Replace([corr_action], "<br>", " ")
df = df.withColumn("FACT", F.regexp_replace("FACT", "<br>", " "))

# 8) Regex_Replace([FACT], "\r*\n*", "")
df = df.withColumn("FACT", F.regexp_replace("FACT", "\r*\n*", ""))

# 9) Regex_Replace([FACT], "[^ -~]", "$1")
df = df.withColumn("FACT", F.regexp_replace("FACT", "[^ -~]", "$1"))

# ------------------------------------------------------------------------

# Step 3: Convert back to DynamicFrame
result_dynamic_frame = DynamicFrame.fromDF(df, glueContext, "result_dynamic_frame")

# Step 4: Return the result
return DynamicFrameCollection({"CustomTransform0": result_dynamic_frame}, glueContext)
`