AWS_FEEDS TABLE_NOT_FOUND discrepancy

Can you provide a link the process you are following?

I believe the command you are running is looking for a data source in the Glue Data Catalog that does not exist. To access CUR data, you need to populate the data in an S3 bucket, crawl that with Glue and then can query it via with reference to the data catalog.

Here is a link to setup Cloud Intelligence Dashboard: https://catalog.workshops.aws/awscid/en-US/dashboards/foundational/cudos-cid-kpi

Hope this helps.

The problem is within the AWS Lambda function: CID-DC-aws-feeds-Whats-New-Lambda

Which was not able to parse the aws-cid feed. The below code will make that drop-in work for you and create the respective files in your cid-data bucket. Then upon triggering the Stepfunction "CID-DC-aws-feeds-Whats-New-StateMachine" manually you should be able to see the table in athena under:

Athena > DataSource - AwsDataCatalog > Database - optimization_data > Tables - aws_feeds_whats_new

import os
import json
import re
import urllib.request
import xml.etree.ElementTree as ET
from html.parser import HTMLParser
from dateutil.parser import parse
import boto3


FEEDS_MAP = {
    SAME_AS_BEFORE
}

class LinkStripper(HTMLParser):
    """
    Convert HTML to plain text while keeping links as [n] references.
    """
    def __init__(self):
        super().__init__()
        self.fragments = []
        self.refs = {}
        self.idx = 0

    def handle_starttag(self, tag, attrs):
        if tag == "a":
            href = dict(attrs).get("href")
            if href:
                if href.startswith("/"):
                    href = f"https://aws.amazon.com{href}"
                self.idx += 1
                self.refs[self.idx] = href

    def handle_endtag(self, tag):
        if tag == "a":
            self.fragments.append(f"[{self.idx}]")

    def handle_data(self, data):
        self.fragments.append(data)

    def get_text(self):
        refs = "\n".join(f"[{i}]: {url}" for i, url in self.refs.items())
        return f"{''.join(self.fragments).strip()}\n\n{refs}"


def clean_html(html_content: str) -> str:
    stripper = LinkStripper()
    stripper.feed(html_content or "")
    return stripper.get_text()

CDATA_WRAPPED = re.compile(r'<!\[CDATA\[.*?\]\]>', re.S)

def _ensure_cdata(match: re.Match) -> str:
    """
    Wrap inner HTML of <description> or <content:encoded> inside CDATA,
    unless it is already wrapped. Also escapes any ']]>' sequences.
    """
    open_tag, inner, close_tag = match.groups()
    if CDATA_WRAPPED.fullmatch(inner.strip()):
        return match.group(0)  # already safe

    inner = inner.replace("]]>", "]]]]><![CDATA[>")
    return f"{open_tag}<![CDATA[{inner}]]>{close_tag}"

def sanitize_feed_xml(xml_text: str) -> str:
    """
    Makes <description> and <content:encoded> safe for ElementTree
    by wrapping their contents in CDATA.
    """
    for tag in ("description", "content:encoded"):
        pattern = re.compile(rf"(<{tag}[^>]*>)(.*?)(</{tag}>)", re.S | re.I)
        xml_text = pattern.sub(_ensure_cdata, xml_text)
    return xml_text

def lambda_handler(event, context):
    """
    Fetches each RSS feed, sanitises it, converts items to JSONL and
    uploads partitioned files to S3. Runs without third‑party parsers.
    """
    feeds_list  = os.environ["FEEDS_LIST"].split(",")  # e.g. "aws,aws-cid"
    bucket_name = os.environ["BUCKET_NAME"]
    s3          = boto3.client("s3")

    processed_feeds = []
    failed_feeds = []

    for feed_key in feeds_list:
        cfg = FEEDS_MAP.get(feed_key)
        if not cfg:
            print(f"Feed '{feed_key}' unknown; skipping.")
            continue

        feed_url   = cfg["feed_url"]
        bucket_path = cfg["path"]
        print(f"Processing '{feed_key}' from {feed_url}")

        try:
            with urllib.request.urlopen(feed_url, timeout=10) as resp:  # nosec
                raw_xml = resp.read().decode("utf-8")

            for bad in ("!ENTITY", ":include"):
                if bad in raw_xml:
                    raise ValueError(f"Malicious content detected: {bad}")

            safe_xml = sanitize_feed_xml(raw_xml)
            root = ET.fromstring(safe_xml)

            date_buckets = {}

            for item in root.findall(".//item"):
                try:
                    link = item.findtext("link", "")
                    title = item.findtext("title", "")
                    description = item.findtext("description", "")
                    pub_date = item.findtext("pubDate", "")
                    categories_tags = item.findall("category")
                    raw_categories = [c.text for c in categories_tags if c.text] or [""]

                    dt = parse(pub_date)
                    iso_date = dt.strftime("%Y-%m-%dT%H:%M:%SZ")
                    yyyy, mm, dd = iso_date[:10].split("-")
                    date_key = f"{yyyy}-{mm}-{dd}"

                    description_clean = clean_html(description)

                    services = set(cfg.get("default_services", []))
                    categories = set()

                    for cat in raw_categories:
                        if cat.startswith("general:products/"):
                            services.add(cat.replace("general:products/", ""))
                        elif cat.startswith("marketing:marchitecture/"):
                            categories.add(cat.replace("marketing:marchitecture/", ""))
                        else:
                            categories.add(cat)

                    if not categories:
                        categories.add("")

                    for svc in services or {""}:
                        for cat in categories:
                            rec = {
                                "link": link,
                                "title": title,
                                "description": description_clean,
                                "date": iso_date,
                                "service": svc,
                                "category": cat,
                            }
                            date_buckets.setdefault(date_key, []).append(rec)

                except Exception as ex:
                    print(f"Skipping an item in '{feed_url}' due to: {ex}")

            for date_key, records in date_buckets.items():
                y, m, d = date_key.split("-")
                s3_key = f"{bucket_path}/year={y}/month={m}/day={d}/whats_new.jsonl"
                body = "\n".join(json.dumps(r) for r in records)
                print(f"Uploading {len(records)} records to s3://{bucket_name}/{s3_key}")
                s3.put_object(Bucket=bucket_name, Key=s3_key, Body=body)

            processed_feeds.append(feed_key)

        except Exception as ex:
            print(f"Failed to process feed '{feed_key}': {ex}")
            failed_feeds.append(f"{feed_key}: {ex}")

    if failed_feeds:
        return {
            "statusCode": 207,
            "body": json.dumps({
                "message": "Completed with errors.",
                "processed_feeds": processed_feeds,
                "failed_feeds": failed_feeds,
            }),
        }

    return {
        "statusCode": 200,
        "body": json.dumps({
            "message": "All feeds processed successfully.",
            "processed_feeds": processed_feeds,
        }),
    }