Sync Parliament Atom fallback

src/sources.ts

@@ -155,107 +155,201 @@ function parseParliamentDate(text: string): string | null {
   return Number.isNaN(value.getTime()) ? null : value.toISOString();
 }
 
-async function fetchParliament(): Promise<NormalizedItem[]> {
+async function fetchParliamentAtom(source: SourceDefinition): Promise<NormalizedItem[]> {
-  const source = SOURCES.find((s) => s.id === "parliament")!;
+  const feedUrl = "https://www3.parliament.nz/en/channellisting/rss/9343/e37f168d-4521-4437-a6fd-6bb7ff54e800";
-  const response = await fetch(source.url, {
+  const response = await fetch(feedUrl, {
     headers: {
-      accept: "text/html,application/xhtml+xml",
+      accept: "application/atom+xml,application/xml,text/xml;q=0.9,*/*;q=0.5",
-      "accept-language": "en-NZ,en;q=0.9",
       "user-agent": "Mozilla/5.0 (compatible; ArrowstackMediaIntelligence/0.1; +https://media.arrowstack.dev)"
     },
     redirect: "follow",
     signal: AbortSignal.timeout(20000)
   });
   if (!response.ok) {
-    throw new Error(`${response.status} ${response.statusText} for ${source.url}`);
+    throw new Error(`Parliament Atom feed HTTP ${response.status} ${response.statusText}`);
   }
 
-  const html = await response.text();
+  const xml = await response.text();
-  const $ = load(html);
+  if (/radware page/i.test(xml) || /<title>\s*radware page\s*<\/title>/i.test(xml)) {
-  const seen = new Set<string>();
+    throw new Error(`Parliament Atom feed blocked by Radware (bytes=${xml.length})`);
-  const items: NormalizedItem[] = [];
+  }
 
-  const addItem = (
+  const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
-    href: string,
+  const parsed = parser.parse(xml);
-    titleText: string,
+  let list = asArray<any>(
-    surroundingText = "",
+    parsed?.feed?.entry ??
-    fallbackIndex = 0
+    parsed?.rss?.channel?.item ??
-  ) => {
+    parsed?.["atom:feed"]?.["atom:entry"]
-    if (items.length >= 30) return;
+  );
-    const title = titleText.replace(/\s+/g, " ").trim();
+  if (!list.length) list = rawFeedItems(xml);
-    if (!href || !title || title.length < 2) return;
 
-    const url = new URL(href, source.url).toString().split("?")[0];
+  const items = list.slice(0, 30).map((entry: any, index: number) => {
-    if (!url.includes("/submissions-and-advice/document/")) return;
+    const title = cleanXmlText(
-    if (seen.has(url)) return;
+      entry.title?.["#text"] ??
-    seen.add(url);
+      entry["atom:title"]?.["#text"] ??
+      entry.title ??
+      entry["atom:title"] ??
+      "Parliament item"
+    );
+    const summary = cleanXmlText(
+      entry.summary?.["#text"] ??
+      entry["atom:summary"]?.["#text"] ??
+      entry.summary ??
+      entry["atom:summary"] ??
+      entry.description ??
+      entry.content ??
+      ""
+    );
+    const linkValue = entry.link ?? entry["atom:link"];
+    const url = firstLink(linkValue) || feedUrl;
+    const publishedAt =
+      cleanXmlText(
+        entry.updated ??
+        entry["atom:updated"] ??
+        entry.published ??
+        entry["atom:published"] ??
+        entry.pubDate ??
+        ""
+      ) || null;
 
-    const surrounding = surroundingText.replace(/\s+/g, " ").trim();
+    return {
-    const summary = (
+      sourceId: "parliament" as const,
-      surrounding
+      externalId: String(
-        .replace(title, "")
+        entry.id?.["#text"] ??
-        .replace(/View more View less/gi, "")
+        entry["atom:id"]?.["#text"] ??
-        .trim() ||
+        entry.id ??
-      "Released evidence or submission listed by New Zealand Parliament."
+        entry["atom:id"] ??
-    ).slice(0, 1400);
+        entry.guid ??
-
+        url ??
-    const committee =
+        index
-      surrounding.match(/Committee:\s*([^|]+?)(?:Date:|$)/i)?.[1]?.trim() ??
+      ),
-      "";
-    const dateText =
-      surrounding.match(/Date:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
-      surrounding;
-
-    items.push({
-      sourceId: "parliament",
-      externalId: url || `parliament-${fallbackIndex}`,
       title,
       summary,
       url,
-      publishedAt: parseParliamentDate(dateText),
+      publishedAt,
-      topics: [
+      topics: ["parliament", "select-committee", "evidence"],
-        "parliament",
+      raw: entry
-        "select-committee",
+    };
-        "evidence",
+  }).filter((item: NormalizedItem) => item.title && item.url);
-        ...(committee ? [committee.toLowerCase().replace(/\s+/g, "-")] : [])
-      ],
-      raw: { title, summary, url, committee }
-    });
-  };
-
-  $('a[href*="submissions-and-advice/document/"]').each((index, element) => {
-    const href = $(element).attr("href") ?? "";
-    const container = $(element).closest("tr").length
-      ? $(element).closest("tr")
-      : $(element).closest("li, article, .list-item, div");
-    addItem(
-      href,
-      $(element).text(),
-      container.text(),
-      index
-    );
-  });
 
   if (!items.length) {
-    const linkPattern = /<a\b[^>]*href=["']([^"']*submissions-and-advice\/document\/[^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi;
-    let match: RegExpExecArray | null;
-    let index = 0;
-    while ((match = linkPattern.exec(html)) !== null && items.length < 30) {
-      const title = load(`<div>${match[2]}</div>`)("div").text();
-      addItem(match[1], title, "", index++);
-    }
-  }
-
-  if (!items.length) {
-    const pageTitle = $("title").text().replace(/\s+/g, " ").trim();
     throw new Error(
-      `Parliament evidence listing returned no document links (bytes=${html.length}, title=${pageTitle || "unknown"})`
+      `Parliament Atom feed contained no entries (bytes=${xml.length}, prefix=${xml.slice(0, 120).replace(/\s+/g, " ")})`
     );
   }
 
   return items;
 }
 
+async function fetchParliament(): Promise<NormalizedItem[]> {
+  const source = SOURCES.find((s) => s.id === "parliament")!;
+  let htmlFailure = "";
+
+  try {
+    const response = await fetch(source.url, {
+      headers: {
+        accept: "text/html,application/xhtml+xml",
+        "accept-language": "en-NZ,en;q=0.9",
+        "user-agent": "Mozilla/5.0 (compatible; ArrowstackMediaIntelligence/0.1; +https://media.arrowstack.dev)"
+      },
+      redirect: "follow",
+      signal: AbortSignal.timeout(20000)
+    });
+    if (!response.ok) {
+      throw new Error(`${response.status} ${response.statusText}`);
+    }
+
+    const html = await response.text();
+    if (/radware page/i.test(html) || /<title>\s*radware page\s*<\/title>/i.test(html)) {
+      throw new Error(`blocked by Radware (bytes=${html.length})`);
+    }
+
+    const $ = load(html);
+    const seen = new Set<string>();
+    const items: NormalizedItem[] = [];
+
+    const addItem = (
+      href: string,
+      titleText: string,
+      surroundingText = "",
+      fallbackIndex = 0
+    ) => {
+      if (items.length >= 30) return;
+      const title = titleText.replace(/\s+/g, " ").trim();
+      if (!href || !title || title.length < 2) return;
+
+      const url = new URL(href, source.url).toString().split("?")[0];
+      if (!url.includes("/submissions-and-advice/document/")) return;
+      if (seen.has(url)) return;
+      seen.add(url);
+
+      const surrounding = surroundingText.replace(/\s+/g, " ").trim();
+      const summary = (
+        surrounding
+          .replace(title, "")
+          .replace(/View more View less/gi, "")
+          .trim() ||
+        "Released evidence or submission listed by New Zealand Parliament."
+      ).slice(0, 1400);
+
+      const committee =
+        surrounding.match(/Committee:\s*([^|]+?)(?:Date:|$)/i)?.[1]?.trim() ??
+        "";
+      const dateText =
+        surrounding.match(/Date:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
+        surrounding;
+
+      items.push({
+        sourceId: "parliament",
+        externalId: url || `parliament-${fallbackIndex}`,
+        title,
+        summary,
+        url,
+        publishedAt: parseParliamentDate(dateText),
+        topics: [
+          "parliament",
+          "select-committee",
+          "evidence",
+          ...(committee ? [committee.toLowerCase().replace(/\s+/g, "-")] : [])
+        ],
+        raw: { title, summary, url, committee }
+      });
+    };
+
+    $('a[href*="submissions-and-advice/document/"]').each((index, element) => {
+      const href = $(element).attr("href") ?? "";
+      const container = $(element).closest("tr").length
+        ? $(element).closest("tr")
+        : $(element).closest("li, article, .list-item, div");
+      addItem(href, $(element).text(), container.text(), index);
+    });
+
+    if (!items.length) {
+      const linkPattern = /<a\b[^>]*href=["']([^"']*submissions-and-advice\/document\/[^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi;
+      let match: RegExpExecArray | null;
+      let index = 0;
+      while ((match = linkPattern.exec(html)) !== null && items.length < 30) {
+        const title = load(`<div>${match[2]}</div>`)("div").text();
+        addItem(match[1], title, "", index++);
+      }
+    }
+
+    if (items.length) return items;
+    htmlFailure = `HTML listing contained no document links (bytes=${html.length})`;
+  } catch (error) {
+    htmlFailure = error instanceof Error ? error.message : String(error);
+  }
+
+  try {
+    return await fetchParliamentAtom(source);
+  } catch (error) {
+    const atomFailure = error instanceof Error ? error.message : String(error);
+    throw new Error(
+      `Parliament unavailable from Arrowstack. HTML: ${htmlFailure || "unknown failure"}; Atom: ${atomFailure}`
+    );
+  }
+}
+
 interface SitemapEntry {
   loc: string;
   lastmod: string | null;