From f5384f84370f0bb6b205c6f7273e7942130e7ab7 Mon Sep 17 00:00:00 2001
From: Admin <dmparrow@gmail.com>
Date: Tue, 29 Sep 2026 06:29:40 +0000
Subject: [PATCH] Sync robust Parliament ingestion

---
 src/sources.ts | 73 +++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 61 insertions(+), 12 deletions(-)

diff --git a/src/sources.ts b/src/sources.ts
index 9871938..9c18b66 100644
--- a/src/sources.ts
+++ b/src/sources.ts
@@ -157,28 +157,47 @@ function parseParliamentDate(text: string): string | null {
 
 async function fetchParliament(): Promise<NormalizedItem[]> {
   const source = SOURCES.find((s) => s.id === "parliament")!;
-  const html = await getText(source.url, "text/html");
+  const response = await fetch(source.url, {
+    headers: {
+      accept: "text/html,application/xhtml+xml",
+      "accept-language": "en-NZ,en;q=0.9",
+      "user-agent": "Mozilla/5.0 (compatible; ArrowstackMediaIntelligence/0.1; +https://media.arrowstack.dev)"
+    },
+    redirect: "follow",
+    signal: AbortSignal.timeout(20000)
+  });
+  if (!response.ok) {
+    throw new Error(`${response.status} ${response.statusText} for ${source.url}`);
+  }
+
+  const html = await response.text();
   const $ = load(html);
   const seen = new Set<string>();
   const items: NormalizedItem[] = [];
 
-  $('a[href*="/en/pb/sc/submissions-and-advice/document/"]').each((_index, element) => {
+  const addItem = (
+    href: string,
+    titleText: string,
+    surroundingText = "",
+    fallbackIndex = 0
+  ) => {
     if (items.length >= 30) return;
-
-    const href = $(element).attr("href");
-    const title = $(element).text().replace(/\s+/g, " ").trim();
+    const title = titleText.replace(/\s+/g, " ").trim();
     if (!href || !title || title.length < 2) return;
 
     const url = new URL(href, source.url).toString().split("?")[0];
+    if (!url.includes("/submissions-and-advice/document/")) return;
     if (seen.has(url)) return;
     seen.add(url);
 
-    const container = $(element).closest("tr").length
-      ? $(element).closest("tr")
-      : $(element).closest("li, article, .list-item, div");
-
-    const surrounding = container.text().replace(/\s+/g, " ").trim();
-    const summary = surrounding.replace(title, "").replace(/View more View less/gi, "").trim().slice(0, 1400);
+    const surrounding = surroundingText.replace(/\s+/g, " ").trim();
+    const summary = (
+      surrounding
+        .replace(title, "")
+        .replace(/View more View less/gi, "")
+        .trim() ||
+      "Released evidence or submission listed by New Zealand Parliament."
+    ).slice(0, 1400);
 
     const committee =
       surrounding.match(/Committee:\s*([^|]+?)(?:Date:|$)/i)?.[1]?.trim() ??
@@ -189,7 +208,7 @@ async function fetchParliament(): Promise<NormalizedItem[]> {
 
     items.push({
       sourceId: "parliament",
-      externalId: url,
+      externalId: url || `parliament-${fallbackIndex}`,
       title,
       summary,
       url,
@@ -202,8 +221,38 @@ async function fetchParliament(): Promise<NormalizedItem[]> {
       ],
       raw: { title, summary, url, committee }
     });
+  };
+
+  $('a[href*="submissions-and-advice/document/"]').each((index, element) => {
+    const href = $(element).attr("href") ?? "";
+    const container = $(element).closest("tr").length
+      ? $(element).closest("tr")
+      : $(element).closest("li, article, .list-item, div");
+    addItem(
+      href,
+      $(element).text(),
+      container.text(),
+      index
+    );
   });
 
+  if (!items.length) {
+    const linkPattern = /<a\b[^>]*href=["']([^"']*submissions-and-advice\/document\/[^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi;
+    let match: RegExpExecArray | null;
+    let index = 0;
+    while ((match = linkPattern.exec(html)) !== null && items.length < 30) {
+      const title = load(`<div>${match[2]}</div>`)("div").text();
+      addItem(match[1], title, "", index++);
+    }
+  }
+
+  if (!items.length) {
+    const pageTitle = $("title").text().replace(/\s+/g, " ").trim();
+    throw new Error(
+      `Parliament evidence listing returned no document links (bytes=${html.length}, title=${pageTitle || "unknown"})`
+    );
+  }
+
   return items;
 }