From 2459d72420886bf62dcb55ae269fc3ab1e9e89bc Mon Sep 17 00:00:00 2001 From: Admin Date: Tue, 29 Sep 2026 06:35:46 +0000 Subject: [PATCH] Sync Parliament Atom fallback --- src/sources.ts | 248 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 171 insertions(+), 77 deletions(-) diff --git a/src/sources.ts b/src/sources.ts index 9c18b66..6856ffb 100644 --- a/src/sources.ts +++ b/src/sources.ts @@ -155,107 +155,201 @@ function parseParliamentDate(text: string): string | null { return Number.isNaN(value.getTime()) ? null : value.toISOString(); } -async function fetchParliament(): Promise { - const source = SOURCES.find((s) => s.id === "parliament")!; - const response = await fetch(source.url, { +async function fetchParliamentAtom(source: SourceDefinition): Promise { + const feedUrl = "https://www3.parliament.nz/en/channellisting/rss/9343/e37f168d-4521-4437-a6fd-6bb7ff54e800"; + const response = await fetch(feedUrl, { headers: { - accept: "text/html,application/xhtml+xml", - "accept-language": "en-NZ,en;q=0.9", + accept: "application/atom+xml,application/xml,text/xml;q=0.9,*/*;q=0.5", "user-agent": "Mozilla/5.0 (compatible; ArrowstackMediaIntelligence/0.1; +https://media.arrowstack.dev)" }, redirect: "follow", signal: AbortSignal.timeout(20000) }); if (!response.ok) { - throw new Error(`${response.status} ${response.statusText} for ${source.url}`); + throw new Error(`Parliament Atom feed HTTP ${response.status} ${response.statusText}`); } - const html = await response.text(); - const $ = load(html); - const seen = new Set(); - const items: NormalizedItem[] = []; + const xml = await response.text(); + if (/radware page/i.test(xml) || /\s*radware page\s*<\/title>/i.test(xml)) { + throw new Error(`Parliament Atom feed blocked by Radware (bytes=${xml.length})`); + } - const addItem = ( - href: string, - titleText: string, - surroundingText = "", - fallbackIndex = 0 - ) => { - if (items.length >= 30) return; - const title = titleText.replace(/\s+/g, " ").trim(); - if (!href || !title || title.length < 2) return; + const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" }); + const parsed = parser.parse(xml); + let list = asArray<any>( + parsed?.feed?.entry ?? + parsed?.rss?.channel?.item ?? + parsed?.["atom:feed"]?.["atom:entry"] + ); + if (!list.length) list = rawFeedItems(xml); - const url = new URL(href, source.url).toString().split("?")[0]; - if (!url.includes("/submissions-and-advice/document/")) return; - if (seen.has(url)) return; - seen.add(url); + const items = list.slice(0, 30).map((entry: any, index: number) => { + const title = cleanXmlText( + entry.title?.["#text"] ?? + entry["atom:title"]?.["#text"] ?? + entry.title ?? + entry["atom:title"] ?? + "Parliament item" + ); + const summary = cleanXmlText( + entry.summary?.["#text"] ?? + entry["atom:summary"]?.["#text"] ?? + entry.summary ?? + entry["atom:summary"] ?? + entry.description ?? + entry.content ?? + "" + ); + const linkValue = entry.link ?? entry["atom:link"]; + const url = firstLink(linkValue) || feedUrl; + const publishedAt = + cleanXmlText( + entry.updated ?? + entry["atom:updated"] ?? + entry.published ?? + entry["atom:published"] ?? + entry.pubDate ?? + "" + ) || null; - const surrounding = surroundingText.replace(/\s+/g, " ").trim(); - const summary = ( - surrounding - .replace(title, "") - .replace(/View more View less/gi, "") - .trim() || - "Released evidence or submission listed by New Zealand Parliament." - ).slice(0, 1400); - - const committee = - surrounding.match(/Committee:\s*([^|]+?)(?:Date:|$)/i)?.[1]?.trim() ?? - ""; - const dateText = - surrounding.match(/Date:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ?? - surrounding; - - items.push({ - sourceId: "parliament", - externalId: url || `parliament-${fallbackIndex}`, + return { + sourceId: "parliament" as const, + externalId: String( + entry.id?.["#text"] ?? + entry["atom:id"]?.["#text"] ?? + entry.id ?? + entry["atom:id"] ?? + entry.guid ?? + url ?? + index + ), title, summary, url, - publishedAt: parseParliamentDate(dateText), - topics: [ - "parliament", - "select-committee", - "evidence", - ...(committee ? [committee.toLowerCase().replace(/\s+/g, "-")] : []) - ], - raw: { title, summary, url, committee } - }); - }; - - $('a[href*="submissions-and-advice/document/"]').each((index, element) => { - const href = $(element).attr("href") ?? ""; - const container = $(element).closest("tr").length - ? $(element).closest("tr") - : $(element).closest("li, article, .list-item, div"); - addItem( - href, - $(element).text(), - container.text(), - index - ); - }); + publishedAt, + topics: ["parliament", "select-committee", "evidence"], + raw: entry + }; + }).filter((item: NormalizedItem) => item.title && item.url); if (!items.length) { - const linkPattern = /<a\b[^>]*href=["']([^"']*submissions-and-advice\/document\/[^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi; - let match: RegExpExecArray | null; - let index = 0; - while ((match = linkPattern.exec(html)) !== null && items.length < 30) { - const title = load(`<div>${match[2]}</div>`)("div").text(); - addItem(match[1], title, "", index++); - } - } - - if (!items.length) { - const pageTitle = $("title").text().replace(/\s+/g, " ").trim(); throw new Error( - `Parliament evidence listing returned no document links (bytes=${html.length}, title=${pageTitle || "unknown"})` + `Parliament Atom feed contained no entries (bytes=${xml.length}, prefix=${xml.slice(0, 120).replace(/\s+/g, " ")})` ); } return items; } +async function fetchParliament(): Promise<NormalizedItem[]> { + const source = SOURCES.find((s) => s.id === "parliament")!; + let htmlFailure = ""; + + try { + const response = await fetch(source.url, { + headers: { + accept: "text/html,application/xhtml+xml", + "accept-language": "en-NZ,en;q=0.9", + "user-agent": "Mozilla/5.0 (compatible; ArrowstackMediaIntelligence/0.1; +https://media.arrowstack.dev)" + }, + redirect: "follow", + signal: AbortSignal.timeout(20000) + }); + if (!response.ok) { + throw new Error(`${response.status} ${response.statusText}`); + } + + const html = await response.text(); + if (/radware page/i.test(html) || /<title>\s*radware page\s*<\/title>/i.test(html)) { + throw new Error(`blocked by Radware (bytes=${html.length})`); + } + + const $ = load(html); + const seen = new Set<string>(); + const items: NormalizedItem[] = []; + + const addItem = ( + href: string, + titleText: string, + surroundingText = "", + fallbackIndex = 0 + ) => { + if (items.length >= 30) return; + const title = titleText.replace(/\s+/g, " ").trim(); + if (!href || !title || title.length < 2) return; + + const url = new URL(href, source.url).toString().split("?")[0]; + if (!url.includes("/submissions-and-advice/document/")) return; + if (seen.has(url)) return; + seen.add(url); + + const surrounding = surroundingText.replace(/\s+/g, " ").trim(); + const summary = ( + surrounding + .replace(title, "") + .replace(/View more View less/gi, "") + .trim() || + "Released evidence or submission listed by New Zealand Parliament." + ).slice(0, 1400); + + const committee = + surrounding.match(/Committee:\s*([^|]+?)(?:Date:|$)/i)?.[1]?.trim() ?? + ""; + const dateText = + surrounding.match(/Date:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ?? + surrounding; + + items.push({ + sourceId: "parliament", + externalId: url || `parliament-${fallbackIndex}`, + title, + summary, + url, + publishedAt: parseParliamentDate(dateText), + topics: [ + "parliament", + "select-committee", + "evidence", + ...(committee ? [committee.toLowerCase().replace(/\s+/g, "-")] : []) + ], + raw: { title, summary, url, committee } + }); + }; + + $('a[href*="submissions-and-advice/document/"]').each((index, element) => { + const href = $(element).attr("href") ?? ""; + const container = $(element).closest("tr").length + ? $(element).closest("tr") + : $(element).closest("li, article, .list-item, div"); + addItem(href, $(element).text(), container.text(), index); + }); + + if (!items.length) { + const linkPattern = /<a\b[^>]*href=["']([^"']*submissions-and-advice\/document\/[^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi; + let match: RegExpExecArray | null; + let index = 0; + while ((match = linkPattern.exec(html)) !== null && items.length < 30) { + const title = load(`<div>${match[2]}</div>`)("div").text(); + addItem(match[1], title, "", index++); + } + } + + if (items.length) return items; + htmlFailure = `HTML listing contained no document links (bytes=${html.length})`; + } catch (error) { + htmlFailure = error instanceof Error ? error.message : String(error); + } + + try { + return await fetchParliamentAtom(source); + } catch (error) { + const atomFailure = error instanceof Error ? error.message : String(error); + throw new Error( + `Parliament unavailable from Arrowstack. HTML: ${htmlFailure || "unknown failure"}; Atom: ${atomFailure}` + ); + } +} + interface SitemapEntry { loc: string; lastmod: string | null;