Admin/media-intelligence-deploy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync robust Parliament ingestion

Browse Source
This commit is contained in:
Admin 2026-09-29 06:29:40 +00:00
parent 0bfb5fcbc8
commit f5384f8437
1 changed files with 61 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
src/sources.ts
View File

@ -157,28 +157,47 @@ function parseParliamentDate(text: string): string | null {
async function fetchParliament(): Promise<NormalizedItem[]> {
const source = SOURCES.find((s) => s.id === "parliament")!;
const html = await getText(source.url, "text/html");
const response = await fetch(source.url, {
headers: {
accept: "text/html,application/xhtml+xml",
"accept-language": "en-NZ,en;q=0.9",
"user-agent": "Mozilla/5.0 (compatible; ArrowstackMediaIntelligence/0.1; +https://media.arrowstack.dev)"
},
redirect: "follow",
signal: AbortSignal.timeout(20000)
});
if (!response.ok) {
throw new Error(`${response.status} ${response.statusText} for ${source.url}`);
}
const html = await response.text();
const $ = load(html);
const seen = new Set<string>();
const items: NormalizedItem[] = [];
$('a[href*="/en/pb/sc/submissions-and-advice/document/"]').each((_index, element) => {
const addItem = (
href: string,
titleText: string,
surroundingText = "",
fallbackIndex = 0
) => {
if (items.length >= 30) return;
const href = $(element).attr("href");
const title = $(element).text().replace(/\s+/g, " ").trim();
const title = titleText.replace(/\s+/g, " ").trim();
if (!href || !title || title.length < 2) return;
const url = new URL(href, source.url).toString().split("?")[0];
if (!url.includes("/submissions-and-advice/document/")) return;
if (seen.has(url)) return;
seen.add(url);
const container = $(element).closest("tr").length
? $(element).closest("tr")
: $(element).closest("li, article, .list-item, div");
const surrounding = container.text().replace(/\s+/g, " ").trim();
const summary = surrounding.replace(title, "").replace(/View more View less/gi, "").trim().slice(0, 1400);
const surrounding = surroundingText.replace(/\s+/g, " ").trim();
const summary = (
surrounding
.replace(title, "")
.replace(/View more View less/gi, "")
.trim() ||
"Released evidence or submission listed by New Zealand Parliament."
).slice(0, 1400);
const committee =
surrounding.match(/Committee:\s*([^|]+?)(?:Date:|$)/i)?.[1]?.trim() ??
@ -189,7 +208,7 @@ async function fetchParliament(): Promise<NormalizedItem[]> {
items.push({
sourceId: "parliament",
externalId: url,
externalId: url || `parliament-${fallbackIndex}`,
title,
summary,
url,
@ -202,8 +221,38 @@ async function fetchParliament(): Promise<NormalizedItem[]> {
],
raw: { title, summary, url, committee }
});
};
$('a[href*="submissions-and-advice/document/"]').each((index, element) => {
const href = $(element).attr("href") ?? "";
const container = $(element).closest("tr").length
? $(element).closest("tr")
: $(element).closest("li, article, .list-item, div");
addItem(
href,
$(element).text(),
container.text(),
index
);
});
if (!items.length) {
const linkPattern = /<a\b[^>]*href=["']([^"']*submissions-and-advice\/document\/[^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi;
let match: RegExpExecArray | null;
let index = 0;
while ((match = linkPattern.exec(html)) !== null && items.length < 30) {
const title = load(`<div>${match[2]}</div>`)("div").text();
addItem(match[1], title, "", index++);
}
}
if (!items.length) {
const pageTitle = $("title").text().replace(/\s+/g, " ").trim();
throw new Error(
`Parliament evidence listing returned no document links (bytes=${html.length}, title=${pageTitle || "unknown"})`
);
}
return items;
}