Sync Parliament Atom fallback
This commit is contained in:
parent
f5384f8437
commit
2459d72420
248
src/sources.ts
248
src/sources.ts
@ -155,107 +155,201 @@ function parseParliamentDate(text: string): string | null {
|
||||
return Number.isNaN(value.getTime()) ? null : value.toISOString();
|
||||
}
|
||||
|
||||
async function fetchParliament(): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === "parliament")!;
|
||||
const response = await fetch(source.url, {
|
||||
async function fetchParliamentAtom(source: SourceDefinition): Promise<NormalizedItem[]> {
|
||||
const feedUrl = "https://www3.parliament.nz/en/channellisting/rss/9343/e37f168d-4521-4437-a6fd-6bb7ff54e800";
|
||||
const response = await fetch(feedUrl, {
|
||||
headers: {
|
||||
accept: "text/html,application/xhtml+xml",
|
||||
"accept-language": "en-NZ,en;q=0.9",
|
||||
accept: "application/atom+xml,application/xml,text/xml;q=0.9,*/*;q=0.5",
|
||||
"user-agent": "Mozilla/5.0 (compatible; ArrowstackMediaIntelligence/0.1; +https://media.arrowstack.dev)"
|
||||
},
|
||||
redirect: "follow",
|
||||
signal: AbortSignal.timeout(20000)
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.status} ${response.statusText} for ${source.url}`);
|
||||
throw new Error(`Parliament Atom feed HTTP ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const $ = load(html);
|
||||
const seen = new Set<string>();
|
||||
const items: NormalizedItem[] = [];
|
||||
const xml = await response.text();
|
||||
if (/radware page/i.test(xml) || /<title>\s*radware page\s*<\/title>/i.test(xml)) {
|
||||
throw new Error(`Parliament Atom feed blocked by Radware (bytes=${xml.length})`);
|
||||
}
|
||||
|
||||
const addItem = (
|
||||
href: string,
|
||||
titleText: string,
|
||||
surroundingText = "",
|
||||
fallbackIndex = 0
|
||||
) => {
|
||||
if (items.length >= 30) return;
|
||||
const title = titleText.replace(/\s+/g, " ").trim();
|
||||
if (!href || !title || title.length < 2) return;
|
||||
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
|
||||
const parsed = parser.parse(xml);
|
||||
let list = asArray<any>(
|
||||
parsed?.feed?.entry ??
|
||||
parsed?.rss?.channel?.item ??
|
||||
parsed?.["atom:feed"]?.["atom:entry"]
|
||||
);
|
||||
if (!list.length) list = rawFeedItems(xml);
|
||||
|
||||
const url = new URL(href, source.url).toString().split("?")[0];
|
||||
if (!url.includes("/submissions-and-advice/document/")) return;
|
||||
if (seen.has(url)) return;
|
||||
seen.add(url);
|
||||
const items = list.slice(0, 30).map((entry: any, index: number) => {
|
||||
const title = cleanXmlText(
|
||||
entry.title?.["#text"] ??
|
||||
entry["atom:title"]?.["#text"] ??
|
||||
entry.title ??
|
||||
entry["atom:title"] ??
|
||||
"Parliament item"
|
||||
);
|
||||
const summary = cleanXmlText(
|
||||
entry.summary?.["#text"] ??
|
||||
entry["atom:summary"]?.["#text"] ??
|
||||
entry.summary ??
|
||||
entry["atom:summary"] ??
|
||||
entry.description ??
|
||||
entry.content ??
|
||||
""
|
||||
);
|
||||
const linkValue = entry.link ?? entry["atom:link"];
|
||||
const url = firstLink(linkValue) || feedUrl;
|
||||
const publishedAt =
|
||||
cleanXmlText(
|
||||
entry.updated ??
|
||||
entry["atom:updated"] ??
|
||||
entry.published ??
|
||||
entry["atom:published"] ??
|
||||
entry.pubDate ??
|
||||
""
|
||||
) || null;
|
||||
|
||||
const surrounding = surroundingText.replace(/\s+/g, " ").trim();
|
||||
const summary = (
|
||||
surrounding
|
||||
.replace(title, "")
|
||||
.replace(/View more View less/gi, "")
|
||||
.trim() ||
|
||||
"Released evidence or submission listed by New Zealand Parliament."
|
||||
).slice(0, 1400);
|
||||
|
||||
const committee =
|
||||
surrounding.match(/Committee:\s*([^|]+?)(?:Date:|$)/i)?.[1]?.trim() ??
|
||||
"";
|
||||
const dateText =
|
||||
surrounding.match(/Date:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
|
||||
surrounding;
|
||||
|
||||
items.push({
|
||||
sourceId: "parliament",
|
||||
externalId: url || `parliament-${fallbackIndex}`,
|
||||
return {
|
||||
sourceId: "parliament" as const,
|
||||
externalId: String(
|
||||
entry.id?.["#text"] ??
|
||||
entry["atom:id"]?.["#text"] ??
|
||||
entry.id ??
|
||||
entry["atom:id"] ??
|
||||
entry.guid ??
|
||||
url ??
|
||||
index
|
||||
),
|
||||
title,
|
||||
summary,
|
||||
url,
|
||||
publishedAt: parseParliamentDate(dateText),
|
||||
topics: [
|
||||
"parliament",
|
||||
"select-committee",
|
||||
"evidence",
|
||||
...(committee ? [committee.toLowerCase().replace(/\s+/g, "-")] : [])
|
||||
],
|
||||
raw: { title, summary, url, committee }
|
||||
});
|
||||
};
|
||||
|
||||
$('a[href*="submissions-and-advice/document/"]').each((index, element) => {
|
||||
const href = $(element).attr("href") ?? "";
|
||||
const container = $(element).closest("tr").length
|
||||
? $(element).closest("tr")
|
||||
: $(element).closest("li, article, .list-item, div");
|
||||
addItem(
|
||||
href,
|
||||
$(element).text(),
|
||||
container.text(),
|
||||
index
|
||||
);
|
||||
});
|
||||
publishedAt,
|
||||
topics: ["parliament", "select-committee", "evidence"],
|
||||
raw: entry
|
||||
};
|
||||
}).filter((item: NormalizedItem) => item.title && item.url);
|
||||
|
||||
if (!items.length) {
|
||||
const linkPattern = /<a\b[^>]*href=["']([^"']*submissions-and-advice\/document\/[^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
let index = 0;
|
||||
while ((match = linkPattern.exec(html)) !== null && items.length < 30) {
|
||||
const title = load(`<div>${match[2]}</div>`)("div").text();
|
||||
addItem(match[1], title, "", index++);
|
||||
}
|
||||
}
|
||||
|
||||
if (!items.length) {
|
||||
const pageTitle = $("title").text().replace(/\s+/g, " ").trim();
|
||||
throw new Error(
|
||||
`Parliament evidence listing returned no document links (bytes=${html.length}, title=${pageTitle || "unknown"})`
|
||||
`Parliament Atom feed contained no entries (bytes=${xml.length}, prefix=${xml.slice(0, 120).replace(/\s+/g, " ")})`
|
||||
);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
async function fetchParliament(): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === "parliament")!;
|
||||
let htmlFailure = "";
|
||||
|
||||
try {
|
||||
const response = await fetch(source.url, {
|
||||
headers: {
|
||||
accept: "text/html,application/xhtml+xml",
|
||||
"accept-language": "en-NZ,en;q=0.9",
|
||||
"user-agent": "Mozilla/5.0 (compatible; ArrowstackMediaIntelligence/0.1; +https://media.arrowstack.dev)"
|
||||
},
|
||||
redirect: "follow",
|
||||
signal: AbortSignal.timeout(20000)
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
if (/radware page/i.test(html) || /<title>\s*radware page\s*<\/title>/i.test(html)) {
|
||||
throw new Error(`blocked by Radware (bytes=${html.length})`);
|
||||
}
|
||||
|
||||
const $ = load(html);
|
||||
const seen = new Set<string>();
|
||||
const items: NormalizedItem[] = [];
|
||||
|
||||
const addItem = (
|
||||
href: string,
|
||||
titleText: string,
|
||||
surroundingText = "",
|
||||
fallbackIndex = 0
|
||||
) => {
|
||||
if (items.length >= 30) return;
|
||||
const title = titleText.replace(/\s+/g, " ").trim();
|
||||
if (!href || !title || title.length < 2) return;
|
||||
|
||||
const url = new URL(href, source.url).toString().split("?")[0];
|
||||
if (!url.includes("/submissions-and-advice/document/")) return;
|
||||
if (seen.has(url)) return;
|
||||
seen.add(url);
|
||||
|
||||
const surrounding = surroundingText.replace(/\s+/g, " ").trim();
|
||||
const summary = (
|
||||
surrounding
|
||||
.replace(title, "")
|
||||
.replace(/View more View less/gi, "")
|
||||
.trim() ||
|
||||
"Released evidence or submission listed by New Zealand Parliament."
|
||||
).slice(0, 1400);
|
||||
|
||||
const committee =
|
||||
surrounding.match(/Committee:\s*([^|]+?)(?:Date:|$)/i)?.[1]?.trim() ??
|
||||
"";
|
||||
const dateText =
|
||||
surrounding.match(/Date:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
|
||||
surrounding;
|
||||
|
||||
items.push({
|
||||
sourceId: "parliament",
|
||||
externalId: url || `parliament-${fallbackIndex}`,
|
||||
title,
|
||||
summary,
|
||||
url,
|
||||
publishedAt: parseParliamentDate(dateText),
|
||||
topics: [
|
||||
"parliament",
|
||||
"select-committee",
|
||||
"evidence",
|
||||
...(committee ? [committee.toLowerCase().replace(/\s+/g, "-")] : [])
|
||||
],
|
||||
raw: { title, summary, url, committee }
|
||||
});
|
||||
};
|
||||
|
||||
$('a[href*="submissions-and-advice/document/"]').each((index, element) => {
|
||||
const href = $(element).attr("href") ?? "";
|
||||
const container = $(element).closest("tr").length
|
||||
? $(element).closest("tr")
|
||||
: $(element).closest("li, article, .list-item, div");
|
||||
addItem(href, $(element).text(), container.text(), index);
|
||||
});
|
||||
|
||||
if (!items.length) {
|
||||
const linkPattern = /<a\b[^>]*href=["']([^"']*submissions-and-advice\/document\/[^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
let index = 0;
|
||||
while ((match = linkPattern.exec(html)) !== null && items.length < 30) {
|
||||
const title = load(`<div>${match[2]}</div>`)("div").text();
|
||||
addItem(match[1], title, "", index++);
|
||||
}
|
||||
}
|
||||
|
||||
if (items.length) return items;
|
||||
htmlFailure = `HTML listing contained no document links (bytes=${html.length})`;
|
||||
} catch (error) {
|
||||
htmlFailure = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
try {
|
||||
return await fetchParliamentAtom(source);
|
||||
} catch (error) {
|
||||
const atomFailure = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(
|
||||
`Parliament unavailable from Arrowstack. HTML: ${htmlFailure || "unknown failure"}; Atom: ${atomFailure}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
interface SitemapEntry {
|
||||
loc: string;
|
||||
lastmod: string | null;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user