From f163f21104248d2550132050658aef8c47eac435 Mon Sep 17 00:00:00 2001 From: Admin Date: Tue, 29 Sep 2026 05:54:53 +0000 Subject: [PATCH] Sync hardened NZ source adapters --- src/sources.ts | 182 +++++++++++++++++++++++++++++++++++++------------ src/types.ts | 2 +- 2 files changed, 139 insertions(+), 45 deletions(-) diff --git a/src/sources.ts b/src/sources.ts index 6b995ba..50ad23a 100644 --- a/src/sources.ts +++ b/src/sources.ts @@ -1,4 +1,3 @@ -import { load } from "cheerio"; import { XMLParser } from "fast-xml-parser"; import type { NormalizedItem, SourceDefinition, SourceId } from "./types.js"; @@ -18,8 +17,8 @@ export const SOURCES: SourceDefinition[] = [ { id: "stats-nz", name: "Stats NZ information releases", - type: "html", - url: "https://www.stats.govt.nz/information-releases/" + type: "sitemap", + url: "https://www.stats.govt.nz/sitemap.xml" }, { id: "data-govt-nz", @@ -82,12 +81,61 @@ async function fetchGeoNet(): Promise { }); } +function asArray(value: T | T[] | undefined | null): T[] { + if (value == null) return []; + return Array.isArray(value) ? value : [value]; +} + function firstLink(value: any): string { if (Array.isArray(value)) { const alt = value.find((v) => v?.["@_rel"] === "alternate") ?? value[0]; - return alt?.["@_href"] ?? ""; + return alt?.["@_href"] ?? alt?.["#text"] ?? ""; } - return value?.["@_href"] ?? ""; + if (typeof value === "string") return value; + return value?.["@_href"] ?? value?.["#text"] ?? ""; +} + +function cleanXmlText(value: unknown): string { + return String(value ?? "") + .replace(/^$/, "") + .replace(/<[^>]+>/g, " ") + .replace(/&/gi, "&") + .replace(/"/gi, '"') + .replace(/'|'/gi, "'") + .replace(/</gi, "<") + .replace(/>/gi, ">") + .replace(/\s+/g, " ") + .trim(); +} + +function xmlTag(block: string, tag: string): string { + const pattern = new RegExp( + `<(?:[\\w-]+:)?${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/(?:[\\w-]+:)?${tag}>`, + "i" + ); + return cleanXmlText(block.match(pattern)?.[1] ?? ""); +} + +function rawFeedItems(xml: string): any[] { + const blocks = [ + ...xml.matchAll(/<(?:[\w-]+:)?entry\b[\s\S]*?<\/(?:[\w-]+:)?entry>/gi), + ...xml.matchAll(/<(?:[\w-]+:)?item\b[\s\S]*?<\/(?:[\w-]+:)?item>/gi) + ].map((match) => match[0]); + + return blocks.map((block, index) => { + const atomHref = + block.match(/<(?:[\w-]+:)?link\b[^>]*\bhref=["']([^"']+)["'][^>]*\/?\s*>/i)?.[1] ?? ""; + const rssLink = xmlTag(block, "link"); + const url = cleanXmlText(atomHref || rssLink); + return { + id: xmlTag(block, "id") || xmlTag(block, "guid") || url || String(index), + title: xmlTag(block, "title") || "Parliament item", + summary: xmlTag(block, "summary") || xmlTag(block, "description") || xmlTag(block, "content"), + link: url, + updated: xmlTag(block, "updated") || xmlTag(block, "published") || xmlTag(block, "pubDate") + }; + }); } async function fetchParliament(): Promise { @@ -95,61 +143,102 @@ async function fetchParliament(): Promise { const xml = await getText(source.url, "application/atom+xml, application/xml, text/xml"); const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" }); const parsed = parser.parse(xml); - const entries = parsed?.feed?.entry ?? parsed?.rss?.channel?.item ?? []; - const list = Array.isArray(entries) ? entries : [entries]; + + let list = asArray(parsed?.feed?.entry ?? parsed?.rss?.channel?.item); + if (!list.length) list = rawFeedItems(xml); + return list.slice(0, 30).map((entry: any, index: number) => { - const title = String(entry.title?.["#text"] ?? entry.title ?? "Parliament item").trim(); - const summary = String(entry.summary?.["#text"] ?? entry.summary ?? entry.description ?? "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim(); + const title = cleanXmlText(entry.title?.["#text"] ?? entry.title ?? "Parliament item"); + const summary = cleanXmlText( + entry.summary?.["#text"] ?? entry.summary ?? entry.description ?? entry.content ?? "" + ); const url = firstLink(entry.link) || String(entry.link ?? source.url); return { sourceId: "parliament" as const, - externalId: String(entry.id ?? entry.guid ?? url ?? index), + externalId: String(entry.id?.["#text"] ?? entry.id ?? entry.guid ?? url ?? index), title, summary, url, - publishedAt: String(entry.updated ?? entry.published ?? entry.pubDate ?? "") || null, + publishedAt: + cleanXmlText(entry.updated ?? entry.published ?? entry.pubDate ?? "") || null, topics: ["parliament", "public-policy"], raw: entry }; }); } -function parseDate(text: string): string | null { - const match = text.match(/\b(\d{1,2})\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+(20\d{2})\b/i); - if (!match) return null; - const d = new Date(`${match[1]} ${match[2]} ${match[3]} 12:00:00 +1200`); - return Number.isNaN(d.getTime()) ? null : d.toISOString(); +interface SitemapEntry { + loc: string; + lastmod: string | null; +} + +async function collectSitemap(url: string, depth = 0): Promise { + const xml = await getText(url, "application/xml, text/xml"); + const parser = new XMLParser({ ignoreAttributes: false }); + const parsed = parser.parse(xml); + + const urls = asArray(parsed?.urlset?.url) + .map((entry) => ({ + loc: cleanXmlText(entry?.loc), + lastmod: cleanXmlText(entry?.lastmod) || null + })) + .filter((entry) => entry.loc); + + if (urls.length) return urls; + if (depth >= 1) return []; + + const children = asArray(parsed?.sitemapindex?.sitemap) + .map((entry) => cleanXmlText(entry?.loc)) + .filter(Boolean) + .slice(0, 30); + + const batches = await Promise.all( + children.map(async (child) => { + try { + return await collectSitemap(child, depth + 1); + } catch { + return []; + } + }) + ); + return batches.flat(); +} + +function titleFromReleaseUrl(url: string): string { + const path = new URL(url).pathname.split("/").filter(Boolean); + const slug = decodeURIComponent(path[path.length - 1] ?? "information-release"); + const title = slug + .split("-") + .filter(Boolean) + .map((part) => (part.length <= 3 ? part.toUpperCase() : part)) + .join(" "); + return title.charAt(0).toUpperCase() + title.slice(1); } async function fetchStatsNz(): Promise { const source = SOURCES.find((s) => s.id === "stats-nz")!; - const html = await getText(source.url, "text/html"); - const $ = load(html); - const seen = new Set(); - const items: NormalizedItem[] = []; - $('a[href*="/information-releases/"]').each((_i, el) => { - if (items.length >= 30) return; - const href = $(el).attr("href"); - const title = $(el).text().replace(/\s+/g, " ").trim(); - if (!href || !title || title.length < 8) return; - const url = new URL(href, source.url).toString().split("?")[0]; - if (seen.has(url)) return; - seen.add(url); - const container = $(el).closest("article, li, div"); - const surrounding = container.text().replace(/\s+/g, " ").trim(); - const summary = surrounding.replace(title, "").trim().slice(0, 900); - items.push({ - sourceId: "stats-nz", - externalId: url, - title, - summary, - url, - publishedAt: parseDate(surrounding), - topics: ["statistics", "new-zealand"], - raw: { title, summary, url } - }); - }); - return items; + const sitemapEntries = await collectSitemap(source.url); + const releases = sitemapEntries + .filter((entry) => { + try { + return new URL(entry.loc).pathname.toLowerCase().startsWith("/information-releases/"); + } catch { + return false; + } + }) + .sort((a, b) => String(b.lastmod ?? "").localeCompare(String(a.lastmod ?? ""))) + .slice(0, 30); + + return releases.map((entry) => ({ + sourceId: "stats-nz" as const, + externalId: entry.loc, + title: titleFromReleaseUrl(entry.loc), + summary: "Stats NZ information release detected or updated in the official site sitemap.", + url: entry.loc, + publishedAt: entry.lastmod, + topics: ["statistics", "new-zealand"], + raw: entry + })); } async function fetchDataGovtNz(): Promise { @@ -163,7 +252,12 @@ async function fetchDataGovtNz(): Promise { summary: String(pkg.notes ?? "").replace(/\s+/g, " ").trim().slice(0, 1200), url: pkg.url || `https://catalogue.data.govt.nz/dataset/${pkg.name}`, publishedAt: pkg.metadata_modified ?? pkg.metadata_created ?? null, - topics: ["open-data", ...(pkg.tags ?? []).slice(0, 6).map((tag: any) => String(tag.display_name ?? tag.name).toLowerCase())], + topics: [ + "open-data", + ...(pkg.tags ?? []).slice(0, 6).map((tag: any) => + String(tag.display_name ?? tag.name).toLowerCase() + ) + ], raw: pkg })); } diff --git a/src/types.ts b/src/types.ts index 4bddb1d..5359cc0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -3,7 +3,7 @@ export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz"; export interface SourceDefinition { id: SourceId; name: string; - type: "geojson" | "atom" | "html" | "ckan"; + type: "geojson" | "atom" | "sitemap" | "ckan"; url: string; }