Sync hardened NZ source adapters
This commit is contained in:
parent
81ffcec7f4
commit
f163f21104
180
src/sources.ts
180
src/sources.ts
@ -1,4 +1,3 @@
|
||||
import { load } from "cheerio";
|
||||
import { XMLParser } from "fast-xml-parser";
|
||||
import type { NormalizedItem, SourceDefinition, SourceId } from "./types.js";
|
||||
|
||||
@ -18,8 +17,8 @@ export const SOURCES: SourceDefinition[] = [
|
||||
{
|
||||
id: "stats-nz",
|
||||
name: "Stats NZ information releases",
|
||||
type: "html",
|
||||
url: "https://www.stats.govt.nz/information-releases/"
|
||||
type: "sitemap",
|
||||
url: "https://www.stats.govt.nz/sitemap.xml"
|
||||
},
|
||||
{
|
||||
id: "data-govt-nz",
|
||||
@ -82,12 +81,61 @@ async function fetchGeoNet(): Promise<NormalizedItem[]> {
|
||||
});
|
||||
}
|
||||
|
||||
function asArray<T>(value: T | T[] | undefined | null): T[] {
|
||||
if (value == null) return [];
|
||||
return Array.isArray(value) ? value : [value];
|
||||
}
|
||||
|
||||
function firstLink(value: any): string {
|
||||
if (Array.isArray(value)) {
|
||||
const alt = value.find((v) => v?.["@_rel"] === "alternate") ?? value[0];
|
||||
return alt?.["@_href"] ?? "";
|
||||
return alt?.["@_href"] ?? alt?.["#text"] ?? "";
|
||||
}
|
||||
return value?.["@_href"] ?? "";
|
||||
if (typeof value === "string") return value;
|
||||
return value?.["@_href"] ?? value?.["#text"] ?? "";
|
||||
}
|
||||
|
||||
function cleanXmlText(value: unknown): string {
|
||||
return String(value ?? "")
|
||||
.replace(/^<!\[CDATA\[/, "")
|
||||
.replace(/\]\]>$/, "")
|
||||
.replace(/<[^>]+>/g, " ")
|
||||
.replace(/&/gi, "&")
|
||||
.replace(/"/gi, '"')
|
||||
.replace(/'|'/gi, "'")
|
||||
.replace(/</gi, "<")
|
||||
.replace(/>/gi, ">")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function xmlTag(block: string, tag: string): string {
|
||||
const pattern = new RegExp(
|
||||
`<(?:[\\w-]+:)?${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/(?:[\\w-]+:)?${tag}>`,
|
||||
"i"
|
||||
);
|
||||
return cleanXmlText(block.match(pattern)?.[1] ?? "");
|
||||
}
|
||||
|
||||
function rawFeedItems(xml: string): any[] {
|
||||
const blocks = [
|
||||
...xml.matchAll(/<(?:[\w-]+:)?entry\b[\s\S]*?<\/(?:[\w-]+:)?entry>/gi),
|
||||
...xml.matchAll(/<(?:[\w-]+:)?item\b[\s\S]*?<\/(?:[\w-]+:)?item>/gi)
|
||||
].map((match) => match[0]);
|
||||
|
||||
return blocks.map((block, index) => {
|
||||
const atomHref =
|
||||
block.match(/<(?:[\w-]+:)?link\b[^>]*\bhref=["']([^"']+)["'][^>]*\/?\s*>/i)?.[1] ?? "";
|
||||
const rssLink = xmlTag(block, "link");
|
||||
const url = cleanXmlText(atomHref || rssLink);
|
||||
return {
|
||||
id: xmlTag(block, "id") || xmlTag(block, "guid") || url || String(index),
|
||||
title: xmlTag(block, "title") || "Parliament item",
|
||||
summary: xmlTag(block, "summary") || xmlTag(block, "description") || xmlTag(block, "content"),
|
||||
link: url,
|
||||
updated: xmlTag(block, "updated") || xmlTag(block, "published") || xmlTag(block, "pubDate")
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchParliament(): Promise<NormalizedItem[]> {
|
||||
@ -95,61 +143,102 @@ async function fetchParliament(): Promise<NormalizedItem[]> {
|
||||
const xml = await getText(source.url, "application/atom+xml, application/xml, text/xml");
|
||||
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
|
||||
const parsed = parser.parse(xml);
|
||||
const entries = parsed?.feed?.entry ?? parsed?.rss?.channel?.item ?? [];
|
||||
const list = Array.isArray(entries) ? entries : [entries];
|
||||
|
||||
let list = asArray<any>(parsed?.feed?.entry ?? parsed?.rss?.channel?.item);
|
||||
if (!list.length) list = rawFeedItems(xml);
|
||||
|
||||
return list.slice(0, 30).map((entry: any, index: number) => {
|
||||
const title = String(entry.title?.["#text"] ?? entry.title ?? "Parliament item").trim();
|
||||
const summary = String(entry.summary?.["#text"] ?? entry.summary ?? entry.description ?? "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
||||
const title = cleanXmlText(entry.title?.["#text"] ?? entry.title ?? "Parliament item");
|
||||
const summary = cleanXmlText(
|
||||
entry.summary?.["#text"] ?? entry.summary ?? entry.description ?? entry.content ?? ""
|
||||
);
|
||||
const url = firstLink(entry.link) || String(entry.link ?? source.url);
|
||||
return {
|
||||
sourceId: "parliament" as const,
|
||||
externalId: String(entry.id ?? entry.guid ?? url ?? index),
|
||||
externalId: String(entry.id?.["#text"] ?? entry.id ?? entry.guid ?? url ?? index),
|
||||
title,
|
||||
summary,
|
||||
url,
|
||||
publishedAt: String(entry.updated ?? entry.published ?? entry.pubDate ?? "") || null,
|
||||
publishedAt:
|
||||
cleanXmlText(entry.updated ?? entry.published ?? entry.pubDate ?? "") || null,
|
||||
topics: ["parliament", "public-policy"],
|
||||
raw: entry
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function parseDate(text: string): string | null {
|
||||
const match = text.match(/\b(\d{1,2})\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+(20\d{2})\b/i);
|
||||
if (!match) return null;
|
||||
const d = new Date(`${match[1]} ${match[2]} ${match[3]} 12:00:00 +1200`);
|
||||
return Number.isNaN(d.getTime()) ? null : d.toISOString();
|
||||
interface SitemapEntry {
|
||||
loc: string;
|
||||
lastmod: string | null;
|
||||
}
|
||||
|
||||
async function collectSitemap(url: string, depth = 0): Promise<SitemapEntry[]> {
|
||||
const xml = await getText(url, "application/xml, text/xml");
|
||||
const parser = new XMLParser({ ignoreAttributes: false });
|
||||
const parsed = parser.parse(xml);
|
||||
|
||||
const urls = asArray<any>(parsed?.urlset?.url)
|
||||
.map((entry) => ({
|
||||
loc: cleanXmlText(entry?.loc),
|
||||
lastmod: cleanXmlText(entry?.lastmod) || null
|
||||
}))
|
||||
.filter((entry) => entry.loc);
|
||||
|
||||
if (urls.length) return urls;
|
||||
if (depth >= 1) return [];
|
||||
|
||||
const children = asArray<any>(parsed?.sitemapindex?.sitemap)
|
||||
.map((entry) => cleanXmlText(entry?.loc))
|
||||
.filter(Boolean)
|
||||
.slice(0, 30);
|
||||
|
||||
const batches = await Promise.all(
|
||||
children.map(async (child) => {
|
||||
try {
|
||||
return await collectSitemap(child, depth + 1);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
})
|
||||
);
|
||||
return batches.flat();
|
||||
}
|
||||
|
||||
function titleFromReleaseUrl(url: string): string {
|
||||
const path = new URL(url).pathname.split("/").filter(Boolean);
|
||||
const slug = decodeURIComponent(path[path.length - 1] ?? "information-release");
|
||||
const title = slug
|
||||
.split("-")
|
||||
.filter(Boolean)
|
||||
.map((part) => (part.length <= 3 ? part.toUpperCase() : part))
|
||||
.join(" ");
|
||||
return title.charAt(0).toUpperCase() + title.slice(1);
|
||||
}
|
||||
|
||||
async function fetchStatsNz(): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === "stats-nz")!;
|
||||
const html = await getText(source.url, "text/html");
|
||||
const $ = load(html);
|
||||
const seen = new Set<string>();
|
||||
const items: NormalizedItem[] = [];
|
||||
$('a[href*="/information-releases/"]').each((_i, el) => {
|
||||
if (items.length >= 30) return;
|
||||
const href = $(el).attr("href");
|
||||
const title = $(el).text().replace(/\s+/g, " ").trim();
|
||||
if (!href || !title || title.length < 8) return;
|
||||
const url = new URL(href, source.url).toString().split("?")[0];
|
||||
if (seen.has(url)) return;
|
||||
seen.add(url);
|
||||
const container = $(el).closest("article, li, div");
|
||||
const surrounding = container.text().replace(/\s+/g, " ").trim();
|
||||
const summary = surrounding.replace(title, "").trim().slice(0, 900);
|
||||
items.push({
|
||||
sourceId: "stats-nz",
|
||||
externalId: url,
|
||||
title,
|
||||
summary,
|
||||
url,
|
||||
publishedAt: parseDate(surrounding),
|
||||
const sitemapEntries = await collectSitemap(source.url);
|
||||
const releases = sitemapEntries
|
||||
.filter((entry) => {
|
||||
try {
|
||||
return new URL(entry.loc).pathname.toLowerCase().startsWith("/information-releases/");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
.sort((a, b) => String(b.lastmod ?? "").localeCompare(String(a.lastmod ?? "")))
|
||||
.slice(0, 30);
|
||||
|
||||
return releases.map((entry) => ({
|
||||
sourceId: "stats-nz" as const,
|
||||
externalId: entry.loc,
|
||||
title: titleFromReleaseUrl(entry.loc),
|
||||
summary: "Stats NZ information release detected or updated in the official site sitemap.",
|
||||
url: entry.loc,
|
||||
publishedAt: entry.lastmod,
|
||||
topics: ["statistics", "new-zealand"],
|
||||
raw: { title, summary, url }
|
||||
});
|
||||
});
|
||||
return items;
|
||||
raw: entry
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchDataGovtNz(): Promise<NormalizedItem[]> {
|
||||
@ -163,7 +252,12 @@ async function fetchDataGovtNz(): Promise<NormalizedItem[]> {
|
||||
summary: String(pkg.notes ?? "").replace(/\s+/g, " ").trim().slice(0, 1200),
|
||||
url: pkg.url || `https://catalogue.data.govt.nz/dataset/${pkg.name}`,
|
||||
publishedAt: pkg.metadata_modified ?? pkg.metadata_created ?? null,
|
||||
topics: ["open-data", ...(pkg.tags ?? []).slice(0, 6).map((tag: any) => String(tag.display_name ?? tag.name).toLowerCase())],
|
||||
topics: [
|
||||
"open-data",
|
||||
...(pkg.tags ?? []).slice(0, 6).map((tag: any) =>
|
||||
String(tag.display_name ?? tag.name).toLowerCase()
|
||||
)
|
||||
],
|
||||
raw: pkg
|
||||
}));
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@ export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz";
|
||||
export interface SourceDefinition {
|
||||
id: SourceId;
|
||||
name: string;
|
||||
type: "geojson" | "atom" | "html" | "ckan";
|
||||
type: "geojson" | "atom" | "sitemap" | "ckan";
|
||||
url: string;
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user