Sync hardened NZ source adapters

src/sources.ts

@@ -1,4 +1,3 @@
-import { load } from "cheerio";
 import { XMLParser } from "fast-xml-parser";
 import type { NormalizedItem, SourceDefinition, SourceId } from "./types.js";
 
@@ -18,8 +17,8 @@ export const SOURCES: SourceDefinition[] = [
   {
     id: "stats-nz",
     name: "Stats NZ information releases",
-    type: "html",
+    type: "sitemap",
-    url: "https://www.stats.govt.nz/information-releases/"
+    url: "https://www.stats.govt.nz/sitemap.xml"
   },
   {
     id: "data-govt-nz",
@@ -82,12 +81,61 @@ async function fetchGeoNet(): Promise<NormalizedItem[]> {
   });
 }
 
+function asArray<T>(value: T | T[] | undefined | null): T[] {
+  if (value == null) return [];
+  return Array.isArray(value) ? value : [value];
+}
+
 function firstLink(value: any): string {
   if (Array.isArray(value)) {
     const alt = value.find((v) => v?.["@_rel"] === "alternate") ?? value[0];
-    return alt?.["@_href"] ?? "";
+    return alt?.["@_href"] ?? alt?.["#text"] ?? "";
   }
-  return value?.["@_href"] ?? "";
+  if (typeof value === "string") return value;
+  return value?.["@_href"] ?? value?.["#text"] ?? "";
+}
+
+function cleanXmlText(value: unknown): string {
+  return String(value ?? "")
+    .replace(/^<!\[CDATA\[/, "")
+    .replace(/\]\]>$/, "")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&amp;/gi, "&")
+    .replace(/&quot;/gi, '"')
+    .replace(/&#39;|&apos;/gi, "'")
+    .replace(/&lt;/gi, "<")
+    .replace(/&gt;/gi, ">")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function xmlTag(block: string, tag: string): string {
+  const pattern = new RegExp(
+    `<(?:[\\w-]+:)?${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/(?:[\\w-]+:)?${tag}>`,
+    "i"
+  );
+  return cleanXmlText(block.match(pattern)?.[1] ?? "");
+}
+
+function rawFeedItems(xml: string): any[] {
+  const blocks = [
+    ...xml.matchAll(/<(?:[\w-]+:)?entry\b[\s\S]*?<\/(?:[\w-]+:)?entry>/gi),
+    ...xml.matchAll(/<(?:[\w-]+:)?item\b[\s\S]*?<\/(?:[\w-]+:)?item>/gi)
+  ].map((match) => match[0]);
+
+  return blocks.map((block, index) => {
+    const atomHref =
+      block.match(/<(?:[\w-]+:)?link\b[^>]*\bhref=["']([^"']+)["'][^>]*\/?\s*>/i)?.[1] ?? "";
+    const rssLink = xmlTag(block, "link");
+    const url = cleanXmlText(atomHref || rssLink);
+    return {
+      id: xmlTag(block, "id") || xmlTag(block, "guid") || url || String(index),
+      title: xmlTag(block, "title") || "Parliament item",
+      summary: xmlTag(block, "summary") || xmlTag(block, "description") || xmlTag(block, "content"),
+      link: url,
+      updated: xmlTag(block, "updated") || xmlTag(block, "published") || xmlTag(block, "pubDate")
+    };
+  });
 }
 
 async function fetchParliament(): Promise<NormalizedItem[]> {
@@ -95,61 +143,102 @@ async function fetchParliament(): Promise<NormalizedItem[]> {
   const xml = await getText(source.url, "application/atom+xml, application/xml, text/xml");
   const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
   const parsed = parser.parse(xml);
-  const entries = parsed?.feed?.entry ?? parsed?.rss?.channel?.item ?? [];
+
-  const list = Array.isArray(entries) ? entries : [entries];
+  let list = asArray<any>(parsed?.feed?.entry ?? parsed?.rss?.channel?.item);
+  if (!list.length) list = rawFeedItems(xml);
+
   return list.slice(0, 30).map((entry: any, index: number) => {
-    const title = String(entry.title?.["#text"] ?? entry.title ?? "Parliament item").trim();
+    const title = cleanXmlText(entry.title?.["#text"] ?? entry.title ?? "Parliament item");
-    const summary = String(entry.summary?.["#text"] ?? entry.summary ?? entry.description ?? "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+    const summary = cleanXmlText(
+      entry.summary?.["#text"] ?? entry.summary ?? entry.description ?? entry.content ?? ""
+    );
     const url = firstLink(entry.link) || String(entry.link ?? source.url);
     return {
       sourceId: "parliament" as const,
-      externalId: String(entry.id ?? entry.guid ?? url ?? index),
+      externalId: String(entry.id?.["#text"] ?? entry.id ?? entry.guid ?? url ?? index),
       title,
       summary,
       url,
-      publishedAt: String(entry.updated ?? entry.published ?? entry.pubDate ?? "") || null,
+      publishedAt:
+        cleanXmlText(entry.updated ?? entry.published ?? entry.pubDate ?? "") || null,
       topics: ["parliament", "public-policy"],
       raw: entry
     };
   });
 }
 
-function parseDate(text: string): string | null {
+interface SitemapEntry {
-  const match = text.match(/\b(\d{1,2})\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+(20\d{2})\b/i);
+  loc: string;
-  if (!match) return null;
+  lastmod: string | null;
-  const d = new Date(`${match[1]} ${match[2]} ${match[3]} 12:00:00 +1200`);
+}
-  return Number.isNaN(d.getTime()) ? null : d.toISOString();
+
+async function collectSitemap(url: string, depth = 0): Promise<SitemapEntry[]> {
+  const xml = await getText(url, "application/xml, text/xml");
+  const parser = new XMLParser({ ignoreAttributes: false });
+  const parsed = parser.parse(xml);
+
+  const urls = asArray<any>(parsed?.urlset?.url)
+    .map((entry) => ({
+      loc: cleanXmlText(entry?.loc),
+      lastmod: cleanXmlText(entry?.lastmod) || null
+    }))
+    .filter((entry) => entry.loc);
+
+  if (urls.length) return urls;
+  if (depth >= 1) return [];
+
+  const children = asArray<any>(parsed?.sitemapindex?.sitemap)
+    .map((entry) => cleanXmlText(entry?.loc))
+    .filter(Boolean)
+    .slice(0, 30);
+
+  const batches = await Promise.all(
+    children.map(async (child) => {
+      try {
+        return await collectSitemap(child, depth + 1);
+      } catch {
+        return [];
+      }
+    })
+  );
+  return batches.flat();
+}
+
+function titleFromReleaseUrl(url: string): string {
+  const path = new URL(url).pathname.split("/").filter(Boolean);
+  const slug = decodeURIComponent(path[path.length - 1] ?? "information-release");
+  const title = slug
+    .split("-")
+    .filter(Boolean)
+    .map((part) => (part.length <= 3 ? part.toUpperCase() : part))
+    .join(" ");
+  return title.charAt(0).toUpperCase() + title.slice(1);
 }
 
 async function fetchStatsNz(): Promise<NormalizedItem[]> {
   const source = SOURCES.find((s) => s.id === "stats-nz")!;
-  const html = await getText(source.url, "text/html");
+  const sitemapEntries = await collectSitemap(source.url);
-  const $ = load(html);
+  const releases = sitemapEntries
-  const seen = new Set<string>();
+    .filter((entry) => {
-  const items: NormalizedItem[] = [];
+      try {
-  $('a[href*="/information-releases/"]').each((_i, el) => {
+        return new URL(entry.loc).pathname.toLowerCase().startsWith("/information-releases/");
-    if (items.length >= 30) return;
+      } catch {
-    const href = $(el).attr("href");
+        return false;
-    const title = $(el).text().replace(/\s+/g, " ").trim();
+      }
-    if (!href || !title || title.length < 8) return;
+    })
-    const url = new URL(href, source.url).toString().split("?")[0];
+    .sort((a, b) => String(b.lastmod ?? "").localeCompare(String(a.lastmod ?? "")))
-    if (seen.has(url)) return;
+    .slice(0, 30);
-    seen.add(url);
+
-    const container = $(el).closest("article, li, div");
+  return releases.map((entry) => ({
-    const surrounding = container.text().replace(/\s+/g, " ").trim();
+    sourceId: "stats-nz" as const,
-    const summary = surrounding.replace(title, "").trim().slice(0, 900);
+    externalId: entry.loc,
-    items.push({
+    title: titleFromReleaseUrl(entry.loc),
-      sourceId: "stats-nz",
+    summary: "Stats NZ information release detected or updated in the official site sitemap.",
-      externalId: url,
+    url: entry.loc,
-      title,
+    publishedAt: entry.lastmod,
-      summary,
+    topics: ["statistics", "new-zealand"],
-      url,
+    raw: entry
-      publishedAt: parseDate(surrounding),
+  }));
-      topics: ["statistics", "new-zealand"],
-      raw: { title, summary, url }
-    });
-  });
-  return items;
 }
 
 async function fetchDataGovtNz(): Promise<NormalizedItem[]> {
@@ -163,7 +252,12 @@ async function fetchDataGovtNz(): Promise<NormalizedItem[]> {
     summary: String(pkg.notes ?? "").replace(/\s+/g, " ").trim().slice(0, 1200),
     url: pkg.url || `https://catalogue.data.govt.nz/dataset/${pkg.name}`,
     publishedAt: pkg.metadata_modified ?? pkg.metadata_created ?? null,
-    topics: ["open-data", ...(pkg.tags ?? []).slice(0, 6).map((tag: any) => String(tag.display_name ?? tag.name).toLowerCase())],
+    topics: [
+      "open-data",
+      ...(pkg.tags ?? []).slice(0, 6).map((tag: any) =>
+        String(tag.display_name ?? tag.name).toLowerCase()
+      )
+    ],
     raw: pkg
   }));
 }

src/types.ts

@@ -3,7 +3,7 @@ export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz";
 export interface SourceDefinition {
   id: SourceId;
   name: string;
-  type: "geojson" | "atom" | "html" | "ckan";
+  type: "geojson" | "atom" | "sitemap" | "ckan";
   url: string;
 }