media-intelligence-deploy/src/sources.ts

import { load } from "cheerio";
import { XMLParser } from "fast-xml-parser";
import type { NormalizedItem, SourceDefinition, SourceId } from "./types.js";

export const SOURCES: SourceDefinition[] = [
  {
    id: "geonet",
    name: "GeoNet earthquakes",
    type: "geojson",
    url: "https://api.geonet.org.nz/quake?MMI=3"
  },
  {
    id: "parliament",
    name: "NZ Parliament evidence and submissions",
    type: "atom",
    url: "https://www3.parliament.nz/en/channellisting/rss/9343/e37f168d-4521-4437-a6fd-6bb7ff54e800"
  },
  {
    id: "stats-nz",
    name: "Stats NZ information releases",
    type: "html",
    url: "https://www.stats.govt.nz/information-releases/"
  },
  {
    id: "data-govt-nz",
    name: "data.govt.nz catalogue",
    type: "ckan",
    url: "https://catalogue.data.govt.nz/api/3/action/package_search?sort=metadata_modified%20desc&rows=25"
  }
];

const UA = "Arrowstack-Media-Intelligence/0.1 (+https://media.arrowstack.dev)";

async function getText(url: string, accept = "*/*") {
  const response = await fetch(url, {
    headers: { accept, "user-agent": UA },
    signal: AbortSignal.timeout(20000)
  });
  if (!response.ok) throw new Error(`${response.status} ${response.statusText} for ${url}`);
  return response.text();
}

async function getJson<T>(url: string, accept = "application/json") {
  const response = await fetch(url, {
    headers: { accept, "user-agent": UA },
    signal: AbortSignal.timeout(20000)
  });
  if (!response.ok) throw new Error(`${response.status} ${response.statusText} for ${url}`);
  return response.json() as Promise<T>;
}

export async function fetchSource(id: SourceId): Promise<NormalizedItem[]> {
  if (id === "geonet") return fetchGeoNet();
  if (id === "parliament") return fetchParliament();
  if (id === "stats-nz") return fetchStatsNz();
  return fetchDataGovtNz();
}

async function fetchGeoNet(): Promise<NormalizedItem[]> {
  const source = SOURCES.find((s) => s.id === "geonet")!;
  const data = await getJson<any>(source.url, "application/vnd.geo+json;version=2");
  return (data.features ?? []).slice(0, 30).map((feature: any) => {
    const p = feature.properties ?? {};
    const coords = feature.geometry?.coordinates ?? [];
    return {
      sourceId: "geonet" as const,
      externalId: String(p.publicID ?? feature.id),
      title: `M${p.magnitude ?? "?"} earthquake — ${p.locality ?? "New Zealand"}`,
      summary: `Magnitude ${p.magnitude ?? "unknown"}, depth ${p.depth ?? "unknown"} km, MMI ${p.mmi ?? "unknown"}, quality ${p.quality ?? "unknown"}.`,
      url: p.publicID ? `https://www.geonet.org.nz/earthquake/${p.publicID}` : source.url,
      publishedAt: p.time ?? null,
      topics: ["natural-hazards", "earthquake"],
      metrics: {
        magnitude: Number(p.magnitude ?? 0),
        mmi: Number(p.mmi ?? 0),
        depthKm: Number(p.depth ?? 0),
        longitude: coords[0] ?? null,
        latitude: coords[1] ?? null
      },
      raw: feature
    };
  });
}

function firstLink(value: any): string {
  if (Array.isArray(value)) {
    const alt = value.find((v) => v?.["@_rel"] === "alternate") ?? value[0];
    return alt?.["@_href"] ?? "";
  }
  return value?.["@_href"] ?? "";
}

async function fetchParliament(): Promise<NormalizedItem[]> {
  const source = SOURCES.find((s) => s.id === "parliament")!;
  const xml = await getText(source.url, "application/atom+xml, application/xml, text/xml");
  const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
  const parsed = parser.parse(xml);
  const entries = parsed?.feed?.entry ?? parsed?.rss?.channel?.item ?? [];
  const list = Array.isArray(entries) ? entries : [entries];
  return list.slice(0, 30).map((entry: any, index: number) => {
    const title = String(entry.title?.["#text"] ?? entry.title ?? "Parliament item").trim();
    const summary = String(entry.summary?.["#text"] ?? entry.summary ?? entry.description ?? "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
    const url = firstLink(entry.link) || String(entry.link ?? source.url);
    return {
      sourceId: "parliament" as const,
      externalId: String(entry.id ?? entry.guid ?? url ?? index),
      title,
      summary,
      url,
      publishedAt: String(entry.updated ?? entry.published ?? entry.pubDate ?? "") || null,
      topics: ["parliament", "public-policy"],
      raw: entry
    };
  });
}

function parseDate(text: string): string | null {
  const match = text.match(/\b(\d{1,2})\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+(20\d{2})\b/i);
  if (!match) return null;
  const d = new Date(`${match[1]} ${match[2]} ${match[3]} 12:00:00 +1200`);
  return Number.isNaN(d.getTime()) ? null : d.toISOString();
}

async function fetchStatsNz(): Promise<NormalizedItem[]> {
  const source = SOURCES.find((s) => s.id === "stats-nz")!;
  const html = await getText(source.url, "text/html");
  const $ = load(html);
  const seen = new Set<string>();
  const items: NormalizedItem[] = [];
  $('a[href*="/information-releases/"]').each((_i, el) => {
    if (items.length >= 30) return;
    const href = $(el).attr("href");
    const title = $(el).text().replace(/\s+/g, " ").trim();
    if (!href || !title || title.length < 8) return;
    const url = new URL(href, source.url).toString().split("?")[0];
    if (seen.has(url)) return;
    seen.add(url);
    const container = $(el).closest("article, li, div");
    const surrounding = container.text().replace(/\s+/g, " ").trim();
    const summary = surrounding.replace(title, "").trim().slice(0, 900);
    items.push({
      sourceId: "stats-nz",
      externalId: url,
      title,
      summary,
      url,
      publishedAt: parseDate(surrounding),
      topics: ["statistics", "new-zealand"],
      raw: { title, summary, url }
    });
  });
  return items;
}

async function fetchDataGovtNz(): Promise<NormalizedItem[]> {
  const source = SOURCES.find((s) => s.id === "data-govt-nz")!;
  const data = await getJson<any>(source.url);
  if (!data.success) throw new Error("data.govt.nz CKAN returned success=false");
  return (data.result?.results ?? []).slice(0, 30).map((pkg: any) => ({
    sourceId: "data-govt-nz" as const,
    externalId: String(pkg.id ?? pkg.name),
    title: String(pkg.title ?? pkg.name ?? "Dataset"),
    summary: String(pkg.notes ?? "").replace(/\s+/g, " ").trim().slice(0, 1200),
    url: pkg.url || `https://catalogue.data.govt.nz/dataset/${pkg.name}`,
    publishedAt: pkg.metadata_modified ?? pkg.metadata_created ?? null,
    topics: ["open-data", ...(pkg.tags ?? []).slice(0, 6).map((tag: any) => String(tag.display_name ?? tag.name).toLowerCase())],
    raw: pkg
  }));
}