Sync direct RBNZ statistical file ingestion

src/sources.ts

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { load } from "cheerio";
 import { XMLParser } from "fast-xml-parser";
 import type { NormalizedItem, SourceDefinition, SourceId } from "./types.js";
@@ -37,7 +38,7 @@ export const SOURCES: SourceDefinition[] = [
     id: "rbnz",
     name: "Reserve Bank of New Zealand statistics",
     type: "html",
-    url: "https://www.rbnz.govt.nz/statistics-"
+    url: "https://www.rbnz.govt.nz/statistics/series/data-file-index-page"
   },
   {
     id: "mbie-jobs-online",
@@ -553,50 +554,105 @@ function excerptAround(text: string, needle: string, length = 2200): string {
   return text.slice(start, start + length);
 }
 
+const RBNZ_SERIES_FILES = [
+  {
+    code: "B1",
+    title: "Exchange rates and Trade Weighted Index",
+    url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/b/b1/hb1-daily.xlsx",
+    topics: ["exchange-rates", "trade", "finance"]
+  },
+  {
+    code: "B2",
+    title: "Wholesale interest rates",
+    url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/b/b2/hb2-daily-close.xlsx",
+    topics: ["interest-rates", "finance"]
+  },
+  {
+    code: "B3",
+    title: "Retail interest rates on lending and deposits",
+    url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/b/b3/hb3.xlsx",
+    topics: ["interest-rates", "lending", "deposits", "households"]
+  },
+  {
+    code: "C5",
+    title: "Sector lending",
+    url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/c/c5/hc5.xlsx",
+    topics: ["credit", "lending", "business", "households"]
+  },
+  {
+    code: "C21",
+    title: "Key household financial statistics",
+    url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/c/c21/hc21.xlsx",
+    topics: ["households", "finance", "debt"]
+  },
+  {
+    code: "C60",
+    title: "Credit conditions",
+    url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/c/c60/hc60.xlsx",
+    topics: ["credit", "business", "lending"]
+  },
+  {
+    code: "M9",
+    title: "Labour market",
+    url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/m/m9/hm9.xlsx",
+    topics: ["employment", "unemployment", "labour-market"]
+  },
+  {
+    code: "M10",
+    title: "Housing",
+    url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/m/m10/hm10.xlsx",
+    topics: ["housing", "mortgages", "households"]
+  },
+  {
+    code: "M14",
+    title: "Survey of expectations",
+    url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/m/m14/hm14.xlsx",
+    topics: ["inflation", "expectations", "economy"]
+  },
+  {
+    code: "S50",
+    title: "Banks: loans by asset quality",
+    url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/l-s/s50/hs50.xlsx",
+    topics: ["banks", "credit-quality", "lending", "business"]
+  }
+] as const;
+
 async function fetchRbnz(): Promise<NormalizedItem[]> {
-  const source = SOURCES.find((s) => s.id === "rbnz")!;
-  const html = await getText(source.url, "text/html,application/xhtml+xml");
-  const $ = load(html);
-  const seen = new Set<string>();
   const items: NormalizedItem[] = [];
 
-  $('a[href*="/statistics/series/"]').each((_index, element) => {
-    if (items.length >= 40) return;
-    const href = $(element).attr("href") ?? "";
-    const title = $(element).text().replace(/\s+/g, " ").trim();
-    if (!href || !title || title.length < 4) return;
+  for (const series of RBNZ_SERIES_FILES) {
+    const response = await fetch(series.url, {
+      headers: {
+        accept: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,*/*;q=0.5",
+        "user-agent": UA
+      },
+      redirect: "follow",
+      signal: AbortSignal.timeout(20000)
+    });
 
-    const url = new URL(href, source.url).toString().split("?")[0];
-    if (seen.has(url)) return;
-    seen.add(url);
+    if (!response.ok) {
+      throw new Error(`RBNZ ${series.code} file HTTP ${response.status} ${response.statusText}`);
+    }
 
-    const container = $(element).closest("article,li,tr,div");
-    const context = container.text().replace(/\s+/g, " ").trim();
-    const publishedAt = parseParliamentDate(context);
+    const bytes = Buffer.from(await response.arrayBuffer());
+    const fileHash = createHash("sha256").update(bytes).digest("hex");
+    const modified = response.headers.get("last-modified");
 
     items.push({
       sourceId: "rbnz",
-      externalId: url,
-      title,
-      summary: context.slice(0, 1400) || "Reserve Bank statistical series.",
-      url,
-      publishedAt,
-      topics: ["rbnz", "economy", "finance", "statistics", "new-zealand"],
-      raw: { title, url, context }
-    });
-  });
-
-  if (!items.length) {
-    const text = pageText(html);
-    items.push({
-      sourceId: "rbnz",
-      externalId: "rbnz-statistics-index",
-      title: "Reserve Bank statistics index",
-      summary: text.slice(0, 1800),
-      url: source.url,
-      publishedAt: null,
-      topics: ["rbnz", "economy", "finance", "statistics", "new-zealand"],
-      raw: { text: text.slice(0, 6000) }
+      externalId: series.code,
+      title: `RBNZ ${series.code}: ${series.title}`,
+      summary: `Reserve Bank statistical data file ${series.code} (${series.title}); file size ${bytes.length} bytes.`,
+      url: series.url,
+      publishedAt: modified ? new Date(modified).toISOString() : null,
+      topics: ["rbnz", "statistics", "new-zealand", ...series.topics],
+      raw: {
+        code: series.code,
+        title: series.title,
+        fileHash,
+        bytes: bytes.length,
+        lastModified: modified
+      }
     });
   }