diff --git a/src/sources.ts b/src/sources.ts index 3eb4e74..acea49d 100644 --- a/src/sources.ts +++ b/src/sources.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { load } from "cheerio"; import { XMLParser } from "fast-xml-parser"; import type { NormalizedItem, SourceDefinition, SourceId } from "./types.js"; @@ -37,7 +38,7 @@ export const SOURCES: SourceDefinition[] = [ id: "rbnz", name: "Reserve Bank of New Zealand statistics", type: "html", - url: "https://www.rbnz.govt.nz/statistics-" + url: "https://www.rbnz.govt.nz/statistics/series/data-file-index-page" }, { id: "mbie-jobs-online", @@ -553,50 +554,105 @@ function excerptAround(text: string, needle: string, length = 2200): string { return text.slice(start, start + length); } +const RBNZ_SERIES_FILES = [ + { + code: "B1", + title: "Exchange rates and Trade Weighted Index", + url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/b/b1/hb1-daily.xlsx", + topics: ["exchange-rates", "trade", "finance"] + }, + { + code: "B2", + title: "Wholesale interest rates", + url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/b/b2/hb2-daily-close.xlsx", + topics: ["interest-rates", "finance"] + }, + { + code: "B3", + title: "Retail interest rates on lending and deposits", + url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/b/b3/hb3.xlsx", + topics: ["interest-rates", "lending", "deposits", "households"] + }, + { + code: "C5", + title: "Sector lending", + url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/c/c5/hc5.xlsx", + topics: ["credit", "lending", "business", "households"] + }, + { + code: "C21", + title: "Key household financial statistics", + url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/c/c21/hc21.xlsx", + topics: ["households", "finance", "debt"] + }, + { + code: "C60", + title: "Credit conditions", + url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/c/c60/hc60.xlsx", + topics: ["credit", "business", "lending"] + }, + { + code: "M9", + title: "Labour market", + url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/m/m9/hm9.xlsx", + topics: ["employment", "unemployment", "labour-market"] + }, + { + code: "M10", + title: "Housing", + url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/m/m10/hm10.xlsx", + topics: ["housing", "mortgages", "households"] + }, + { + code: "M14", + title: "Survey of expectations", + url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/m/m14/hm14.xlsx", + topics: ["inflation", "expectations", "economy"] + }, + { + code: "S50", + title: "Banks: loans by asset quality", + url: "https://rbnz.govt.nz/-/media/project/sites/rbnz/files/statistics/series/l-s/s50/hs50.xlsx", + topics: ["banks", "credit-quality", "lending", "business"] + } +] as const; + async function fetchRbnz(): Promise { - const source = SOURCES.find((s) => s.id === "rbnz")!; - const html = await getText(source.url, "text/html,application/xhtml+xml"); - const $ = load(html); - const seen = new Set(); const items: NormalizedItem[] = []; - $('a[href*="/statistics/series/"]').each((_index, element) => { - if (items.length >= 40) return; - const href = $(element).attr("href") ?? ""; - const title = $(element).text().replace(/\s+/g, " ").trim(); - if (!href || !title || title.length < 4) return; - - const url = new URL(href, source.url).toString().split("?")[0]; - if (seen.has(url)) return; - seen.add(url); - - const container = $(element).closest("article,li,tr,div"); - const context = container.text().replace(/\s+/g, " ").trim(); - const publishedAt = parseParliamentDate(context); - - items.push({ - sourceId: "rbnz", - externalId: url, - title, - summary: context.slice(0, 1400) || "Reserve Bank statistical series.", - url, - publishedAt, - topics: ["rbnz", "economy", "finance", "statistics", "new-zealand"], - raw: { title, url, context } + for (const series of RBNZ_SERIES_FILES) { + const response = await fetch(series.url, { + headers: { + accept: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,*/*;q=0.5", + "user-agent": UA + }, + redirect: "follow", + signal: AbortSignal.timeout(20000) }); - }); - if (!items.length) { - const text = pageText(html); + if (!response.ok) { + throw new Error(`RBNZ ${series.code} file HTTP ${response.status} ${response.statusText}`); + } + + const bytes = Buffer.from(await response.arrayBuffer()); + const fileHash = createHash("sha256").update(bytes).digest("hex"); + const modified = response.headers.get("last-modified"); + items.push({ sourceId: "rbnz", - externalId: "rbnz-statistics-index", - title: "Reserve Bank statistics index", - summary: text.slice(0, 1800), - url: source.url, - publishedAt: null, - topics: ["rbnz", "economy", "finance", "statistics", "new-zealand"], - raw: { text: text.slice(0, 6000) } + externalId: series.code, + title: `RBNZ ${series.code}: ${series.title}`, + summary: `Reserve Bank statistical data file ${series.code} (${series.title}); file size ${bytes.length} bytes.`, + url: series.url, + publishedAt: modified ? new Date(modified).toISOString() : null, + topics: ["rbnz", "statistics", "new-zealand", ...series.topics], + raw: { + code: series.code, + title: series.title, + fileHash, + bytes: bytes.length, + lastModified: modified + } }); }