From 8e989c01bb342ada80eee222b2ab33a0f93cf71d Mon Sep 17 00:00:00 2001 From: Admin Date: Tue, 29 Sep 2026 07:17:07 +0000 Subject: [PATCH] Sync expanded intelligence source network --- src/mcp.ts | 2 +- src/scoring.ts | 25 ++++++ src/sources.ts | 234 ++++++++++++++++++++++++++++++++++++++++++++++++- src/types.ts | 2 +- 4 files changed, 260 insertions(+), 3 deletions(-) diff --git a/src/mcp.ts b/src/mcp.ts index db0e52c..67acc92 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -31,7 +31,7 @@ const tools = [ description: "Run ingestion now for one source or all sources.", inputSchema: { type: "object", - properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury"] } }, + properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury", "rbnz", "mbie-jobs-online", "companies-office", "nz-gazette", "electricity-authority"] } }, additionalProperties: false } }, diff --git a/src/scoring.ts b/src/scoring.ts index a36fe98..950ea78 100644 --- a/src/scoring.ts +++ b/src/scoring.ts @@ -59,5 +59,30 @@ export function scoreItem(item: NormalizedItem): ScoredItem { reasons.push("New or updated Treasury publication"); } + if (item.sourceId === "rbnz") { + score += 1; + reasons.push("New or updated Reserve Bank statistical release"); + } + + if (item.sourceId === "mbie-jobs-online") { + score += 1; + reasons.push("New or updated MBIE labour-demand data"); + } + + if (item.sourceId === "companies-office") { + score += 1; + reasons.push("New or updated Companies Office business statistics"); + } + + if (item.sourceId === "nz-gazette") { + score += 1; + reasons.push("New official Gazette notice"); + } + + if (item.sourceId === "electricity-authority") { + score += 1; + reasons.push("New or updated Electricity Authority supply snapshot"); + } + return { score, reasons }; } diff --git a/src/sources.ts b/src/sources.ts index b3bd372..3eb4e74 100644 --- a/src/sources.ts +++ b/src/sources.ts @@ -32,6 +32,36 @@ export const SOURCES: SourceDefinition[] = [ name: "New Zealand Treasury publications", type: "rss", url: "https://www.treasury.govt.nz/feeds/publications" + }, + { + id: "rbnz", + name: "Reserve Bank of New Zealand statistics", + type: "html", + url: "https://www.rbnz.govt.nz/statistics-" + }, + { + id: "mbie-jobs-online", + name: "MBIE Jobs Online", + type: "html", + url: "https://www.mbie.govt.nz/business-and-employment/employment-and-skills/labour-market-reports-data-and-analysis/jobs-online" + }, + { + id: "companies-office", + name: "Companies Office company statistics", + type: "html", + url: "https://www.companiesoffice.govt.nz/insights-and-articles/latest-company-statistics/" + }, + { + id: "nz-gazette", + name: "New Zealand Gazette notices", + type: "html", + url: "https://gazette.govt.nz/home/search?from=0&sortField=publish_date&sortOrder=desc" + }, + { + id: "electricity-authority", + name: "Electricity Authority security of supply", + type: "html", + url: "https://www.ea.govt.nz/your-power/keeping-the-lights-on/electricity-security-of-supply/" } ]; @@ -60,7 +90,12 @@ export async function fetchSource(id: SourceId): Promise { if (id === "parliament") return fetchParliament(); if (id === "stats-nz") return fetchStatsNz(); if (id === "data-govt-nz") return fetchDataGovtNz(); - return fetchTreasury(); + if (id === "treasury") return fetchTreasury(); + if (id === "rbnz") return fetchRbnz(); + if (id === "mbie-jobs-online") return fetchMbieJobsOnline(); + if (id === "companies-office") return fetchCompaniesOffice(); + if (id === "nz-gazette") return fetchNzGazette(); + return fetchElectricityAuthority(); } async function fetchGeoNet(): Promise { @@ -502,3 +537,200 @@ async function fetchTreasury(): Promise { }; }).filter((item: NormalizedItem) => item.title && item.url); } + + +function pageText(html: string): string { + const $ = load(html); + $("script,style,noscript,svg").remove(); + return $("body").text().replace(/\s+/g, " ").trim(); +} + +function excerptAround(text: string, needle: string, length = 2200): string { + const lower = text.toLowerCase(); + const index = lower.indexOf(needle.toLowerCase()); + if (index < 0) return text.slice(0, length); + const start = Math.max(0, index - 250); + return text.slice(start, start + length); +} + +async function fetchRbnz(): Promise { + const source = SOURCES.find((s) => s.id === "rbnz")!; + const html = await getText(source.url, "text/html,application/xhtml+xml"); + const $ = load(html); + const seen = new Set(); + const items: NormalizedItem[] = []; + + $('a[href*="/statistics/series/"]').each((_index, element) => { + if (items.length >= 40) return; + const href = $(element).attr("href") ?? ""; + const title = $(element).text().replace(/\s+/g, " ").trim(); + if (!href || !title || title.length < 4) return; + + const url = new URL(href, source.url).toString().split("?")[0]; + if (seen.has(url)) return; + seen.add(url); + + const container = $(element).closest("article,li,tr,div"); + const context = container.text().replace(/\s+/g, " ").trim(); + const publishedAt = parseParliamentDate(context); + + items.push({ + sourceId: "rbnz", + externalId: url, + title, + summary: context.slice(0, 1400) || "Reserve Bank statistical series.", + url, + publishedAt, + topics: ["rbnz", "economy", "finance", "statistics", "new-zealand"], + raw: { title, url, context } + }); + }); + + if (!items.length) { + const text = pageText(html); + items.push({ + sourceId: "rbnz", + externalId: "rbnz-statistics-index", + title: "Reserve Bank statistics index", + summary: text.slice(0, 1800), + url: source.url, + publishedAt: null, + topics: ["rbnz", "economy", "finance", "statistics", "new-zealand"], + raw: { text: text.slice(0, 6000) } + }); + } + + return items; +} + +async function fetchMbieJobsOnline(): Promise { + const source = SOURCES.find((s) => s.id === "mbie-jobs-online")!; + const html = await getText(source.url, "text/html,application/xhtml+xml"); + const text = pageText(html); + const updated = + text.match(/Last updated:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ?? + text.match(/updated\s+(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ?? + ""; + + return [{ + sourceId: "mbie-jobs-online", + externalId: "jobs-online", + title: "MBIE Jobs Online labour demand", + summary: excerptAround(text, "Jobs Online monthly data release", 2600), + url: source.url, + publishedAt: parseParliamentDate(updated), + topics: ["employment", "jobs", "vacancies", "labour-market", "mbie", "new-zealand"], + raw: { updated, text: text.slice(0, 9000) } + }]; +} + +async function fetchCompaniesOffice(): Promise { + const source = SOURCES.find((s) => s.id === "companies-office")!; + const html = await getText(source.url, "text/html,application/xhtml+xml"); + const text = pageText(html); + const updated = + text.match(/Last updated\s+(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ?? + ""; + + return [{ + sourceId: "companies-office", + externalId: "latest-company-statistics", + title: "Companies Office latest company statistics", + summary: excerptAround(text, "Monthly data file", 3000), + url: source.url, + publishedAt: parseParliamentDate(updated), + topics: ["business", "companies", "insolvency", "liquidation", "registrations", "new-zealand"], + raw: { updated, text: text.slice(0, 10000) } + }]; +} + +async function fetchNzGazette(): Promise { + const source = SOURCES.find((s) => s.id === "nz-gazette")!; + const html = await getText(source.url, "text/html,application/xhtml+xml"); + const $ = load(html); + const seen = new Set(); + const items: NormalizedItem[] = []; + + $('a[href*="/notice/id/"]').each((_index, element) => { + if (items.length >= 50) return; + const href = $(element).attr("href") ?? ""; + const title = $(element).text().replace(/\s+/g, " ").trim(); + if (!href || !title || title.length < 4) return; + + const url = new URL(href, source.url).toString().split("?")[0]; + if (seen.has(url)) return; + seen.add(url); + + const row = $(element).closest("tr,li,article,div"); + const context = row.text().replace(/\s+/g, " ").trim(); + const noticeId = + context.match(/\b20\d{2}-[a-z]{2}\d{3,6}\b/i)?.[0] ?? + url.split("/").filter(Boolean).pop() ?? + url; + + items.push({ + sourceId: "nz-gazette", + externalId: String(noticeId), + title, + summary: context.slice(0, 1500), + url, + publishedAt: parseParliamentDate(context), + topics: ["gazette", "official-notice", "new-zealand"], + raw: { noticeId, title, context, url } + }); + }); + + if (!items.length) { + throw new Error(`NZ Gazette returned no notice links (bytes=${html.length})`); + } + + return items; +} + +async function fetchElectricityAuthority(): Promise { + const source = SOURCES.find((s) => s.id === "electricity-authority")!; + const html = await getText(source.url, "text/html,application/xhtml+xml"); + const $ = load(html); + const seen = new Set(); + const items: NormalizedItem[] = []; + + $("a").each((_index, element) => { + if (items.length >= 30) return; + const title = $(element).text().replace(/\s+/g, " ").trim(); + if (!/^Electricity security of supply snapshot\s*-/i.test(title)) return; + + const href = $(element).attr("href") ?? ""; + if (!href) return; + const url = new URL(href, source.url).toString().split("?")[0]; + if (seen.has(url)) return; + seen.add(url); + + const context = $(element).closest("li,article,div").text().replace(/\s+/g, " ").trim(); + items.push({ + sourceId: "electricity-authority", + externalId: url, + title, + summary: context.slice(0, 1600) || title, + url, + publishedAt: parseParliamentDate(title), + topics: ["electricity", "energy", "security-of-supply", "hydro", "prices", "new-zealand"], + raw: { title, context, url } + }); + }); + + if (!items.length) { + const text = pageText(html); + return [{ + sourceId: "electricity-authority", + externalId: "security-of-supply-index", + title: "Electricity Authority security of supply", + summary: excerptAround(text, "This week's snapshot", 2400), + url: source.url, + publishedAt: null, + topics: ["electricity", "energy", "security-of-supply", "new-zealand"], + raw: { text: text.slice(0, 8000) } + }]; + } + + return items; +} diff --git a/src/types.ts b/src/types.ts index bdbcf29..aa2d50e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,4 @@ -export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz" | "treasury"; +export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz" | "treasury" | "rbnz" | "mbie-jobs-online" | "companies-office" | "nz-gazette" | "electricity-authority"; export interface SourceDefinition { id: SourceId;