Sync expanded intelligence source network

src/mcp.ts

@@ -31,7 +31,7 @@ const tools = [
     description: "Run ingestion now for one source or all sources.",
     inputSchema: {
       type: "object",
-      properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury"] } },
+      properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury", "rbnz", "mbie-jobs-online", "companies-office", "nz-gazette", "electricity-authority"] } },
       additionalProperties: false
     }
   },

src/scoring.ts

@@ -59,5 +59,30 @@ export function scoreItem(item: NormalizedItem): ScoredItem {
     reasons.push("New or updated Treasury publication");
   }
 
+  if (item.sourceId === "rbnz") {
+    score += 1;
+    reasons.push("New or updated Reserve Bank statistical release");
+  }
+
+  if (item.sourceId === "mbie-jobs-online") {
+    score += 1;
+    reasons.push("New or updated MBIE labour-demand data");
+  }
+
+  if (item.sourceId === "companies-office") {
+    score += 1;
+    reasons.push("New or updated Companies Office business statistics");
+  }
+
+  if (item.sourceId === "nz-gazette") {
+    score += 1;
+    reasons.push("New official Gazette notice");
+  }
+
+  if (item.sourceId === "electricity-authority") {
+    score += 1;
+    reasons.push("New or updated Electricity Authority supply snapshot");
+  }
+
   return { score, reasons };
 }

src/sources.ts

@@ -32,6 +32,36 @@ export const SOURCES: SourceDefinition[] = [
     name: "New Zealand Treasury publications",
     type: "rss",
     url: "https://www.treasury.govt.nz/feeds/publications"
+  },
+  {
+    id: "rbnz",
+    name: "Reserve Bank of New Zealand statistics",
+    type: "html",
+    url: "https://www.rbnz.govt.nz/statistics-"
+  },
+  {
+    id: "mbie-jobs-online",
+    name: "MBIE Jobs Online",
+    type: "html",
+    url: "https://www.mbie.govt.nz/business-and-employment/employment-and-skills/labour-market-reports-data-and-analysis/jobs-online"
+  },
+  {
+    id: "companies-office",
+    name: "Companies Office company statistics",
+    type: "html",
+    url: "https://www.companiesoffice.govt.nz/insights-and-articles/latest-company-statistics/"
+  },
+  {
+    id: "nz-gazette",
+    name: "New Zealand Gazette notices",
+    type: "html",
+    url: "https://gazette.govt.nz/home/search?from=0&sortField=publish_date&sortOrder=desc"
+  },
+  {
+    id: "electricity-authority",
+    name: "Electricity Authority security of supply",
+    type: "html",
+    url: "https://www.ea.govt.nz/your-power/keeping-the-lights-on/electricity-security-of-supply/"
   }
 ];
 
@@ -60,7 +90,12 @@ export async function fetchSource(id: SourceId): Promise<NormalizedItem[]> {
   if (id === "parliament") return fetchParliament();
   if (id === "stats-nz") return fetchStatsNz();
   if (id === "data-govt-nz") return fetchDataGovtNz();
-  return fetchTreasury();
+  if (id === "treasury") return fetchTreasury();
+  if (id === "rbnz") return fetchRbnz();
+  if (id === "mbie-jobs-online") return fetchMbieJobsOnline();
+  if (id === "companies-office") return fetchCompaniesOffice();
+  if (id === "nz-gazette") return fetchNzGazette();
+  return fetchElectricityAuthority();
 }
 
 async function fetchGeoNet(): Promise<NormalizedItem[]> {
@@ -502,3 +537,200 @@ async function fetchTreasury(): Promise<NormalizedItem[]> {
     };
   }).filter((item: NormalizedItem) => item.title && item.url);
 }
+
+
+function pageText(html: string): string {
+  const $ = load(html);
+  $("script,style,noscript,svg").remove();
+  return $("body").text().replace(/\s+/g, " ").trim();
+}
+
+function excerptAround(text: string, needle: string, length = 2200): string {
+  const lower = text.toLowerCase();
+  const index = lower.indexOf(needle.toLowerCase());
+  if (index < 0) return text.slice(0, length);
+  const start = Math.max(0, index - 250);
+  return text.slice(start, start + length);
+}
+
+async function fetchRbnz(): Promise<NormalizedItem[]> {
+  const source = SOURCES.find((s) => s.id === "rbnz")!;
+  const html = await getText(source.url, "text/html,application/xhtml+xml");
+  const $ = load(html);
+  const seen = new Set<string>();
+  const items: NormalizedItem[] = [];
+
+  $('a[href*="/statistics/series/"]').each((_index, element) => {
+    if (items.length >= 40) return;
+    const href = $(element).attr("href") ?? "";
+    const title = $(element).text().replace(/\s+/g, " ").trim();
+    if (!href || !title || title.length < 4) return;
+
+    const url = new URL(href, source.url).toString().split("?")[0];
+    if (seen.has(url)) return;
+    seen.add(url);
+
+    const container = $(element).closest("article,li,tr,div");
+    const context = container.text().replace(/\s+/g, " ").trim();
+    const publishedAt = parseParliamentDate(context);
+
+    items.push({
+      sourceId: "rbnz",
+      externalId: url,
+      title,
+      summary: context.slice(0, 1400) || "Reserve Bank statistical series.",
+      url,
+      publishedAt,
+      topics: ["rbnz", "economy", "finance", "statistics", "new-zealand"],
+      raw: { title, url, context }
+    });
+  });
+
+  if (!items.length) {
+    const text = pageText(html);
+    items.push({
+      sourceId: "rbnz",
+      externalId: "rbnz-statistics-index",
+      title: "Reserve Bank statistics index",
+      summary: text.slice(0, 1800),
+      url: source.url,
+      publishedAt: null,
+      topics: ["rbnz", "economy", "finance", "statistics", "new-zealand"],
+      raw: { text: text.slice(0, 6000) }
+    });
+  }
+
+  return items;
+}
+
+async function fetchMbieJobsOnline(): Promise<NormalizedItem[]> {
+  const source = SOURCES.find((s) => s.id === "mbie-jobs-online")!;
+  const html = await getText(source.url, "text/html,application/xhtml+xml");
+  const text = pageText(html);
+  const updated =
+    text.match(/Last updated:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
+    text.match(/updated\s+(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
+    "";
+
+  return [{
+    sourceId: "mbie-jobs-online",
+    externalId: "jobs-online",
+    title: "MBIE Jobs Online labour demand",
+    summary: excerptAround(text, "Jobs Online monthly data release", 2600),
+    url: source.url,
+    publishedAt: parseParliamentDate(updated),
+    topics: ["employment", "jobs", "vacancies", "labour-market", "mbie", "new-zealand"],
+    raw: { updated, text: text.slice(0, 9000) }
+  }];
+}
+
+async function fetchCompaniesOffice(): Promise<NormalizedItem[]> {
+  const source = SOURCES.find((s) => s.id === "companies-office")!;
+  const html = await getText(source.url, "text/html,application/xhtml+xml");
+  const text = pageText(html);
+  const updated =
+    text.match(/Last updated\s+(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
+    "";
+
+  return [{
+    sourceId: "companies-office",
+    externalId: "latest-company-statistics",
+    title: "Companies Office latest company statistics",
+    summary: excerptAround(text, "Monthly data file", 3000),
+    url: source.url,
+    publishedAt: parseParliamentDate(updated),
+    topics: ["business", "companies", "insolvency", "liquidation", "registrations", "new-zealand"],
+    raw: { updated, text: text.slice(0, 10000) }
+  }];
+}
+
+async function fetchNzGazette(): Promise<NormalizedItem[]> {
+  const source = SOURCES.find((s) => s.id === "nz-gazette")!;
+  const html = await getText(source.url, "text/html,application/xhtml+xml");
+  const $ = load(html);
+  const seen = new Set<string>();
+  const items: NormalizedItem[] = [];
+
+  $('a[href*="/notice/id/"]').each((_index, element) => {
+    if (items.length >= 50) return;
+    const href = $(element).attr("href") ?? "";
+    const title = $(element).text().replace(/\s+/g, " ").trim();
+    if (!href || !title || title.length < 4) return;
+
+    const url = new URL(href, source.url).toString().split("?")[0];
+    if (seen.has(url)) return;
+    seen.add(url);
+
+    const row = $(element).closest("tr,li,article,div");
+    const context = row.text().replace(/\s+/g, " ").trim();
+    const noticeId =
+      context.match(/\b20\d{2}-[a-z]{2}\d{3,6}\b/i)?.[0] ??
+      url.split("/").filter(Boolean).pop() ??
+      url;
+
+    items.push({
+      sourceId: "nz-gazette",
+      externalId: String(noticeId),
+      title,
+      summary: context.slice(0, 1500),
+      url,
+      publishedAt: parseParliamentDate(context),
+      topics: ["gazette", "official-notice", "new-zealand"],
+      raw: { noticeId, title, context, url }
+    });
+  });
+
+  if (!items.length) {
+    throw new Error(`NZ Gazette returned no notice links (bytes=${html.length})`);
+  }
+
+  return items;
+}
+
+async function fetchElectricityAuthority(): Promise<NormalizedItem[]> {
+  const source = SOURCES.find((s) => s.id === "electricity-authority")!;
+  const html = await getText(source.url, "text/html,application/xhtml+xml");
+  const $ = load(html);
+  const seen = new Set<string>();
+  const items: NormalizedItem[] = [];
+
+  $("a").each((_index, element) => {
+    if (items.length >= 30) return;
+    const title = $(element).text().replace(/\s+/g, " ").trim();
+    if (!/^Electricity security of supply snapshot\s*-/i.test(title)) return;
+
+    const href = $(element).attr("href") ?? "";
+    if (!href) return;
+    const url = new URL(href, source.url).toString().split("?")[0];
+    if (seen.has(url)) return;
+    seen.add(url);
+
+    const context = $(element).closest("li,article,div").text().replace(/\s+/g, " ").trim();
+    items.push({
+      sourceId: "electricity-authority",
+      externalId: url,
+      title,
+      summary: context.slice(0, 1600) || title,
+      url,
+      publishedAt: parseParliamentDate(title),
+      topics: ["electricity", "energy", "security-of-supply", "hydro", "prices", "new-zealand"],
+      raw: { title, context, url }
+    });
+  });
+
+  if (!items.length) {
+    const text = pageText(html);
+    return [{
+      sourceId: "electricity-authority",
+      externalId: "security-of-supply-index",
+      title: "Electricity Authority security of supply",
+      summary: excerptAround(text, "This week's snapshot", 2400),
+      url: source.url,
+      publishedAt: null,
+      topics: ["electricity", "energy", "security-of-supply", "new-zealand"],
+      raw: { text: text.slice(0, 8000) }
+    }];
+  }
+
+  return items;
+}

src/types.ts

@@ -1,4 +1,4 @@
-export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz" | "treasury";
+export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz" | "treasury" | "rbnz" | "mbie-jobs-online" | "companies-office" | "nz-gazette" | "electricity-authority";
 
 export interface SourceDefinition {
   id: SourceId;