Sync expanded intelligence source network
This commit is contained in:
parent
4adef5c58b
commit
8e989c01bb
@ -31,7 +31,7 @@ const tools = [
|
||||
description: "Run ingestion now for one source or all sources.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury"] } },
|
||||
properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury", "rbnz", "mbie-jobs-online", "companies-office", "nz-gazette", "electricity-authority"] } },
|
||||
additionalProperties: false
|
||||
}
|
||||
},
|
||||
|
||||
@ -59,5 +59,30 @@ export function scoreItem(item: NormalizedItem): ScoredItem {
|
||||
reasons.push("New or updated Treasury publication");
|
||||
}
|
||||
|
||||
if (item.sourceId === "rbnz") {
|
||||
score += 1;
|
||||
reasons.push("New or updated Reserve Bank statistical release");
|
||||
}
|
||||
|
||||
if (item.sourceId === "mbie-jobs-online") {
|
||||
score += 1;
|
||||
reasons.push("New or updated MBIE labour-demand data");
|
||||
}
|
||||
|
||||
if (item.sourceId === "companies-office") {
|
||||
score += 1;
|
||||
reasons.push("New or updated Companies Office business statistics");
|
||||
}
|
||||
|
||||
if (item.sourceId === "nz-gazette") {
|
||||
score += 1;
|
||||
reasons.push("New official Gazette notice");
|
||||
}
|
||||
|
||||
if (item.sourceId === "electricity-authority") {
|
||||
score += 1;
|
||||
reasons.push("New or updated Electricity Authority supply snapshot");
|
||||
}
|
||||
|
||||
return { score, reasons };
|
||||
}
|
||||
|
||||
234
src/sources.ts
234
src/sources.ts
@ -32,6 +32,36 @@ export const SOURCES: SourceDefinition[] = [
|
||||
name: "New Zealand Treasury publications",
|
||||
type: "rss",
|
||||
url: "https://www.treasury.govt.nz/feeds/publications"
|
||||
},
|
||||
{
|
||||
id: "rbnz",
|
||||
name: "Reserve Bank of New Zealand statistics",
|
||||
type: "html",
|
||||
url: "https://www.rbnz.govt.nz/statistics-"
|
||||
},
|
||||
{
|
||||
id: "mbie-jobs-online",
|
||||
name: "MBIE Jobs Online",
|
||||
type: "html",
|
||||
url: "https://www.mbie.govt.nz/business-and-employment/employment-and-skills/labour-market-reports-data-and-analysis/jobs-online"
|
||||
},
|
||||
{
|
||||
id: "companies-office",
|
||||
name: "Companies Office company statistics",
|
||||
type: "html",
|
||||
url: "https://www.companiesoffice.govt.nz/insights-and-articles/latest-company-statistics/"
|
||||
},
|
||||
{
|
||||
id: "nz-gazette",
|
||||
name: "New Zealand Gazette notices",
|
||||
type: "html",
|
||||
url: "https://gazette.govt.nz/home/search?from=0&sortField=publish_date&sortOrder=desc"
|
||||
},
|
||||
{
|
||||
id: "electricity-authority",
|
||||
name: "Electricity Authority security of supply",
|
||||
type: "html",
|
||||
url: "https://www.ea.govt.nz/your-power/keeping-the-lights-on/electricity-security-of-supply/"
|
||||
}
|
||||
];
|
||||
|
||||
@ -60,7 +90,12 @@ export async function fetchSource(id: SourceId): Promise<NormalizedItem[]> {
|
||||
if (id === "parliament") return fetchParliament();
|
||||
if (id === "stats-nz") return fetchStatsNz();
|
||||
if (id === "data-govt-nz") return fetchDataGovtNz();
|
||||
return fetchTreasury();
|
||||
if (id === "treasury") return fetchTreasury();
|
||||
if (id === "rbnz") return fetchRbnz();
|
||||
if (id === "mbie-jobs-online") return fetchMbieJobsOnline();
|
||||
if (id === "companies-office") return fetchCompaniesOffice();
|
||||
if (id === "nz-gazette") return fetchNzGazette();
|
||||
return fetchElectricityAuthority();
|
||||
}
|
||||
|
||||
async function fetchGeoNet(): Promise<NormalizedItem[]> {
|
||||
@ -502,3 +537,200 @@ async function fetchTreasury(): Promise<NormalizedItem[]> {
|
||||
};
|
||||
}).filter((item: NormalizedItem) => item.title && item.url);
|
||||
}
|
||||
|
||||
|
||||
function pageText(html: string): string {
|
||||
const $ = load(html);
|
||||
$("script,style,noscript,svg").remove();
|
||||
return $("body").text().replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function excerptAround(text: string, needle: string, length = 2200): string {
|
||||
const lower = text.toLowerCase();
|
||||
const index = lower.indexOf(needle.toLowerCase());
|
||||
if (index < 0) return text.slice(0, length);
|
||||
const start = Math.max(0, index - 250);
|
||||
return text.slice(start, start + length);
|
||||
}
|
||||
|
||||
async function fetchRbnz(): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === "rbnz")!;
|
||||
const html = await getText(source.url, "text/html,application/xhtml+xml");
|
||||
const $ = load(html);
|
||||
const seen = new Set<string>();
|
||||
const items: NormalizedItem[] = [];
|
||||
|
||||
$('a[href*="/statistics/series/"]').each((_index, element) => {
|
||||
if (items.length >= 40) return;
|
||||
const href = $(element).attr("href") ?? "";
|
||||
const title = $(element).text().replace(/\s+/g, " ").trim();
|
||||
if (!href || !title || title.length < 4) return;
|
||||
|
||||
const url = new URL(href, source.url).toString().split("?")[0];
|
||||
if (seen.has(url)) return;
|
||||
seen.add(url);
|
||||
|
||||
const container = $(element).closest("article,li,tr,div");
|
||||
const context = container.text().replace(/\s+/g, " ").trim();
|
||||
const publishedAt = parseParliamentDate(context);
|
||||
|
||||
items.push({
|
||||
sourceId: "rbnz",
|
||||
externalId: url,
|
||||
title,
|
||||
summary: context.slice(0, 1400) || "Reserve Bank statistical series.",
|
||||
url,
|
||||
publishedAt,
|
||||
topics: ["rbnz", "economy", "finance", "statistics", "new-zealand"],
|
||||
raw: { title, url, context }
|
||||
});
|
||||
});
|
||||
|
||||
if (!items.length) {
|
||||
const text = pageText(html);
|
||||
items.push({
|
||||
sourceId: "rbnz",
|
||||
externalId: "rbnz-statistics-index",
|
||||
title: "Reserve Bank statistics index",
|
||||
summary: text.slice(0, 1800),
|
||||
url: source.url,
|
||||
publishedAt: null,
|
||||
topics: ["rbnz", "economy", "finance", "statistics", "new-zealand"],
|
||||
raw: { text: text.slice(0, 6000) }
|
||||
});
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
async function fetchMbieJobsOnline(): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === "mbie-jobs-online")!;
|
||||
const html = await getText(source.url, "text/html,application/xhtml+xml");
|
||||
const text = pageText(html);
|
||||
const updated =
|
||||
text.match(/Last updated:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
|
||||
text.match(/updated\s+(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
|
||||
"";
|
||||
|
||||
return [{
|
||||
sourceId: "mbie-jobs-online",
|
||||
externalId: "jobs-online",
|
||||
title: "MBIE Jobs Online labour demand",
|
||||
summary: excerptAround(text, "Jobs Online monthly data release", 2600),
|
||||
url: source.url,
|
||||
publishedAt: parseParliamentDate(updated),
|
||||
topics: ["employment", "jobs", "vacancies", "labour-market", "mbie", "new-zealand"],
|
||||
raw: { updated, text: text.slice(0, 9000) }
|
||||
}];
|
||||
}
|
||||
|
||||
async function fetchCompaniesOffice(): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === "companies-office")!;
|
||||
const html = await getText(source.url, "text/html,application/xhtml+xml");
|
||||
const text = pageText(html);
|
||||
const updated =
|
||||
text.match(/Last updated\s+(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
|
||||
"";
|
||||
|
||||
return [{
|
||||
sourceId: "companies-office",
|
||||
externalId: "latest-company-statistics",
|
||||
title: "Companies Office latest company statistics",
|
||||
summary: excerptAround(text, "Monthly data file", 3000),
|
||||
url: source.url,
|
||||
publishedAt: parseParliamentDate(updated),
|
||||
topics: ["business", "companies", "insolvency", "liquidation", "registrations", "new-zealand"],
|
||||
raw: { updated, text: text.slice(0, 10000) }
|
||||
}];
|
||||
}
|
||||
|
||||
async function fetchNzGazette(): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === "nz-gazette")!;
|
||||
const html = await getText(source.url, "text/html,application/xhtml+xml");
|
||||
const $ = load(html);
|
||||
const seen = new Set<string>();
|
||||
const items: NormalizedItem[] = [];
|
||||
|
||||
$('a[href*="/notice/id/"]').each((_index, element) => {
|
||||
if (items.length >= 50) return;
|
||||
const href = $(element).attr("href") ?? "";
|
||||
const title = $(element).text().replace(/\s+/g, " ").trim();
|
||||
if (!href || !title || title.length < 4) return;
|
||||
|
||||
const url = new URL(href, source.url).toString().split("?")[0];
|
||||
if (seen.has(url)) return;
|
||||
seen.add(url);
|
||||
|
||||
const row = $(element).closest("tr,li,article,div");
|
||||
const context = row.text().replace(/\s+/g, " ").trim();
|
||||
const noticeId =
|
||||
context.match(/\b20\d{2}-[a-z]{2}\d{3,6}\b/i)?.[0] ??
|
||||
url.split("/").filter(Boolean).pop() ??
|
||||
url;
|
||||
|
||||
items.push({
|
||||
sourceId: "nz-gazette",
|
||||
externalId: String(noticeId),
|
||||
title,
|
||||
summary: context.slice(0, 1500),
|
||||
url,
|
||||
publishedAt: parseParliamentDate(context),
|
||||
topics: ["gazette", "official-notice", "new-zealand"],
|
||||
raw: { noticeId, title, context, url }
|
||||
});
|
||||
});
|
||||
|
||||
if (!items.length) {
|
||||
throw new Error(`NZ Gazette returned no notice links (bytes=${html.length})`);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
async function fetchElectricityAuthority(): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === "electricity-authority")!;
|
||||
const html = await getText(source.url, "text/html,application/xhtml+xml");
|
||||
const $ = load(html);
|
||||
const seen = new Set<string>();
|
||||
const items: NormalizedItem[] = [];
|
||||
|
||||
$("a").each((_index, element) => {
|
||||
if (items.length >= 30) return;
|
||||
const title = $(element).text().replace(/\s+/g, " ").trim();
|
||||
if (!/^Electricity security of supply snapshot\s*-/i.test(title)) return;
|
||||
|
||||
const href = $(element).attr("href") ?? "";
|
||||
if (!href) return;
|
||||
const url = new URL(href, source.url).toString().split("?")[0];
|
||||
if (seen.has(url)) return;
|
||||
seen.add(url);
|
||||
|
||||
const context = $(element).closest("li,article,div").text().replace(/\s+/g, " ").trim();
|
||||
items.push({
|
||||
sourceId: "electricity-authority",
|
||||
externalId: url,
|
||||
title,
|
||||
summary: context.slice(0, 1600) || title,
|
||||
url,
|
||||
publishedAt: parseParliamentDate(title),
|
||||
topics: ["electricity", "energy", "security-of-supply", "hydro", "prices", "new-zealand"],
|
||||
raw: { title, context, url }
|
||||
});
|
||||
});
|
||||
|
||||
if (!items.length) {
|
||||
const text = pageText(html);
|
||||
return [{
|
||||
sourceId: "electricity-authority",
|
||||
externalId: "security-of-supply-index",
|
||||
title: "Electricity Authority security of supply",
|
||||
summary: excerptAround(text, "This week's snapshot", 2400),
|
||||
url: source.url,
|
||||
publishedAt: null,
|
||||
topics: ["electricity", "energy", "security-of-supply", "new-zealand"],
|
||||
raw: { text: text.slice(0, 8000) }
|
||||
}];
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz" | "treasury";
|
||||
export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz" | "treasury" | "rbnz" | "mbie-jobs-online" | "companies-office" | "nz-gazette" | "electricity-authority";
|
||||
|
||||
export interface SourceDefinition {
|
||||
id: SourceId;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user