Sync Treasury source integration

This commit is contained in:
Admin 2026-09-29 07:09:38 +00:00
parent 0e85ca5f15
commit 4adef5c58b
4 changed files with 67 additions and 4 deletions

View File

@ -31,7 +31,7 @@ const tools = [
description: "Run ingestion now for one source or all sources.",
inputSchema: {
type: "object",
properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz"] } },
properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury"] } },
additionalProperties: false
}
},

View File

@ -54,5 +54,10 @@ export function scoreItem(item: NormalizedItem): ScoredItem {
reasons.push("New or updated public dataset metadata");
}
if (item.sourceId === "treasury") {
score += 1;
reasons.push("New or updated Treasury publication");
}
return { score, reasons };
}

View File

@ -26,6 +26,12 @@ export const SOURCES: SourceDefinition[] = [
name: "data.govt.nz catalogue",
type: "ckan",
url: "https://catalogue.data.govt.nz/api/3/action/package_search?sort=metadata_modified%20desc&rows=25"
},
{
id: "treasury",
name: "New Zealand Treasury publications",
type: "rss",
url: "https://www.treasury.govt.nz/feeds/publications"
}
];
@ -53,7 +59,8 @@ export async function fetchSource(id: SourceId): Promise<NormalizedItem[]> {
if (id === "geonet") return fetchGeoNet();
if (id === "parliament") return fetchParliament();
if (id === "stats-nz") return fetchStatsNz();
return fetchDataGovtNz();
if (id === "data-govt-nz") return fetchDataGovtNz();
return fetchTreasury();
}
async function fetchGeoNet(): Promise<NormalizedItem[]> {
@ -444,3 +451,54 @@ async function fetchDataGovtNz(): Promise<NormalizedItem[]> {
raw: pkg
}));
}
async function fetchTreasury(): Promise<NormalizedItem[]> {
const source = SOURCES.find((s) => s.id === "treasury")!;
const xml = await getText(
source.url,
"application/rss+xml,application/xml,text/xml;q=0.9,*/*;q=0.5"
);
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
const parsed = parser.parse(xml);
let list = asArray<any>(parsed?.rss?.channel?.item ?? parsed?.feed?.entry);
if (!list.length) list = rawFeedItems(xml);
return list.slice(0, 40).map((entry: any, index: number) => {
const title = cleanXmlText(
entry.title?.["#text"] ??
entry.title ??
entry["atom:title"]?.["#text"] ??
entry["atom:title"] ??
"Treasury publication"
);
const summary = cleanXmlText(
entry.description?.["#text"] ??
entry.description ??
entry.summary?.["#text"] ??
entry.summary ??
entry.content ??
""
);
const url = firstLink(entry.link ?? entry["atom:link"]) || source.url;
const publishedAt =
cleanXmlText(
entry.pubDate ??
entry.published ??
entry.updated ??
entry["dc:date"] ??
""
) || null;
return {
sourceId: "treasury" as const,
externalId: String(entry.guid?.["#text"] ?? entry.guid ?? entry.id ?? url ?? index),
title,
summary,
url,
publishedAt,
topics: ["treasury", "economy", "public-finance", "new-zealand"],
raw: entry
};
}).filter((item: NormalizedItem) => item.title && item.url);
}

View File

@ -1,9 +1,9 @@
export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz";
export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz" | "treasury";
export interface SourceDefinition {
id: SourceId;
name: string;
type: "geojson" | "html" | "sitemap" | "ckan";
type: "geojson" | "html" | "sitemap" | "ckan" | "rss";
url: string;
}