Sync Treasury source integration
This commit is contained in:
parent
0e85ca5f15
commit
4adef5c58b
@ -31,7 +31,7 @@ const tools = [
|
||||
description: "Run ingestion now for one source or all sources.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz"] } },
|
||||
properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury"] } },
|
||||
additionalProperties: false
|
||||
}
|
||||
},
|
||||
|
||||
@ -54,5 +54,10 @@ export function scoreItem(item: NormalizedItem): ScoredItem {
|
||||
reasons.push("New or updated public dataset metadata");
|
||||
}
|
||||
|
||||
if (item.sourceId === "treasury") {
|
||||
score += 1;
|
||||
reasons.push("New or updated Treasury publication");
|
||||
}
|
||||
|
||||
return { score, reasons };
|
||||
}
|
||||
|
||||
@ -26,6 +26,12 @@ export const SOURCES: SourceDefinition[] = [
|
||||
name: "data.govt.nz catalogue",
|
||||
type: "ckan",
|
||||
url: "https://catalogue.data.govt.nz/api/3/action/package_search?sort=metadata_modified%20desc&rows=25"
|
||||
},
|
||||
{
|
||||
id: "treasury",
|
||||
name: "New Zealand Treasury publications",
|
||||
type: "rss",
|
||||
url: "https://www.treasury.govt.nz/feeds/publications"
|
||||
}
|
||||
];
|
||||
|
||||
@ -53,7 +59,8 @@ export async function fetchSource(id: SourceId): Promise<NormalizedItem[]> {
|
||||
if (id === "geonet") return fetchGeoNet();
|
||||
if (id === "parliament") return fetchParliament();
|
||||
if (id === "stats-nz") return fetchStatsNz();
|
||||
return fetchDataGovtNz();
|
||||
if (id === "data-govt-nz") return fetchDataGovtNz();
|
||||
return fetchTreasury();
|
||||
}
|
||||
|
||||
async function fetchGeoNet(): Promise<NormalizedItem[]> {
|
||||
@ -444,3 +451,54 @@ async function fetchDataGovtNz(): Promise<NormalizedItem[]> {
|
||||
raw: pkg
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
async function fetchTreasury(): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === "treasury")!;
|
||||
const xml = await getText(
|
||||
source.url,
|
||||
"application/rss+xml,application/xml,text/xml;q=0.9,*/*;q=0.5"
|
||||
);
|
||||
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
|
||||
const parsed = parser.parse(xml);
|
||||
let list = asArray<any>(parsed?.rss?.channel?.item ?? parsed?.feed?.entry);
|
||||
if (!list.length) list = rawFeedItems(xml);
|
||||
|
||||
return list.slice(0, 40).map((entry: any, index: number) => {
|
||||
const title = cleanXmlText(
|
||||
entry.title?.["#text"] ??
|
||||
entry.title ??
|
||||
entry["atom:title"]?.["#text"] ??
|
||||
entry["atom:title"] ??
|
||||
"Treasury publication"
|
||||
);
|
||||
const summary = cleanXmlText(
|
||||
entry.description?.["#text"] ??
|
||||
entry.description ??
|
||||
entry.summary?.["#text"] ??
|
||||
entry.summary ??
|
||||
entry.content ??
|
||||
""
|
||||
);
|
||||
const url = firstLink(entry.link ?? entry["atom:link"]) || source.url;
|
||||
const publishedAt =
|
||||
cleanXmlText(
|
||||
entry.pubDate ??
|
||||
entry.published ??
|
||||
entry.updated ??
|
||||
entry["dc:date"] ??
|
||||
""
|
||||
) || null;
|
||||
|
||||
return {
|
||||
sourceId: "treasury" as const,
|
||||
externalId: String(entry.guid?.["#text"] ?? entry.guid ?? entry.id ?? url ?? index),
|
||||
title,
|
||||
summary,
|
||||
url,
|
||||
publishedAt,
|
||||
topics: ["treasury", "economy", "public-finance", "new-zealand"],
|
||||
raw: entry
|
||||
};
|
||||
}).filter((item: NormalizedItem) => item.title && item.url);
|
||||
}
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz";
|
||||
export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz" | "treasury";
|
||||
|
||||
export interface SourceDefinition {
|
||||
id: SourceId;
|
||||
name: string;
|
||||
type: "geojson" | "html" | "sitemap" | "ckan";
|
||||
type: "geojson" | "html" | "sitemap" | "ckan" | "rss";
|
||||
url: string;
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user