Sync Treasury source integration
This commit is contained in:
parent
0e85ca5f15
commit
4adef5c58b
@ -31,7 +31,7 @@ const tools = [
|
|||||||
description: "Run ingestion now for one source or all sources.",
|
description: "Run ingestion now for one source or all sources.",
|
||||||
inputSchema: {
|
inputSchema: {
|
||||||
type: "object",
|
type: "object",
|
||||||
properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz"] } },
|
properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury"] } },
|
||||||
additionalProperties: false
|
additionalProperties: false
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
@ -54,5 +54,10 @@ export function scoreItem(item: NormalizedItem): ScoredItem {
|
|||||||
reasons.push("New or updated public dataset metadata");
|
reasons.push("New or updated public dataset metadata");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (item.sourceId === "treasury") {
|
||||||
|
score += 1;
|
||||||
|
reasons.push("New or updated Treasury publication");
|
||||||
|
}
|
||||||
|
|
||||||
return { score, reasons };
|
return { score, reasons };
|
||||||
}
|
}
|
||||||
|
|||||||
@ -26,6 +26,12 @@ export const SOURCES: SourceDefinition[] = [
|
|||||||
name: "data.govt.nz catalogue",
|
name: "data.govt.nz catalogue",
|
||||||
type: "ckan",
|
type: "ckan",
|
||||||
url: "https://catalogue.data.govt.nz/api/3/action/package_search?sort=metadata_modified%20desc&rows=25"
|
url: "https://catalogue.data.govt.nz/api/3/action/package_search?sort=metadata_modified%20desc&rows=25"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "treasury",
|
||||||
|
name: "New Zealand Treasury publications",
|
||||||
|
type: "rss",
|
||||||
|
url: "https://www.treasury.govt.nz/feeds/publications"
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
|
|
||||||
@ -53,7 +59,8 @@ export async function fetchSource(id: SourceId): Promise<NormalizedItem[]> {
|
|||||||
if (id === "geonet") return fetchGeoNet();
|
if (id === "geonet") return fetchGeoNet();
|
||||||
if (id === "parliament") return fetchParliament();
|
if (id === "parliament") return fetchParliament();
|
||||||
if (id === "stats-nz") return fetchStatsNz();
|
if (id === "stats-nz") return fetchStatsNz();
|
||||||
return fetchDataGovtNz();
|
if (id === "data-govt-nz") return fetchDataGovtNz();
|
||||||
|
return fetchTreasury();
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchGeoNet(): Promise<NormalizedItem[]> {
|
async function fetchGeoNet(): Promise<NormalizedItem[]> {
|
||||||
@ -444,3 +451,54 @@ async function fetchDataGovtNz(): Promise<NormalizedItem[]> {
|
|||||||
raw: pkg
|
raw: pkg
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async function fetchTreasury(): Promise<NormalizedItem[]> {
|
||||||
|
const source = SOURCES.find((s) => s.id === "treasury")!;
|
||||||
|
const xml = await getText(
|
||||||
|
source.url,
|
||||||
|
"application/rss+xml,application/xml,text/xml;q=0.9,*/*;q=0.5"
|
||||||
|
);
|
||||||
|
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
|
||||||
|
const parsed = parser.parse(xml);
|
||||||
|
let list = asArray<any>(parsed?.rss?.channel?.item ?? parsed?.feed?.entry);
|
||||||
|
if (!list.length) list = rawFeedItems(xml);
|
||||||
|
|
||||||
|
return list.slice(0, 40).map((entry: any, index: number) => {
|
||||||
|
const title = cleanXmlText(
|
||||||
|
entry.title?.["#text"] ??
|
||||||
|
entry.title ??
|
||||||
|
entry["atom:title"]?.["#text"] ??
|
||||||
|
entry["atom:title"] ??
|
||||||
|
"Treasury publication"
|
||||||
|
);
|
||||||
|
const summary = cleanXmlText(
|
||||||
|
entry.description?.["#text"] ??
|
||||||
|
entry.description ??
|
||||||
|
entry.summary?.["#text"] ??
|
||||||
|
entry.summary ??
|
||||||
|
entry.content ??
|
||||||
|
""
|
||||||
|
);
|
||||||
|
const url = firstLink(entry.link ?? entry["atom:link"]) || source.url;
|
||||||
|
const publishedAt =
|
||||||
|
cleanXmlText(
|
||||||
|
entry.pubDate ??
|
||||||
|
entry.published ??
|
||||||
|
entry.updated ??
|
||||||
|
entry["dc:date"] ??
|
||||||
|
""
|
||||||
|
) || null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
sourceId: "treasury" as const,
|
||||||
|
externalId: String(entry.guid?.["#text"] ?? entry.guid ?? entry.id ?? url ?? index),
|
||||||
|
title,
|
||||||
|
summary,
|
||||||
|
url,
|
||||||
|
publishedAt,
|
||||||
|
topics: ["treasury", "economy", "public-finance", "new-zealand"],
|
||||||
|
raw: entry
|
||||||
|
};
|
||||||
|
}).filter((item: NormalizedItem) => item.title && item.url);
|
||||||
|
}
|
||||||
|
|||||||
@ -1,9 +1,9 @@
|
|||||||
export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz";
|
export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz" | "treasury";
|
||||||
|
|
||||||
export interface SourceDefinition {
|
export interface SourceDefinition {
|
||||||
id: SourceId;
|
id: SourceId;
|
||||||
name: string;
|
name: string;
|
||||||
type: "geojson" | "html" | "sitemap" | "ckan";
|
type: "geojson" | "html" | "sitemap" | "ckan" | "rss";
|
||||||
url: string;
|
url: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user