Sync NZX and AI company source expansion
This commit is contained in:
parent
5a92a2d792
commit
1eb9374b33
18
src/db.ts
18
src/db.ts
@ -395,6 +395,9 @@ function defaultTrustWeight(sourceClass: string, authorityTier: number) {
|
||||
official_registry: 0.97,
|
||||
official_notice: 0.95,
|
||||
official_analysis: 0.93,
|
||||
official_market_notice: 0.99,
|
||||
official_market_data: 1.0,
|
||||
corporate_primary: 0.95,
|
||||
catalogue: 0.75
|
||||
};
|
||||
if (sourceClass in byClass) return byClass[sourceClass];
|
||||
@ -403,6 +406,19 @@ function defaultTrustWeight(sourceClass: string, authorityTier: number) {
|
||||
return 0.65;
|
||||
}
|
||||
|
||||
function defaultTrustNote(source: SourceDefinition) {
|
||||
if (source.sourceClass === "corporate_primary") {
|
||||
return "Strong primary evidence of what the company announced or claimed; not independent corroboration of the company's performance or impact claims.";
|
||||
}
|
||||
if (source.sourceClass === "official_market_notice") {
|
||||
return "Primary exchange-hosted issuer disclosure. Strong evidence that the issuer made the disclosure; not an independent assessment of the issuer's claims.";
|
||||
}
|
||||
if (source.sourceClass === "official_market_data") {
|
||||
return "Direct market-data source. Strong for observed price, volume and market-state facts within the feed's licensing and timing constraints.";
|
||||
}
|
||||
return "Seeded from configured ingestion source. Weight is an editorial prior, not a truth guarantee.";
|
||||
}
|
||||
|
||||
export async function seedSourceTrustPolicy(sources: SourceDefinition[]) {
|
||||
for (const source of sources) {
|
||||
const domain = normaliseTrustDomain(source.url);
|
||||
@ -426,7 +442,7 @@ export async function seedSourceTrustPolicy(sources: SourceDefinition[]) {
|
||||
weight,
|
||||
source.independenceGroup || domain,
|
||||
source.sourceClass,
|
||||
"Seeded from configured ingestion source. Weight is an editorial prior, not a truth guarantee.",
|
||||
defaultTrustNote(source),
|
||||
source.id
|
||||
]
|
||||
);
|
||||
|
||||
@ -30,7 +30,7 @@ import { mediaRoleAtLeast, type MediaRole } from "./auth.js";
|
||||
const tools = [
|
||||
{
|
||||
name: "sources_list",
|
||||
description: "List configured NZ public information sources.",
|
||||
description: "List configured public, market and first-party corporate information sources.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false }
|
||||
},
|
||||
{
|
||||
@ -73,7 +73,7 @@ const tools = [
|
||||
description: "Run ingestion now for one source or all sources.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury", "rbnz", "mbie-jobs-online", "companies-office", "nz-gazette", "electricity-authority"] } },
|
||||
properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury", "rbnz", "mbie-jobs-online", "companies-office", "nz-gazette", "electricity-authority", "nzx-announcements", "google-news", "meta-news", "anthropic-news", "openai-news"] } },
|
||||
additionalProperties: false
|
||||
}
|
||||
},
|
||||
|
||||
@ -3,7 +3,9 @@ import type { NormalizedItem, ScoredItem } from "./types.js";
|
||||
const DEFAULT_TERMS = [
|
||||
"health", "technology", "agriculture", "primary production", "horticulture",
|
||||
"infrastructure", "economy", "employment", "trade", "climate", "energy",
|
||||
"transport", "earthquake", "food", "housing", "business", "environment"
|
||||
"transport", "earthquake", "food", "housing", "business", "environment",
|
||||
"artificial intelligence", "machine learning", "cybersecurity", "data center",
|
||||
"data centre", "semiconductor", "software", "robotics", "automation"
|
||||
];
|
||||
|
||||
function watchTerms() {
|
||||
@ -84,5 +86,25 @@ export function scoreItem(item: NormalizedItem): ScoredItem {
|
||||
reasons.push("New or updated Electricity Authority supply snapshot");
|
||||
}
|
||||
|
||||
if (item.sourceId === "nzx-announcements") {
|
||||
score += 1;
|
||||
reasons.push("New NZX market announcement");
|
||||
const eventText = `${item.title} ${item.metrics?.announcementType ?? ""}`.toLowerCase();
|
||||
if (/(fll?yr|hal?fyr|transact|corpact|takeover|guidance|results|earnings|dividend|trading halt|capital raise|acquisition|merger|scheme)/i.test(eventText)) {
|
||||
score += 2;
|
||||
reasons.push("Potentially material market event");
|
||||
}
|
||||
}
|
||||
|
||||
if (["google-news","meta-news","anthropic-news","openai-news"].includes(item.sourceId)) {
|
||||
score += 1;
|
||||
reasons.push("First-party technology company update");
|
||||
const corporateSignal = `${item.title} ${item.summary}`.toLowerCase();
|
||||
if (/(introduc|launch|release|model|agent|api|security|research|partnership|acquisition|funding|data cent(?:er|re)|pricing|benchmark)/i.test(corporateSignal)) {
|
||||
score += 1;
|
||||
reasons.push("Potentially significant technology/company announcement");
|
||||
}
|
||||
}
|
||||
|
||||
return { score, reasons };
|
||||
}
|
||||
|
||||
242
src/sources.ts
242
src/sources.ts
@ -103,6 +103,56 @@ export const SOURCES: SourceDefinition[] = [
|
||||
sourceClass: "official_data",
|
||||
independenceGroup: "electricity-authority",
|
||||
authorityTier: 1,
|
||||
},
|
||||
{
|
||||
id: "nzx-announcements",
|
||||
name: "NZX market announcements",
|
||||
type: "html",
|
||||
url: "https://announcements.nzx.com/",
|
||||
publisher: "NZX Limited",
|
||||
sourceClass: "official_market_notice",
|
||||
independenceGroup: "nzx",
|
||||
authorityTier: 1,
|
||||
},
|
||||
{
|
||||
id: "google-news",
|
||||
name: "Google official blog",
|
||||
type: "rss",
|
||||
url: "https://blog.google/feed/",
|
||||
publisher: "Google",
|
||||
sourceClass: "corporate_primary",
|
||||
independenceGroup: "google",
|
||||
authorityTier: 1,
|
||||
},
|
||||
{
|
||||
id: "meta-news",
|
||||
name: "Meta Newsroom",
|
||||
type: "html",
|
||||
url: "https://about.fb.com/news/",
|
||||
publisher: "Meta",
|
||||
sourceClass: "corporate_primary",
|
||||
independenceGroup: "meta",
|
||||
authorityTier: 1,
|
||||
},
|
||||
{
|
||||
id: "anthropic-news",
|
||||
name: "Anthropic Newsroom",
|
||||
type: "html",
|
||||
url: "https://www.anthropic.com/news",
|
||||
publisher: "Anthropic",
|
||||
sourceClass: "corporate_primary",
|
||||
independenceGroup: "anthropic",
|
||||
authorityTier: 1,
|
||||
},
|
||||
{
|
||||
id: "openai-news",
|
||||
name: "OpenAI News",
|
||||
type: "rss",
|
||||
url: "https://openai.com/news/rss.xml",
|
||||
publisher: "OpenAI",
|
||||
sourceClass: "corporate_primary",
|
||||
independenceGroup: "openai",
|
||||
authorityTier: 1,
|
||||
}
|
||||
];
|
||||
|
||||
@ -136,7 +186,13 @@ export async function fetchSource(id: SourceId): Promise<NormalizedItem[]> {
|
||||
if (id === "mbie-jobs-online") return fetchMbieJobsOnline();
|
||||
if (id === "companies-office") return fetchCompaniesOffice();
|
||||
if (id === "nz-gazette") return fetchNzGazette();
|
||||
return fetchElectricityAuthority();
|
||||
if (id === "electricity-authority") return fetchElectricityAuthority();
|
||||
if (id === "nzx-announcements") return fetchNzxAnnouncements();
|
||||
if (id === "google-news") return fetchCorporateRss("google-news", ["google","technology","ai"]);
|
||||
if (id === "meta-news") return fetchCorporateNewsroom("meta-news");
|
||||
if (id === "anthropic-news") return fetchCorporateNewsroom("anthropic-news");
|
||||
if (id === "openai-news") return fetchCorporateRss("openai-news", ["openai","technology","ai"]);
|
||||
throw new Error(`Unknown source: ${id}`);
|
||||
}
|
||||
|
||||
async function fetchGeoNet(): Promise<NormalizedItem[]> {
|
||||
@ -238,6 +294,190 @@ function parseParliamentDate(text: string): string | null {
|
||||
return Number.isNaN(value.getTime()) ? null : value.toISOString();
|
||||
}
|
||||
|
||||
function parseNewsDate(text: string): string | null {
|
||||
const direct = parseParliamentDate(text);
|
||||
if (direct) return direct;
|
||||
|
||||
const match = text.match(/\b(January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\s+(\d{1,2}),?\s+(20\d{2})\b/i);
|
||||
if (!match) return null;
|
||||
const months: Record<string, number> = {
|
||||
january:0, jan:0, february:1, feb:1, march:2, mar:2, april:3, apr:3,
|
||||
may:4, june:5, jun:5, july:6, jul:6, august:7, aug:7,
|
||||
september:8, sep:8, sept:8, october:9, oct:9, november:10, nov:10,
|
||||
december:11, dec:11
|
||||
};
|
||||
const month = months[match[1].toLowerCase()];
|
||||
if (month == null) return null;
|
||||
return new Date(Date.UTC(Number(match[3]), month, Number(match[2]))).toISOString();
|
||||
}
|
||||
|
||||
function parseNzxDate(text: string): string | null {
|
||||
const match = text.match(/(\d{1,2})\/(\d{1,2})\/(20\d{2})\s+(\d{1,2}):(\d{2})\s+(NZDT|NZST)/i);
|
||||
if (!match) return null;
|
||||
const offsetHours = match[6].toUpperCase() === "NZDT" ? 13 : 12;
|
||||
const utc = Date.UTC(
|
||||
Number(match[3]),
|
||||
Number(match[2]) - 1,
|
||||
Number(match[1]),
|
||||
Number(match[4]) - offsetHours,
|
||||
Number(match[5])
|
||||
);
|
||||
return new Date(utc).toISOString();
|
||||
}
|
||||
|
||||
async function fetchCorporateRss(
|
||||
sourceId: Extract<SourceId, "google-news" | "openai-news">,
|
||||
topics: string[]
|
||||
): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === sourceId)!;
|
||||
const xml = await getText(
|
||||
source.url,
|
||||
"application/rss+xml,application/atom+xml,application/xml,text/xml;q=0.9,*/*;q=0.5"
|
||||
);
|
||||
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
|
||||
const parsed = parser.parse(xml);
|
||||
let list = asArray<any>(parsed?.rss?.channel?.item ?? parsed?.feed?.entry);
|
||||
if (!list.length) list = rawFeedItems(xml);
|
||||
|
||||
return list.slice(0, 40).map((entry: any, index: number) => {
|
||||
const title = cleanXmlText(
|
||||
entry.title?.["#text"] ?? entry.title ??
|
||||
entry["atom:title"]?.["#text"] ?? entry["atom:title"] ??
|
||||
`${source.publisher} update`
|
||||
);
|
||||
const summary = cleanXmlText(
|
||||
entry.description?.["#text"] ?? entry.description ??
|
||||
entry.summary?.["#text"] ?? entry.summary ??
|
||||
entry.content?.["#text"] ?? entry.content ??
|
||||
entry["content:encoded"] ?? ""
|
||||
);
|
||||
const url = firstLink(entry.link ?? entry["atom:link"]) || source.url;
|
||||
const publishedAt = cleanXmlText(
|
||||
entry.pubDate ?? entry.published ?? entry.updated ?? entry["dc:date"] ?? ""
|
||||
) || null;
|
||||
|
||||
return {
|
||||
sourceId,
|
||||
externalId: String(entry.guid?.["#text"] ?? entry.guid ?? entry.id ?? url ?? index),
|
||||
title,
|
||||
summary: summary.slice(0, 4000),
|
||||
url,
|
||||
publishedAt,
|
||||
topics: [...topics, "corporate-primary"],
|
||||
raw: entry
|
||||
};
|
||||
}).filter((item: NormalizedItem) => item.title && item.url);
|
||||
}
|
||||
|
||||
async function fetchNzxAnnouncements(): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === "nzx-announcements")!;
|
||||
const html = await getText(source.url, "text/html,application/xhtml+xml");
|
||||
const $ = load(html);
|
||||
const items: NormalizedItem[] = [];
|
||||
|
||||
$("table tbody tr, table tr").each((_index, row) => {
|
||||
if (items.length >= 200) return;
|
||||
const cells = $(row).find("td");
|
||||
if (cells.length < 6) return;
|
||||
|
||||
const id = $(cells[1]).text().trim();
|
||||
const company = $(cells[2]).text().trim();
|
||||
const titleCell = $(cells[3]);
|
||||
const title = titleCell.text().replace(/\s+/g, " ").trim();
|
||||
const dateText = $(cells[4]).text().replace(/\s+/g, " ").trim();
|
||||
const announcementType = $(cells[5]).text().replace(/\s+/g, " ").trim();
|
||||
const href = titleCell.find("a").attr("href") ?? "";
|
||||
|
||||
if (!id || !company || !title || !/^\d+$/.test(id)) return;
|
||||
const url = href ? new URL(href, source.url).toString() : source.url;
|
||||
|
||||
items.push({
|
||||
sourceId: "nzx-announcements",
|
||||
externalId: id,
|
||||
title: `${company}: ${title}`,
|
||||
summary: `NZX market announcement from ${company}. Type: ${announcementType || "unspecified"}.`,
|
||||
url,
|
||||
publishedAt: parseNzxDate(dateText),
|
||||
topics: ["markets","nzx","company-announcement",announcementType.toLowerCase()].filter(Boolean),
|
||||
metrics: {
|
||||
ticker: company,
|
||||
announcementType: announcementType || null
|
||||
},
|
||||
raw: { id, company, title, dateText, announcementType, url }
|
||||
});
|
||||
});
|
||||
|
||||
if (!items.length) {
|
||||
throw new Error(`NZX announcements returned no table rows (bytes=${html.length})`);
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
async function fetchCorporateNewsroom(
|
||||
sourceId: Extract<SourceId, "meta-news" | "anthropic-news">
|
||||
): Promise<NormalizedItem[]> {
|
||||
const source = SOURCES.find((s) => s.id === sourceId)!;
|
||||
const html = await getText(source.url, "text/html,application/xhtml+xml");
|
||||
const $ = load(html);
|
||||
const seen = new Set<string>();
|
||||
const items: NormalizedItem[] = [];
|
||||
|
||||
$("a[href]").each((_index, element) => {
|
||||
if (items.length >= 40) return;
|
||||
const href = $(element).attr("href") ?? "";
|
||||
if (!href) return;
|
||||
|
||||
let url: string;
|
||||
try { url = new URL(href, source.url).toString().split("?")[0].replace(/\/$/, ""); }
|
||||
catch { return; }
|
||||
|
||||
const parsed = new URL(url);
|
||||
const sourceHost = new URL(source.url).hostname.replace(/^www\./, "");
|
||||
if (parsed.hostname.replace(/^www\./, "") !== sourceHost) return;
|
||||
|
||||
const path = parsed.pathname.replace(/\/$/, "");
|
||||
const isArticle = sourceId === "meta-news"
|
||||
? /^\/news\/20\d{2}\/\d{2}\/[a-z0-9-]+/i.test(path)
|
||||
: /^\/news\/[a-z0-9][a-z0-9-]+/i.test(path) && path !== "/news";
|
||||
if (!isArticle || seen.has(url)) return;
|
||||
|
||||
const anchorText = $(element).text().replace(/\s+/g, " ").trim();
|
||||
const container = $(element).closest("article,li,section,div");
|
||||
const context = container.text().replace(/\s+/g, " ").trim();
|
||||
const title = anchorText.length >= 8
|
||||
? anchorText
|
||||
: (container.find("h1,h2,h3,h4").first().text().replace(/\s+/g, " ").trim() || anchorText);
|
||||
if (!title || title.length < 8) return;
|
||||
|
||||
seen.add(url);
|
||||
const publishedAt = parseNewsDate(context);
|
||||
const summary = context
|
||||
.replace(title, "")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.slice(0, 3000);
|
||||
|
||||
items.push({
|
||||
sourceId,
|
||||
externalId: url,
|
||||
title,
|
||||
summary,
|
||||
url,
|
||||
publishedAt,
|
||||
topics: [
|
||||
sourceId === "meta-news" ? "meta" : "anthropic",
|
||||
"technology","ai","corporate-primary"
|
||||
],
|
||||
raw: { title, summary, url, publishedAt }
|
||||
});
|
||||
});
|
||||
|
||||
if (!items.length) {
|
||||
throw new Error(`${source.name} returned no newsroom article links (bytes=${html.length})`);
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
async function fetchParliamentAtom(source: SourceDefinition): Promise<NormalizedItem[]> {
|
||||
const feedUrl = "https://www3.parliament.nz/en/channellisting/rss/9343/e37f168d-4521-4437-a6fd-6bb7ff54e800";
|
||||
const response = await fetch(feedUrl, {
|
||||
|
||||
28
src/types.ts
28
src/types.ts
@ -1,4 +1,19 @@
|
||||
export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz" | "treasury" | "rbnz" | "mbie-jobs-online" | "companies-office" | "nz-gazette" | "electricity-authority";
|
||||
export type SourceId =
|
||||
| "geonet"
|
||||
| "parliament"
|
||||
| "stats-nz"
|
||||
| "data-govt-nz"
|
||||
| "treasury"
|
||||
| "rbnz"
|
||||
| "mbie-jobs-online"
|
||||
| "companies-office"
|
||||
| "nz-gazette"
|
||||
| "electricity-authority"
|
||||
| "nzx-announcements"
|
||||
| "google-news"
|
||||
| "meta-news"
|
||||
| "anthropic-news"
|
||||
| "openai-news";
|
||||
|
||||
export interface SourceDefinition {
|
||||
id: SourceId;
|
||||
@ -6,7 +21,16 @@ export interface SourceDefinition {
|
||||
type: "geojson" | "html" | "sitemap" | "ckan" | "rss";
|
||||
url: string;
|
||||
publisher: string;
|
||||
sourceClass: "official_primary" | "official_data" | "official_registry" | "official_notice" | "official_analysis" | "catalogue";
|
||||
sourceClass:
|
||||
| "official_primary"
|
||||
| "official_data"
|
||||
| "official_registry"
|
||||
| "official_notice"
|
||||
| "official_analysis"
|
||||
| "official_market_notice"
|
||||
| "official_market_data"
|
||||
| "corporate_primary"
|
||||
| "catalogue";
|
||||
independenceGroup: string;
|
||||
authorityTier: 1 | 2 | 3;
|
||||
}
|
||||
|
||||
@ -478,6 +478,9 @@ export function renderTrustPolicy(args:{identity:string;entries:AnyRow[];role?:s
|
||||
<option value="official_registry">official registry</option>
|
||||
<option value="official_notice">official notice</option>
|
||||
<option value="official_analysis">official analysis</option>
|
||||
<option value="official_market_notice">official market notice</option>
|
||||
<option value="official_market_data">official market data</option>
|
||||
<option value="corporate_primary">corporate primary</option>
|
||||
<option value="independent_research">independent research</option>
|
||||
<option value="newsroom">newsroom</option>
|
||||
<option value="specialist">specialist</option>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user