diff --git a/src/db.ts b/src/db.ts index 11f2e51..428578f 100644 --- a/src/db.ts +++ b/src/db.ts @@ -395,6 +395,9 @@ function defaultTrustWeight(sourceClass: string, authorityTier: number) { official_registry: 0.97, official_notice: 0.95, official_analysis: 0.93, + official_market_notice: 0.99, + official_market_data: 1.0, + corporate_primary: 0.95, catalogue: 0.75 }; if (sourceClass in byClass) return byClass[sourceClass]; @@ -403,6 +406,19 @@ function defaultTrustWeight(sourceClass: string, authorityTier: number) { return 0.65; } +function defaultTrustNote(source: SourceDefinition) { + if (source.sourceClass === "corporate_primary") { + return "Strong primary evidence of what the company announced or claimed; not independent corroboration of the company's performance or impact claims."; + } + if (source.sourceClass === "official_market_notice") { + return "Primary exchange-hosted issuer disclosure. Strong evidence that the issuer made the disclosure; not an independent assessment of the issuer's claims."; + } + if (source.sourceClass === "official_market_data") { + return "Direct market-data source. Strong for observed price, volume and market-state facts within the feed's licensing and timing constraints."; + } + return "Seeded from configured ingestion source. Weight is an editorial prior, not a truth guarantee."; +} + export async function seedSourceTrustPolicy(sources: SourceDefinition[]) { for (const source of sources) { const domain = normaliseTrustDomain(source.url); @@ -426,7 +442,7 @@ export async function seedSourceTrustPolicy(sources: SourceDefinition[]) { weight, source.independenceGroup || domain, source.sourceClass, - "Seeded from configured ingestion source. Weight is an editorial prior, not a truth guarantee.", + defaultTrustNote(source), source.id ] ); diff --git a/src/mcp.ts b/src/mcp.ts index d8604e9..4c59368 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -30,7 +30,7 @@ import { mediaRoleAtLeast, type MediaRole } from "./auth.js"; const tools = [ { name: "sources_list", - description: "List configured NZ public information sources.", + description: "List configured public, market and first-party corporate information sources.", inputSchema: { type: "object", properties: {}, additionalProperties: false } }, { @@ -73,7 +73,7 @@ const tools = [ description: "Run ingestion now for one source or all sources.", inputSchema: { type: "object", - properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury", "rbnz", "mbie-jobs-online", "companies-office", "nz-gazette", "electricity-authority"] } }, + properties: { source_id: { type: "string", enum: ["geonet", "parliament", "stats-nz", "data-govt-nz", "treasury", "rbnz", "mbie-jobs-online", "companies-office", "nz-gazette", "electricity-authority", "nzx-announcements", "google-news", "meta-news", "anthropic-news", "openai-news"] } }, additionalProperties: false } }, diff --git a/src/scoring.ts b/src/scoring.ts index 950ea78..8173bb6 100644 --- a/src/scoring.ts +++ b/src/scoring.ts @@ -3,7 +3,9 @@ import type { NormalizedItem, ScoredItem } from "./types.js"; const DEFAULT_TERMS = [ "health", "technology", "agriculture", "primary production", "horticulture", "infrastructure", "economy", "employment", "trade", "climate", "energy", - "transport", "earthquake", "food", "housing", "business", "environment" + "transport", "earthquake", "food", "housing", "business", "environment", + "artificial intelligence", "machine learning", "cybersecurity", "data center", + "data centre", "semiconductor", "software", "robotics", "automation" ]; function watchTerms() { @@ -84,5 +86,25 @@ export function scoreItem(item: NormalizedItem): ScoredItem { reasons.push("New or updated Electricity Authority supply snapshot"); } + if (item.sourceId === "nzx-announcements") { + score += 1; + reasons.push("New NZX market announcement"); + const eventText = `${item.title} ${item.metrics?.announcementType ?? ""}`.toLowerCase(); + if (/(fll?yr|hal?fyr|transact|corpact|takeover|guidance|results|earnings|dividend|trading halt|capital raise|acquisition|merger|scheme)/i.test(eventText)) { + score += 2; + reasons.push("Potentially material market event"); + } + } + + if (["google-news","meta-news","anthropic-news","openai-news"].includes(item.sourceId)) { + score += 1; + reasons.push("First-party technology company update"); + const corporateSignal = `${item.title} ${item.summary}`.toLowerCase(); + if (/(introduc|launch|release|model|agent|api|security|research|partnership|acquisition|funding|data cent(?:er|re)|pricing|benchmark)/i.test(corporateSignal)) { + score += 1; + reasons.push("Potentially significant technology/company announcement"); + } + } + return { score, reasons }; } diff --git a/src/sources.ts b/src/sources.ts index e23756f..5beaa0a 100644 --- a/src/sources.ts +++ b/src/sources.ts @@ -103,6 +103,56 @@ export const SOURCES: SourceDefinition[] = [ sourceClass: "official_data", independenceGroup: "electricity-authority", authorityTier: 1, + }, + { + id: "nzx-announcements", + name: "NZX market announcements", + type: "html", + url: "https://announcements.nzx.com/", + publisher: "NZX Limited", + sourceClass: "official_market_notice", + independenceGroup: "nzx", + authorityTier: 1, + }, + { + id: "google-news", + name: "Google official blog", + type: "rss", + url: "https://blog.google/feed/", + publisher: "Google", + sourceClass: "corporate_primary", + independenceGroup: "google", + authorityTier: 1, + }, + { + id: "meta-news", + name: "Meta Newsroom", + type: "html", + url: "https://about.fb.com/news/", + publisher: "Meta", + sourceClass: "corporate_primary", + independenceGroup: "meta", + authorityTier: 1, + }, + { + id: "anthropic-news", + name: "Anthropic Newsroom", + type: "html", + url: "https://www.anthropic.com/news", + publisher: "Anthropic", + sourceClass: "corporate_primary", + independenceGroup: "anthropic", + authorityTier: 1, + }, + { + id: "openai-news", + name: "OpenAI News", + type: "rss", + url: "https://openai.com/news/rss.xml", + publisher: "OpenAI", + sourceClass: "corporate_primary", + independenceGroup: "openai", + authorityTier: 1, } ]; @@ -136,7 +186,13 @@ export async function fetchSource(id: SourceId): Promise { if (id === "mbie-jobs-online") return fetchMbieJobsOnline(); if (id === "companies-office") return fetchCompaniesOffice(); if (id === "nz-gazette") return fetchNzGazette(); - return fetchElectricityAuthority(); + if (id === "electricity-authority") return fetchElectricityAuthority(); + if (id === "nzx-announcements") return fetchNzxAnnouncements(); + if (id === "google-news") return fetchCorporateRss("google-news", ["google","technology","ai"]); + if (id === "meta-news") return fetchCorporateNewsroom("meta-news"); + if (id === "anthropic-news") return fetchCorporateNewsroom("anthropic-news"); + if (id === "openai-news") return fetchCorporateRss("openai-news", ["openai","technology","ai"]); + throw new Error(`Unknown source: ${id}`); } async function fetchGeoNet(): Promise { @@ -238,6 +294,190 @@ function parseParliamentDate(text: string): string | null { return Number.isNaN(value.getTime()) ? null : value.toISOString(); } +function parseNewsDate(text: string): string | null { + const direct = parseParliamentDate(text); + if (direct) return direct; + + const match = text.match(/\b(January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\s+(\d{1,2}),?\s+(20\d{2})\b/i); + if (!match) return null; + const months: Record = { + january:0, jan:0, february:1, feb:1, march:2, mar:2, april:3, apr:3, + may:4, june:5, jun:5, july:6, jul:6, august:7, aug:7, + september:8, sep:8, sept:8, october:9, oct:9, november:10, nov:10, + december:11, dec:11 + }; + const month = months[match[1].toLowerCase()]; + if (month == null) return null; + return new Date(Date.UTC(Number(match[3]), month, Number(match[2]))).toISOString(); +} + +function parseNzxDate(text: string): string | null { + const match = text.match(/(\d{1,2})\/(\d{1,2})\/(20\d{2})\s+(\d{1,2}):(\d{2})\s+(NZDT|NZST)/i); + if (!match) return null; + const offsetHours = match[6].toUpperCase() === "NZDT" ? 13 : 12; + const utc = Date.UTC( + Number(match[3]), + Number(match[2]) - 1, + Number(match[1]), + Number(match[4]) - offsetHours, + Number(match[5]) + ); + return new Date(utc).toISOString(); +} + +async function fetchCorporateRss( + sourceId: Extract, + topics: string[] +): Promise { + const source = SOURCES.find((s) => s.id === sourceId)!; + const xml = await getText( + source.url, + "application/rss+xml,application/atom+xml,application/xml,text/xml;q=0.9,*/*;q=0.5" + ); + const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" }); + const parsed = parser.parse(xml); + let list = asArray(parsed?.rss?.channel?.item ?? parsed?.feed?.entry); + if (!list.length) list = rawFeedItems(xml); + + return list.slice(0, 40).map((entry: any, index: number) => { + const title = cleanXmlText( + entry.title?.["#text"] ?? entry.title ?? + entry["atom:title"]?.["#text"] ?? entry["atom:title"] ?? + `${source.publisher} update` + ); + const summary = cleanXmlText( + entry.description?.["#text"] ?? entry.description ?? + entry.summary?.["#text"] ?? entry.summary ?? + entry.content?.["#text"] ?? entry.content ?? + entry["content:encoded"] ?? "" + ); + const url = firstLink(entry.link ?? entry["atom:link"]) || source.url; + const publishedAt = cleanXmlText( + entry.pubDate ?? entry.published ?? entry.updated ?? entry["dc:date"] ?? "" + ) || null; + + return { + sourceId, + externalId: String(entry.guid?.["#text"] ?? entry.guid ?? entry.id ?? url ?? index), + title, + summary: summary.slice(0, 4000), + url, + publishedAt, + topics: [...topics, "corporate-primary"], + raw: entry + }; + }).filter((item: NormalizedItem) => item.title && item.url); +} + +async function fetchNzxAnnouncements(): Promise { + const source = SOURCES.find((s) => s.id === "nzx-announcements")!; + const html = await getText(source.url, "text/html,application/xhtml+xml"); + const $ = load(html); + const items: NormalizedItem[] = []; + + $("table tbody tr, table tr").each((_index, row) => { + if (items.length >= 200) return; + const cells = $(row).find("td"); + if (cells.length < 6) return; + + const id = $(cells[1]).text().trim(); + const company = $(cells[2]).text().trim(); + const titleCell = $(cells[3]); + const title = titleCell.text().replace(/\s+/g, " ").trim(); + const dateText = $(cells[4]).text().replace(/\s+/g, " ").trim(); + const announcementType = $(cells[5]).text().replace(/\s+/g, " ").trim(); + const href = titleCell.find("a").attr("href") ?? ""; + + if (!id || !company || !title || !/^\d+$/.test(id)) return; + const url = href ? new URL(href, source.url).toString() : source.url; + + items.push({ + sourceId: "nzx-announcements", + externalId: id, + title: `${company}: ${title}`, + summary: `NZX market announcement from ${company}. Type: ${announcementType || "unspecified"}.`, + url, + publishedAt: parseNzxDate(dateText), + topics: ["markets","nzx","company-announcement",announcementType.toLowerCase()].filter(Boolean), + metrics: { + ticker: company, + announcementType: announcementType || null + }, + raw: { id, company, title, dateText, announcementType, url } + }); + }); + + if (!items.length) { + throw new Error(`NZX announcements returned no table rows (bytes=${html.length})`); + } + return items; +} + +async function fetchCorporateNewsroom( + sourceId: Extract +): Promise { + const source = SOURCES.find((s) => s.id === sourceId)!; + const html = await getText(source.url, "text/html,application/xhtml+xml"); + const $ = load(html); + const seen = new Set(); + const items: NormalizedItem[] = []; + + $("a[href]").each((_index, element) => { + if (items.length >= 40) return; + const href = $(element).attr("href") ?? ""; + if (!href) return; + + let url: string; + try { url = new URL(href, source.url).toString().split("?")[0].replace(/\/$/, ""); } + catch { return; } + + const parsed = new URL(url); + const sourceHost = new URL(source.url).hostname.replace(/^www\./, ""); + if (parsed.hostname.replace(/^www\./, "") !== sourceHost) return; + + const path = parsed.pathname.replace(/\/$/, ""); + const isArticle = sourceId === "meta-news" + ? /^\/news\/20\d{2}\/\d{2}\/[a-z0-9-]+/i.test(path) + : /^\/news\/[a-z0-9][a-z0-9-]+/i.test(path) && path !== "/news"; + if (!isArticle || seen.has(url)) return; + + const anchorText = $(element).text().replace(/\s+/g, " ").trim(); + const container = $(element).closest("article,li,section,div"); + const context = container.text().replace(/\s+/g, " ").trim(); + const title = anchorText.length >= 8 + ? anchorText + : (container.find("h1,h2,h3,h4").first().text().replace(/\s+/g, " ").trim() || anchorText); + if (!title || title.length < 8) return; + + seen.add(url); + const publishedAt = parseNewsDate(context); + const summary = context + .replace(title, "") + .replace(/\s+/g, " ") + .trim() + .slice(0, 3000); + + items.push({ + sourceId, + externalId: url, + title, + summary, + url, + publishedAt, + topics: [ + sourceId === "meta-news" ? "meta" : "anthropic", + "technology","ai","corporate-primary" + ], + raw: { title, summary, url, publishedAt } + }); + }); + + if (!items.length) { + throw new Error(`${source.name} returned no newsroom article links (bytes=${html.length})`); + } + return items; +} + async function fetchParliamentAtom(source: SourceDefinition): Promise { const feedUrl = "https://www3.parliament.nz/en/channellisting/rss/9343/e37f168d-4521-4437-a6fd-6bb7ff54e800"; const response = await fetch(feedUrl, { diff --git a/src/types.ts b/src/types.ts index 265f6ad..c896d2f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,19 @@ -export type SourceId = "geonet" | "parliament" | "stats-nz" | "data-govt-nz" | "treasury" | "rbnz" | "mbie-jobs-online" | "companies-office" | "nz-gazette" | "electricity-authority"; +export type SourceId = + | "geonet" + | "parliament" + | "stats-nz" + | "data-govt-nz" + | "treasury" + | "rbnz" + | "mbie-jobs-online" + | "companies-office" + | "nz-gazette" + | "electricity-authority" + | "nzx-announcements" + | "google-news" + | "meta-news" + | "anthropic-news" + | "openai-news"; export interface SourceDefinition { id: SourceId; @@ -6,7 +21,16 @@ export interface SourceDefinition { type: "geojson" | "html" | "sitemap" | "ckan" | "rss"; url: string; publisher: string; - sourceClass: "official_primary" | "official_data" | "official_registry" | "official_notice" | "official_analysis" | "catalogue"; + sourceClass: + | "official_primary" + | "official_data" + | "official_registry" + | "official_notice" + | "official_analysis" + | "official_market_notice" + | "official_market_data" + | "corporate_primary" + | "catalogue"; independenceGroup: string; authorityTier: 1 | 2 | 3; } diff --git a/src/web.ts b/src/web.ts index 916d011..68addd1 100644 --- a/src/web.ts +++ b/src/web.ts @@ -478,6 +478,9 @@ export function renderTrustPolicy(args:{identity:string;entries:AnyRow[];role?:s + + +