Deploy categorised newsroom sources
This commit is contained in:
parent
3ed6c73b4c
commit
66a1026fd0
13
src/db.ts
13
src/db.ts
@ -31,6 +31,7 @@ export async function migrate() {
|
||||
baseline_complete BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
publisher TEXT NOT NULL DEFAULT '',
|
||||
source_class TEXT NOT NULL DEFAULT 'catalogue',
|
||||
source_category TEXT NOT NULL DEFAULT 'other',
|
||||
independence_group TEXT NOT NULL DEFAULT '',
|
||||
authority_tier INTEGER NOT NULL DEFAULT 3,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
@ -246,6 +247,7 @@ export async function migrate() {
|
||||
await pool.query("ALTER TABLE sources ADD COLUMN IF NOT EXISTS baseline_complete BOOLEAN NOT NULL DEFAULT FALSE");
|
||||
await pool.query("ALTER TABLE sources ADD COLUMN IF NOT EXISTS publisher TEXT NOT NULL DEFAULT ''");
|
||||
await pool.query("ALTER TABLE sources ADD COLUMN IF NOT EXISTS source_class TEXT NOT NULL DEFAULT 'catalogue'");
|
||||
await pool.query("ALTER TABLE sources ADD COLUMN IF NOT EXISTS source_category TEXT NOT NULL DEFAULT 'other'");
|
||||
await pool.query("ALTER TABLE sources ADD COLUMN IF NOT EXISTS independence_group TEXT NOT NULL DEFAULT ''");
|
||||
await pool.query("ALTER TABLE sources ADD COLUMN IF NOT EXISTS authority_tier INTEGER NOT NULL DEFAULT 3");
|
||||
await pool.query("ALTER TABLE evidence ADD COLUMN IF NOT EXISTS snapshot_id TEXT REFERENCES source_snapshots(id) ON DELETE SET NULL");
|
||||
@ -347,20 +349,21 @@ export async function seedSources(sources: SourceDefinition[]) {
|
||||
for (const source of sources) {
|
||||
await pool.query(
|
||||
`INSERT INTO sources(
|
||||
id,name,source_type,url,publisher,source_class,independence_group,authority_tier
|
||||
id,name,source_type,url,publisher,source_class,source_category,independence_group,authority_tier
|
||||
)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
name=EXCLUDED.name,
|
||||
source_type=EXCLUDED.source_type,
|
||||
url=EXCLUDED.url,
|
||||
publisher=EXCLUDED.publisher,
|
||||
source_class=EXCLUDED.source_class,
|
||||
source_category=EXCLUDED.source_category,
|
||||
independence_group=EXCLUDED.independence_group,
|
||||
authority_tier=EXCLUDED.authority_tier`,
|
||||
[
|
||||
source.id, source.name, source.type, source.url,
|
||||
source.publisher, source.sourceClass, source.independenceGroup, source.authorityTier
|
||||
source.publisher, source.sourceClass, source.category, source.independenceGroup, source.authorityTier
|
||||
]
|
||||
);
|
||||
}
|
||||
@ -809,7 +812,7 @@ export async function storeItem(
|
||||
}
|
||||
|
||||
export async function listSources() {
|
||||
const result = await pool.query("SELECT * FROM sources ORDER BY id");
|
||||
const result = await pool.query("SELECT * FROM sources ORDER BY source_category, name");
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
@ -825,7 +828,7 @@ export async function sourceStatus() {
|
||||
WHERE sig.source_id=s.id
|
||||
AND o.status <> 'closed') AS opportunity_count
|
||||
FROM sources s
|
||||
ORDER BY s.id
|
||||
ORDER BY s.source_category, s.name
|
||||
`);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
@ -354,14 +354,14 @@ app.get("/opportunities", requireWebAuth, async (req, res) => {
|
||||
|
||||
app.get("/", requireWebAuth, async (req, res) => {
|
||||
const sourceRows = await pool.query(
|
||||
`SELECT s.id,s.name,s.source_type,s.source_class,s.publisher,s.authority_tier,
|
||||
`SELECT s.id,s.name,s.source_type,s.source_class,s.source_category,s.publisher,s.authority_tier,
|
||||
s.last_synced_at,s.last_error,s.baseline_complete,
|
||||
COUNT(sig.id)::int AS signal_count
|
||||
FROM sources s
|
||||
LEFT JOIN signals sig ON sig.source_id=s.id
|
||||
GROUP BY s.id,s.name,s.source_type,s.source_class,s.publisher,s.authority_tier,
|
||||
GROUP BY s.id,s.name,s.source_type,s.source_class,s.source_category,s.publisher,s.authority_tier,
|
||||
s.last_synced_at,s.last_error,s.baseline_complete
|
||||
ORDER BY s.id`
|
||||
ORDER BY s.source_category,s.name`
|
||||
);
|
||||
const counts = await pool.query(`
|
||||
SELECT
|
||||
|
||||
@ -30,12 +30,12 @@ import { mediaRoleAtLeast, type MediaRole } from "./auth.js";
|
||||
const tools = [
|
||||
{
|
||||
name: "sources_list",
|
||||
description: "List configured public, market and first-party corporate information sources.",
|
||||
description: "List configured information sources with editorial category, evidence class, authority tier and provenance metadata.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false }
|
||||
},
|
||||
{
|
||||
name: "sources_status",
|
||||
description: "Show sync health and counts for each source.",
|
||||
description: "Show sync health and counts for each source, grouped logically by the source_category field.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false }
|
||||
},
|
||||
{
|
||||
|
||||
@ -10,6 +10,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "geojson",
|
||||
url: "https://api.geonet.org.nz/quake?MMI=3",
|
||||
publisher: "GeoNet",
|
||||
category: "infrastructure_hazards",
|
||||
sourceClass: "official_data",
|
||||
independenceGroup: "geonet",
|
||||
authorityTier: 1,
|
||||
@ -20,6 +21,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "html",
|
||||
url: "https://www3.parliament.nz/en/pb/sc/evidence-submissions/",
|
||||
publisher: "New Zealand Parliament",
|
||||
category: "government_civic",
|
||||
sourceClass: "official_primary",
|
||||
independenceGroup: "parliament",
|
||||
authorityTier: 1,
|
||||
@ -30,6 +32,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "sitemap",
|
||||
url: "https://www.stats.govt.nz/sitemap.xml",
|
||||
publisher: "Stats NZ",
|
||||
category: "economy_labour",
|
||||
sourceClass: "official_data",
|
||||
independenceGroup: "stats-nz",
|
||||
authorityTier: 1,
|
||||
@ -40,6 +43,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "ckan",
|
||||
url: "https://catalogue.data.govt.nz/api/3/action/package_search?sort=metadata_modified%20desc&rows=25",
|
||||
publisher: "New Zealand Government Data Catalogue",
|
||||
category: "government_civic",
|
||||
sourceClass: "catalogue",
|
||||
independenceGroup: "data-govt-nz",
|
||||
authorityTier: 2,
|
||||
@ -50,6 +54,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "rss",
|
||||
url: "https://www.treasury.govt.nz/feeds/publications",
|
||||
publisher: "New Zealand Treasury",
|
||||
category: "economy_labour",
|
||||
sourceClass: "official_analysis",
|
||||
independenceGroup: "treasury",
|
||||
authorityTier: 1,
|
||||
@ -60,6 +65,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "html",
|
||||
url: "https://www.rbnz.govt.nz/statistics/series/data-file-index-page",
|
||||
publisher: "Reserve Bank of New Zealand",
|
||||
category: "economy_labour",
|
||||
sourceClass: "official_data",
|
||||
independenceGroup: "rbnz",
|
||||
authorityTier: 1,
|
||||
@ -70,6 +76,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "html",
|
||||
url: "https://www.mbie.govt.nz/business-and-employment/employment-and-skills/labour-market-reports-data-and-analysis/jobs-online",
|
||||
publisher: "Ministry of Business, Innovation and Employment",
|
||||
category: "economy_labour",
|
||||
sourceClass: "official_data",
|
||||
independenceGroup: "mbie",
|
||||
authorityTier: 1,
|
||||
@ -80,6 +87,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "html",
|
||||
url: "https://www.companiesoffice.govt.nz/insights-and-articles/latest-company-statistics/",
|
||||
publisher: "New Zealand Companies Office",
|
||||
category: "markets_business",
|
||||
sourceClass: "official_registry",
|
||||
independenceGroup: "companies-office",
|
||||
authorityTier: 1,
|
||||
@ -90,6 +98,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "html",
|
||||
url: "https://gazette.govt.nz/home/search?from=0&sortField=publish_date&sortOrder=desc",
|
||||
publisher: "New Zealand Gazette",
|
||||
category: "government_civic",
|
||||
sourceClass: "official_notice",
|
||||
independenceGroup: "nz-gazette",
|
||||
authorityTier: 1,
|
||||
@ -100,6 +109,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "html",
|
||||
url: "https://www.ea.govt.nz/your-power/keeping-the-lights-on/electricity-security-of-supply/",
|
||||
publisher: "Electricity Authority",
|
||||
category: "infrastructure_hazards",
|
||||
sourceClass: "official_data",
|
||||
independenceGroup: "electricity-authority",
|
||||
authorityTier: 1,
|
||||
@ -110,6 +120,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "html",
|
||||
url: "https://announcements.nzx.com/",
|
||||
publisher: "NZX Limited",
|
||||
category: "markets_business",
|
||||
sourceClass: "official_market_notice",
|
||||
independenceGroup: "nzx",
|
||||
authorityTier: 1,
|
||||
@ -120,6 +131,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "rss",
|
||||
url: "https://blog.google/rss/",
|
||||
publisher: "Google",
|
||||
category: "technology_ai",
|
||||
sourceClass: "corporate_primary",
|
||||
independenceGroup: "google",
|
||||
authorityTier: 1,
|
||||
@ -130,6 +142,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "html",
|
||||
url: "https://about.fb.com/news/",
|
||||
publisher: "Meta",
|
||||
category: "technology_ai",
|
||||
sourceClass: "corporate_primary",
|
||||
independenceGroup: "meta",
|
||||
authorityTier: 1,
|
||||
@ -140,6 +153,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "html",
|
||||
url: "https://www.anthropic.com/news",
|
||||
publisher: "Anthropic",
|
||||
category: "technology_ai",
|
||||
sourceClass: "corporate_primary",
|
||||
independenceGroup: "anthropic",
|
||||
authorityTier: 1,
|
||||
@ -150,6 +164,7 @@ export const SOURCES: SourceDefinition[] = [
|
||||
type: "rss",
|
||||
url: "https://openai.com/news/rss.xml",
|
||||
publisher: "OpenAI",
|
||||
category: "technology_ai",
|
||||
sourceClass: "corporate_primary",
|
||||
independenceGroup: "openai",
|
||||
authorityTier: 1,
|
||||
|
||||
11
src/types.ts
11
src/types.ts
@ -15,12 +15,23 @@ export type SourceId =
|
||||
| "anthropic-news"
|
||||
| "openai-news";
|
||||
|
||||
export type SourceCategory =
|
||||
| "government_civic"
|
||||
| "economy_labour"
|
||||
| "markets_business"
|
||||
| "infrastructure_hazards"
|
||||
| "technology_ai"
|
||||
| "news_media"
|
||||
| "research"
|
||||
| "other";
|
||||
|
||||
export interface SourceDefinition {
|
||||
id: SourceId;
|
||||
name: string;
|
||||
type: "geojson" | "html" | "sitemap" | "ckan" | "rss";
|
||||
url: string;
|
||||
publisher: string;
|
||||
category: SourceCategory;
|
||||
sourceClass:
|
||||
| "official_primary"
|
||||
| "official_data"
|
||||
|
||||
30
src/web.ts
30
src/web.ts
@ -154,6 +154,10 @@ button,input,select{font:inherit}
|
||||
.callout.warning{border-left-color:var(--amber)}
|
||||
.callout.success{border-left-color:var(--green)}
|
||||
.source-row{display:block}
|
||||
.source-category-group+.source-category-group{border-top:1px solid #e4e9ef;margin-top:8px;padding-top:8px}
|
||||
.source-category-head{display:flex;align-items:center;justify-content:space-between;gap:10px;padding:10px 5px 5px}
|
||||
.source-category-title{font-size:.74rem;text-transform:uppercase;letter-spacing:.07em;font-weight:820;color:#596477}
|
||||
.source-category-summary{font-size:.72rem;color:var(--muted);white-space:nowrap}
|
||||
.dot{width:8px;height:8px;border-radius:50%;display:inline-block;margin-right:6px;background:#94a3b8}
|
||||
.dot.ok{background:#22a06b}.dot.bad{background:#d04f4f}.dot.warn{background:#d78a1f}.attention-grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:10px;margin-bottom:20px}.attention-card{display:block;text-decoration:none;background:#fff;border:1px solid var(--line);border-radius:12px;padding:14px 15px;box-shadow:var(--shadow);transition:transform .12s ease,border-color .12s ease}.attention-card:hover{transform:translateY(-1px);border-color:#c8d0db}.attention-top{display:flex;align-items:center;justify-content:space-between;gap:8px}.attention-label{font-size:.78rem;color:var(--muted);font-weight:700}.attention-value{font-size:1.45rem;font-weight:800;letter-spacing:-.03em;margin-top:4px}.attention-copy{font-size:.78rem;color:var(--muted);margin-top:3px}.role-pill{display:inline-flex;align-items:center;border:1px solid rgba(255,255,255,.13);border-radius:999px;padding:2px 7px;font-size:.68rem;color:#cbd5e1;margin-left:5px;text-transform:capitalize}.live-pill{display:inline-flex;align-items:center;gap:5px;border:1px solid rgba(255,255,255,.13);border-radius:999px;padding:3px 7px;font-size:.66rem;color:#9ee7c5;text-transform:uppercase;letter-spacing:.05em}.live-pill::before{content:"";width:6px;height:6px;border-radius:50%;background:currentColor}.live-pill.reconnecting{color:#f0c57a}.thread-jumps{position:sticky;top:64px;z-index:35;display:flex;gap:6px;flex-wrap:wrap;margin:15px 0 2px;padding:7px;background:rgba(243,245,247,.92);backdrop-filter:blur(8px);border:1px solid rgba(214,220,228,.9);border-radius:11px}.thread-jumps a{font-size:.77rem;text-decoration:none;color:#536075;padding:6px 9px;border-radius:7px;font-weight:650}.thread-jumps a:hover{background:#fff;color:#172033}.progress-line{height:6px;background:#edf0f4;border-radius:999px;overflow:hidden;margin-top:10px}.progress-line span{display:block;height:100%;background:#16805c;border-radius:999px}.trust-summary{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:10px;margin-bottom:14px}.trust-stat{padding:13px 14px;background:#f8fafc;border:1px solid #e6eaf0;border-radius:10px}.trust-stat strong{display:block;font-size:1.15rem;letter-spacing:-.025em}.trust-stat span{font-size:.76rem;color:var(--muted)}
|
||||
.thread-hero{display:grid;grid-template-columns:minmax(0,1fr) auto;gap:22px;align-items:start}
|
||||
@ -539,6 +543,22 @@ export function renderOpportunities(args:{opportunities:AnyRow[];identity:string
|
||||
|
||||
export function renderDashboard(args:{identity:string;role?:string;counts:AnyRow;messages:AnyRow[];sources:AnyRow[];stories:AnyRow[]}) {
|
||||
const c=args.counts??{}, sourceErrors=args.sources.filter((s)=>s.last_error), storyAttention=args.stories.filter((s)=>!s.source_audit?.robust && s.status!=="closed");
|
||||
const categoryMeta:Record<string,{label:string;order:number}> = {
|
||||
government_civic:{label:"Government & civic",order:10},
|
||||
economy_labour:{label:"Economy & labour",order:20},
|
||||
markets_business:{label:"Markets & business",order:30},
|
||||
infrastructure_hazards:{label:"Infrastructure & hazards",order:40},
|
||||
technology_ai:{label:"Technology & AI",order:50},
|
||||
news_media:{label:"News & media",order:60},
|
||||
research:{label:"Research",order:70},
|
||||
other:{label:"Other",order:99}
|
||||
};
|
||||
const sourceGroups=Array.from(args.sources.reduce((groups:Map<string,AnyRow[]>,source:AnyRow)=>{
|
||||
const category=String(source.source_category||"other");
|
||||
if(!groups.has(category)) groups.set(category,[]);
|
||||
groups.get(category)!.push(source);
|
||||
return groups;
|
||||
},new Map<string,AnyRow[]>()).entries()).sort(([a],[b])=>(categoryMeta[a]?.order??99)-(categoryMeta[b]?.order??99));
|
||||
const body=`<div class="page-head"><div><div class="eyebrow">Newsroom desk</div><h1>Media Intelligence</h1><p>Public-source monitoring, evidence research and human editorial control in one shared workspace.</p></div><div class="actions"><a class="btn btn-primary" href="/stories">Open research desk</a></div></div>
|
||||
<div class="grid grid-metrics">
|
||||
<div class="panel metric"><div class="metric-value">${compact(c.signals)}</div><div class="metric-label">stored signals</div></div>
|
||||
@ -552,7 +572,11 @@ export function renderDashboard(args:{identity:string;role?:string;counts:AnyRow
|
||||
<div class="panel">${storyAttention.length?storyAttention.slice(0,6).map((s:AnyRow)=>`<a class="story-row" href="/stories/${encodeURIComponent(s.id)}"><div class="list-row"><div><div class="row-title">${esc(s.title)}</div><div class="row-meta">${badge(s.status,statusTone(s.status))}${badge(`${s.source_audit?.counts?.unsupported_claims??0} unsupported`,(s.source_audit?.counts?.unsupported_claims??0)?"red":"green")}${badge(`${s.source_audit?.counts?.distinct_independence_groups??0} source groups`,"amber")}</div></div><div class="right-meta meta">${esc(nz(s.updated_at))}</div></div></a>`).join(""):`<div class="empty">No sourcing gaps in active threads.</div>`}</div>
|
||||
</div>
|
||||
<div id="source-health"><div class="section-head"><div><h2>Source health</h2><p>Current ingestion state.</p></div></div>
|
||||
<div class="panel">${args.sources.map((s:AnyRow)=>`<div class="source-row"><div class="list-row"><div><div class="row-title"><span class="dot ${s.last_error?"bad":s.baseline_complete?"ok":"warn"}"></span>${esc(s.name)}</div><div class="row-meta">${badge(s.source_class??s.source_type,"neutral")}${s.authority_tier?badge(`Tier ${s.authority_tier}`,"neutral"):""}</div></div><div class="right-meta meta">${compact(s.signal_count)} signals<br>${s.last_error?"error":esc(nz(s.last_synced_at))}</div></div></div>`).join("")}</div>
|
||||
<div class="panel">${sourceGroups.map(([category,sources])=>{
|
||||
const meta=categoryMeta[category]??categoryMeta.other;
|
||||
const errors=sources.filter((s:AnyRow)=>s.last_error).length;
|
||||
return `<div class="source-category-group"><div class="source-category-head"><div class="source-category-title">${esc(meta.label)}</div><div class="source-category-summary">${sources.length} source${sources.length===1?"":"s"}${errors?` · ${errors} issue${errors===1?"":"s"}`:""}</div></div>${sources.map((s:AnyRow)=>`<div class="source-row"><div class="list-row"><div><div class="row-title"><span class="dot ${s.last_error?"bad":s.baseline_complete?"ok":"warn"}"></span>${esc(s.name)}</div><div class="row-meta">${badge(s.source_class??s.source_type,"neutral")}${s.authority_tier?badge(`Tier ${s.authority_tier}`,"neutral"):""}</div></div><div class="right-meta meta">${compact(s.signal_count)} signals<br>${s.last_error?"error":esc(nz(s.last_synced_at))}</div></div></div>`).join("")}</div>`;
|
||||
}).join("")}</div>
|
||||
${sourceErrors.length?`<div class="panel callout warning" style="margin-top:10px"><strong>${sourceErrors.length} source issue${sourceErrors.length===1?"":"s"}</strong><div class="meta" style="margin-top:4px">${sourceErrors.map((s)=>esc(s.name)).join(" · ")}</div></div>`:""}
|
||||
</div>
|
||||
</div>
|
||||
@ -640,7 +664,7 @@ export function renderAboutNewsroom(args:{
|
||||
}) {
|
||||
const enabledSources = args.sources.filter((source:AnyRow)=>source.enabled !== false);
|
||||
const healthySources = enabledSources.filter((source:AnyRow)=>!source.last_error);
|
||||
const sourceClasses = Array.from(new Set(enabledSources.map((source:AnyRow)=>String(source.source_class||"other")))).sort();
|
||||
const sourceCategories = Array.from(new Set(enabledSources.map((source:AnyRow)=>String(source.source_category||"other")))).sort();
|
||||
const trustEnabled = args.trustEntries.filter((entry:AnyRow)=>entry.enabled !== false);
|
||||
|
||||
const body = `
|
||||
@ -670,7 +694,7 @@ export function renderAboutNewsroom(args:{
|
||||
<div class="grid grid-metrics">
|
||||
<div class="panel metric"><div class="metric-value">${compact(enabledSources.length)}</div><div class="metric-label">enabled sources</div></div>
|
||||
<div class="panel metric"><div class="metric-value">${compact(healthySources.length)}</div><div class="metric-label">currently without sync errors</div></div>
|
||||
<div class="panel metric"><div class="metric-value">${compact(sourceClasses.length)}</div><div class="metric-label">source classes</div></div>
|
||||
<div class="panel metric"><div class="metric-value">${compact(sourceCategories.length)}</div><div class="metric-label">source categories</div></div>
|
||||
<div class="panel metric"><div class="metric-value">${compact(trustEnabled.length)}</div><div class="metric-label">enabled trust-policy entries</div></div>
|
||||
</div>
|
||||
<div class="panel" style="margin-top:14px">
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user