Sync About the Newsroom documentation page

src/index.ts

@@ -33,7 +33,7 @@ import {
 import { handleMcp } from "./mcp.js";
 import { SOURCES } from "./sources.js";
 import { syncAll } from "./sync.js";
-import { renderDashboard, renderOpportunities, renderOpportunity, renderStories, renderStory, renderTrustPolicy } from "./web.js";
+import { renderAboutNewsroom, renderDashboard, renderOpportunities, renderOpportunity, renderStories, renderStory, renderTrustPolicy } from "./web.js";
 
 const app = express();
 app.set("trust proxy", true);
@@ -284,6 +284,22 @@ app.post("/opportunities/:id/status", requireWebAuth, requireWebRole("editor"),
   return res.redirect(req.body.returnTo === "detail" ? `/opportunities/${encodeURIComponent(id)}` : "/opportunities");
 });
 
+app.get("/about", requireWebAuth, async (req, res) => {
+  const [sources, trustEntries] = await Promise.all([
+    pool.query("SELECT * FROM sources ORDER BY id"),
+    listSourceTrustPolicy()
+  ]);
+  const user = (req as any).user ?? {};
+  const identity = String(user.name || user.preferred_username || user.email || "Signed-in user");
+  const role = webMediaRole(req);
+  return res.type("html").send(renderAboutNewsroom({
+    identity,
+    role,
+    sources: sources.rows,
+    trustEntries
+  }));
+});
+
 app.get("/trust", requireWebAuth, async (req, res) => {
   const entries = await listSourceTrustPolicy();
   const user = (req as any).user ?? {};

src/web.ts

@@ -62,7 +62,7 @@ function canRole(role: string | undefined, required: "viewer" | "editor" | "admi
 
 function shell(args: {
   title: string;
-  active: "dashboard" | "stories" | "opportunities" | "trust";
+  active: "dashboard" | "stories" | "opportunities" | "trust" | "about";
   identity: string;
   role?: string;
   body: string;
@@ -71,7 +71,8 @@ function shell(args: {
     ["dashboard", "/", "Desk"],
     ["stories", "/stories", "Story Threads"],
     ["opportunities", "/opportunities", "Opportunities"],
-    ["trust", "/trust", "Source Trust"]
+    ["trust", "/trust", "Source Trust"],
+    ["about", "/about", "About"]
   ] as const;
 
   return `<!doctype html>
@@ -198,6 +199,23 @@ button,input,select{font:inherit}
 .review-irrelevant{opacity:.52}
 .review-resolved{border-style:dashed}
 .readonly-note{padding:11px 13px;background:#f7f9fb;border:1px solid #e4e9ef;border-radius:9px;color:#5d6879;font-size:.84rem}
+.about-hero{background:linear-gradient(135deg,#111827,#1f2937);color:#fff;border-radius:18px;padding:30px;box-shadow:var(--shadow)}
+.about-hero .eyebrow{color:#aeb8c7}.about-hero p{color:#d7dde7;max-width:900px;margin:10px 0 0}
+.about-toc{display:flex;gap:8px;flex-wrap:wrap;margin-top:18px}.about-toc a{color:#dbe7ff;text-decoration:none;border:1px solid rgba(255,255,255,.16);border-radius:999px;padding:6px 10px;font-size:.8rem}
+.about-flow{display:grid;grid-template-columns:repeat(6,minmax(0,1fr));gap:8px}
+.flow-step{position:relative;padding:15px 13px;background:#fff;border:1px solid var(--line);border-radius:12px;min-height:118px}
+.flow-step:not(:last-child)::after{content:"→";position:absolute;right:-8px;top:42px;z-index:2;color:#8792a4;font-weight:900}
+.flow-num{font-size:.7rem;color:var(--muted);font-weight:800;text-transform:uppercase;letter-spacing:.08em}.flow-title{font-weight:780;margin-top:6px}.flow-copy{font-size:.8rem;color:var(--muted);margin-top:5px}
+.about-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:14px}
+.about-grid-3{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:14px}
+.about-section{scroll-margin-top:82px}
+.about-section h2{font-size:1.25rem;margin:0 0 7px;letter-spacing:-.02em}.about-section>p{color:var(--muted);max-width:980px}
+.about-list{margin:0;padding-left:19px;color:#465268}.about-list li+li{margin-top:6px}
+.about-table{width:100%;border-collapse:collapse;font-size:.84rem}.about-table th,.about-table td{text-align:left;padding:10px;border-bottom:1px solid #edf0f4;vertical-align:top}.about-table th{font-size:.74rem;text-transform:uppercase;letter-spacing:.06em;color:var(--muted)}
+.principle{padding:18px}.principle h3{margin:0 0 6px;font-size:.96rem}.principle p{margin:0;color:var(--muted);font-size:.86rem}
+.codeflow{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;background:#111827;color:#e5e7eb;border-radius:12px;padding:16px;overflow:auto;white-space:pre;line-height:1.65;font-size:.8rem}
+@media(max-width:1100px){.about-flow{grid-template-columns:repeat(3,minmax(0,1fr))}.flow-step::after{display:none}.about-grid-3{grid-template-columns:1fr 1fr}}
+@media(max-width:760px){.about-flow,.about-grid,.about-grid-3{grid-template-columns:1fr}.about-hero{padding:22px}}
 @media(max-width:850px){.workbench{grid-template-columns:1fr}}
 .trust-form{display:grid;grid-template-columns:1.2fr 1.2fr .6fr 1fr 1fr;gap:10px;align-items:end}
 .trust-form .wide{grid-column:1/-1}
@@ -525,3 +543,185 @@ export function renderTrustPolicy(args:{identity:string;entries:AnyRow[];role?:s
   </script>`;
   return shell({title:"Source Trust",active:"trust",identity:args.identity,role:args.role,body});
 }
+
+
+export function renderAboutNewsroom(args:{
+  identity:string;
+  role?:string;
+  sources:AnyRow[];
+  trustEntries:AnyRow[];
+}) {
+  const enabledSources = args.sources.filter((source:AnyRow)=>source.enabled !== false);
+  const healthySources = enabledSources.filter((source:AnyRow)=>!source.last_error);
+  const sourceClasses = Array.from(new Set(enabledSources.map((source:AnyRow)=>String(source.source_class||"other")))).sort();
+  const trustEnabled = args.trustEntries.filter((entry:AnyRow)=>entry.enabled !== false);
+
+  const body = `
+  <div class="about-hero">
+    <div class="eyebrow">About the newsroom</div>
+    <h1 style="margin:0;font-size:clamp(1.9rem,3.5vw,3rem);letter-spacing:-.045em">How Media Intelligence works</h1>
+    <p>This newsroom is a research and editorial operating system. It continuously watches configured public sources, preserves evidence, links claims to provenance, asks agents to look for contradictory or independent evidence, and then hands the result to people for editorial decisions. It is designed to make research faster without hiding where a claim came from or automating the final editorial judgement.</p>
+    <div class="about-toc">
+      <a href="#workflow">Workflow</a><a href="#sources">Sources</a><a href="#trust">Trust & independence</a><a href="#evidence">Evidence</a><a href="#threads">Story Threads</a><a href="#agents">Agents</a><a href="#human-control">Human control</a><a href="#roles">Roles</a><a href="#auditability">Auditability</a><a href="#limits">Limits</a>
+    </div>
+  </div>
+
+  <section class="section about-section" id="workflow">
+    <div class="section-head"><div><h2>Newsroom workflow</h2><p>The system moves from detection to evidence to a human editorial decision.</p></div></div>
+    <div class="about-flow">
+      <div class="flow-step"><div class="flow-num">01</div><div class="flow-title">Monitor</div><div class="flow-copy">Configured feeds, data sources, notices and official pages are checked on schedule.</div></div>
+      <div class="flow-step"><div class="flow-num">02</div><div class="flow-title">Detect</div><div class="flow-copy">New or changed items become signals with source, timestamp and raw payload metadata.</div></div>
+      <div class="flow-step"><div class="flow-num">03</div><div class="flow-title">Triage</div><div class="flow-copy">Scoring highlights items that match editorial watch terms or source-specific importance rules.</div></div>
+      <div class="flow-step"><div class="flow-num">04</div><div class="flow-title">Research</div><div class="flow-copy">Story Threads connect evidence, claims, counterclaims, questions, trends and relationships.</div></div>
+      <div class="flow-step"><div class="flow-num">05</div><div class="flow-title">Challenge</div><div class="flow-copy">Agents and editors seek independent corroboration and evidence that weakens the first interpretation.</div></div>
+      <div class="flow-step"><div class="flow-num">06</div><div class="flow-title">Decide</div><div class="flow-copy">People decide newsworthiness, framing, final message, recipients and whether anything is sent or published.</div></div>
+    </div>
+  </section>
+
+  <section class="section about-section" id="sources">
+    <div class="section-head"><div><h2>1. Source monitoring</h2><p>The source layer is intentionally mixed: primary records, official datasets, notices, analysis, market data and corporate first-party material can all be monitored, but they are not treated as equivalent evidence.</p></div></div>
+    <div class="grid grid-metrics">
+      <div class="panel metric"><div class="metric-value">${compact(enabledSources.length)}</div><div class="metric-label">enabled sources</div></div>
+      <div class="panel metric"><div class="metric-value">${compact(healthySources.length)}</div><div class="metric-label">currently without sync errors</div></div>
+      <div class="panel metric"><div class="metric-value">${compact(sourceClasses.length)}</div><div class="metric-label">source classes</div></div>
+      <div class="panel metric"><div class="metric-value">${compact(trustEnabled.length)}</div><div class="metric-label">enabled trust-policy entries</div></div>
+    </div>
+    <div class="panel" style="margin-top:14px">
+      <table class="about-table"><thead><tr><th>Class</th><th>What it means</th><th>Typical use</th></tr></thead><tbody>
+        <tr><td><span class="mono">official_primary</span></td><td>Primary institutional record or publication.</td><td>What an institution formally published, filed or stated.</td></tr>
+        <tr><td><span class="mono">official_data</span></td><td>First-party statistical or operational data.</td><td>Metrics, series, releases and machine-readable evidence.</td></tr>
+        <tr><td><span class="mono">official_registry</span></td><td>Authoritative registry records.</td><td>Companies, registrations, removals and formal status.</td></tr>
+        <tr><td><span class="mono">official_notice</span></td><td>Legally or formally published notices.</td><td>Gazette notices, market announcements and similar events.</td></tr>
+        <tr><td><span class="mono">official_analysis</span></td><td>Analysis published by an authoritative institution.</td><td>Forecasts, commentary and analytical reports.</td></tr>
+        <tr><td><span class="mono">official_market_notice / official_market_data</span></td><td>Exchange announcements or direct market observations.</td><td>Company disclosures, price/volume context and market events.</td></tr>
+        <tr><td><span class="mono">corporate_primary</span></td><td>A company speaking for itself.</td><td>Strong evidence of what the company announced; not independent validation of its claims.</td></tr>
+        <tr><td><span class="mono">independent_research / newsroom / specialist</span></td><td>External analysis and reporting.</td><td>Independent corroboration, criticism, interpretation and subject expertise.</td></tr>
+      </tbody></table>
+    </div>
+  </section>
+
+  <section class="section about-section" id="trust">
+    <div class="section-head"><div><h2>2. Trust weights and independence</h2><p>A trust weight is an editorial prior about a source, not a probability that the source is telling the truth.</p></div></div>
+    <div class="about-grid">
+      <div class="panel principle"><h3>Trust weight</h3><p>Each configured domain can carry a 0–1 editorial weight. The weight represents how useful and authoritative that source generally is for the type of evidence it publishes. Editors can change these weights in Source Trust.</p></div>
+      <div class="panel principle"><h3>Independence group</h3><p>Multiple URLs owned by the same publisher belong to one independence group. Ten articles from one organisation therefore do not become ten independent confirmations.</p></div>
+      <div class="panel principle"><h3>Primary ≠ independent</h3><p>A company newsroom is excellent evidence that the company made an announcement. It is not independent evidence that the company's performance, safety or impact claims are correct.</p></div>
+      <div class="panel principle"><h3>Corroboration bonus</h3><p>Agents evaluate the strongest source in each independence group. Additional genuinely independent groups can strengthen a source set, while duplicates inside one group do not inflate confidence.</p></div>
+    </div>
+    <div class="codeflow" style="margin-top:14px">OpenAI announcement  ─┐
+                     ├─ independence group: openai
+OpenAI documentation ─┘
+
+Independent benchmark ─── independence group: benchmark-provider
+University research  ──── independence group: university-x
+
+Result: three documents, but three independent groups only if their underlying evidence is genuinely independent.</div>
+  </section>
+
+  <section class="section about-section" id="evidence">
+    <div class="section-head"><div><h2>3. Evidence preservation</h2><p>The newsroom distinguishes discovery metadata from substantive evidence so an attractive headline cannot masquerade as a verified source.</p></div></div>
+    <div class="about-grid-3">
+      <div class="panel principle"><h3>Discovery metadata</h3><p>A title, sitemap entry or listing tells the newsroom that something exists. It is useful for discovery, but usually does not satisfy the sourcing gate.</p></div>
+      <div class="panel principle"><h3>Source payload</h3><p>First-party RSS, APIs and structured payloads can be preserved directly when they contain enough substantive content and provenance.</p></div>
+      <div class="panel principle"><h3>Source document</h3><p>The strongest class in the current system: the actual page or document is fetched, text is extracted, hashed, timestamped and frozen as evidence.</p></div>
+    </div>
+    <div class="panel" style="margin-top:14px">
+      <p style="margin-top:0"><strong>Why freeze evidence?</strong> Web pages change. A Story Thread should be able to show what the newsroom actually saw when a claim was researched, including the source URL, retrieval time, content hash, excerpt and snapshot identity.</p>
+      <p style="margin-bottom:0;color:var(--muted)">The original URL remains available for readers, but the frozen snapshot is the provenance record used by the research graph.</p>
+    </div>
+  </section>
+
+  <section class="section about-section" id="threads">
+    <div class="section-head"><div><h2>4. Story Threads and 360° research</h2><p>A Story Thread is not an article draft. It is the research object from which an editorial decision can later be made.</p></div></div>
+    <div class="about-grid">
+      <div class="panel"><h3 style="margin-top:0">Graph model</h3><p class="muted">A thread can contain evidence, claims, counterclaims, trends, forecasts, actuals, questions, context, organisations, people, places and metrics. Relationships state why one node supports, contradicts, qualifies, updates or investigates another.</p></div>
+      <div class="panel"><h3 style="margin-top:0">360° rule</h3><p class="muted">For every apparent story, the newsroom asks: <em>what does this appear to say, and what is the strongest evidence that this interpretation is incomplete or wrong?</em> Counter-evidence is a required research pass, not an optional decoration.</p></div>
+    </div>
+    <div class="codeflow" style="margin-top:14px">Primary signal
+      ↓
+Initial interpretation
+      ↓
+Direct supporting evidence
+      ↓
+Counter-evidence / alternative explanation
+      ↓
+Independent source groups
+      ↓
+Historical / regional / industry context
+      ↓
+Unresolved questions
+      ↓
+Sourcing gate
+      ↓
+Human editorial review</div>
+  </section>
+
+  <section class="section about-section" id="agents">
+    <div class="section-head"><div><h2>5. What agents do</h2><p>Agents accelerate research and maintenance. They do not own the editorial decision.</p></div></div>
+    <div class="about-grid-3">
+      <div class="panel principle"><h3>Good agent work</h3><p>Monitor sources, gather evidence, freeze documents, identify entities, compare statistics, find contradictory evidence, propose relationships, audit sourcing gaps and surface source-health problems.</p></div>
+      <div class="panel principle"><h3>MCP tools</h3><p>Agents use the Media Intelligence MCP to search evidence, inspect source health, evaluate trust and independence, create or extend Story Threads, and post research findings into Agent Activity.</p></div>
+      <div class="panel principle"><h3>Transparent output</h3><p>Research actions are attached to identifiable sources, nodes, messages and audit events so a human can inspect why an agent reached a research state.</p></div>
+    </div>
+  </section>
+
+  <section class="section about-section" id="human-control">
+    <div class="section-head"><div><h2>6. Human editorial control</h2><p>The system is intentionally built around explicit human gates.</p></div></div>
+    <div class="panel">
+      <table class="about-table"><thead><tr><th>Decision</th><th>Who owns it</th><th>System role</th></tr></thead><tbody>
+        <tr><td>Is this newsworthy?</td><td><strong>Human editor</strong></td><td>Agents can surface evidence and opportunity signals.</td></tr>
+        <tr><td>What is the message or framing?</td><td><strong>Human editor</strong></td><td>Agents can show competing interpretations and factual constraints.</td></tr>
+        <tr><td>Is the evidence sufficient?</td><td><strong>Human editor</strong></td><td>The sourcing gate makes gaps visible and blocks premature workflow progression.</td></tr>
+        <tr><td>Who should receive it?</td><td><strong>Human editor</strong></td><td>Future contact intelligence can propose audiences, not silently choose them.</td></tr>
+        <tr><td>Send or publish?</td><td><strong>Human editor</strong></td><td>No autonomous publishing or distribution is assumed by the newsroom workflow.</td></tr>
+      </tbody></table>
+    </div>
+  </section>
+
+  <section class="section about-section" id="roles">
+    <div class="section-head"><div><h2>7. Roles and permissions</h2><p>The same permission model applies to the website and MCP actions.</p></div></div>
+    <div class="about-grid-3">
+      <div class="panel principle"><h3>Viewer</h3><p>Read Story Threads, inspect evidence and provenance, browse opportunities, view Source Trust and use read-only research tools.</p></div>
+      <div class="panel principle"><h3>Editor</h3><p>Viewer access plus research mutations, Story Thread workbench actions, triage, status changes, evidence enrichment and Agent Activity actions.</p></div>
+      <div class="panel principle"><h3>Admin</h3><p>Editor access plus newsroom policy/configuration actions such as editing the Source Trust policy. Unrecognised authenticated roles fail safe to viewer.</p></div>
+    </div>
+  </section>
+
+  <section class="section about-section" id="auditability">
+    <div class="section-head"><div><h2>8. Sourcing gate and auditability</h2><p>A thread cannot become editorially ready just because an agent sounds confident.</p></div></div>
+    <div class="about-grid">
+      <div class="panel"><h3 style="margin-top:0">Current sourcing checks</h3><ul class="about-list">
+        <li>At least two independent source groups.</li>
+        <li>At least two substantive evidence-backed nodes.</li>
+        <li>At least one Tier 1 source.</li>
+        <li>Active assertions connected directly to qualifying evidence.</li>
+        <li>No source-label-only node pretending to be direct evidence.</li>
+        <li>A completed counter-evidence pass before review/develop.</li>
+      </ul></div>
+      <div class="panel"><h3 style="margin-top:0">Audit trail</h3><ul class="about-list">
+        <li>Signal ingestion and change detection are recorded.</li>
+        <li>Evidence snapshots have hashes and retrieval timestamps.</li>
+        <li>Story nodes and relationships record their creator.</li>
+        <li>Editorial status changes are recorded.</li>
+        <li>Agent messages and acknowledgements remain visible.</li>
+        <li>Source failures and recoveries appear in newsroom activity.</li>
+      </ul></div>
+    </div>
+  </section>
+
+  <section class="section about-section" id="limits">
+    <div class="section-head"><div><h2>9. What the newsroom does not claim</h2><p>Automation helps organise evidence; it does not remove uncertainty.</p></div></div>
+    <div class="about-grid">
+      <div class="panel principle"><h3>A weight is not truth</h3><p>A highly trusted source can still contain errors. A low-weight source can still publish an important fact. Weights guide research priority and corroboration, not verdicts.</p></div>
+      <div class="panel principle"><h3>Independent publishers can share one upstream source</h3><p>Two newspapers repeating one press release are not genuinely independent evidence. Agents are instructed to check shared upstream provenance.</p></div>
+      <div class="panel principle"><h3>Source health is imperfect</h3><p>WAFs, blocked feeds, changed HTML, missing APIs and licensing restrictions can make sources temporarily unavailable. The Desk surfaces source failures instead of silently hiding them.</p></div>
+      <div class="panel principle"><h3>Research-ready is not publication-ready</h3><p>Passing the sourcing gate means the configured research requirements are met. It does not replace editorial judgement, legal review, subject-matter expertise or final fact checking.</p></div>
+    </div>
+  </section>
+
+  <section class="section">
+    <div class="panel callout"><strong>Operating principle:</strong> the newsroom should make it easy to answer two questions for any important statement: <em>“What is the evidence?”</em> and <em>“What would make us change our mind?”</em></div>
+  </section>`;
+
+  return shell({title:"About the Newsroom",active:"about",identity:args.identity,role:args.role,body});
+}