Sync stronger source document extraction

src/db.ts

@@ -713,19 +713,19 @@ function buildStorySourceAudit(story: any) {
     Number(n.provenance_authority_tier) === 1
   ).length;
 
-  const evidentiaryTypes = new Set(["evidence","actual","forecast","trend","signal","metric"]);
+  const directEvidenceTypes = new Set(["evidence","actual","signal"]);
   const sourceTaggedButUnbacked = nodes
-    .filter((n: any) => evidentiaryTypes.has(n.node_type) && n.source_id && !n.evidence_id)
+    .filter((n: any) => directEvidenceTypes.has(n.node_type) && n.source_id && !n.evidence_id)
     .map((n: any) => ({ id: n.id, title: n.title, node_type: n.node_type, source_id: n.source_id }));
 
-  const claimTypes = new Set(["claim","counterclaim"]);
+  const assertionTypes = new Set(["claim","counterclaim","forecast","trend","metric"]);
   const evidenceNodeIds = new Set(evidenceBacked.map((n: any) => n.id));
   const supportingRelationships = new Set([
     "supports","contradicts","qualifies","forecast_for","actual_for","regional_breakdown_of"
   ]);
 
   const unsupportedClaims = nodes
-    .filter((n: any) => claimTypes.has(n.node_type))
+    .filter((n: any) => assertionTypes.has(n.node_type))
     .filter((claim: any) => !edges.some((edge: any) =>
       edge.to_node_id === claim.id &&
       evidenceNodeIds.has(edge.from_node_id) &&
@@ -746,8 +746,8 @@ function buildStorySourceAudit(story: any) {
     at_least_two_independent_sources: distinctGroups.length >= 2,
     at_least_two_evidence_backed_nodes: evidenceBacked.length >= 2,
     at_least_one_tier_one_source: tierOneCount >= 1,
-    all_claims_have_direct_evidence: unsupportedClaims.length === 0,
+    all_assertions_have_direct_evidence: unsupportedClaims.length === 0,
-    no_source_tag_only_evidence_nodes: sourceTaggedButUnbacked.length === 0
+    no_source_tag_only_direct_evidence_nodes: sourceTaggedButUnbacked.length === 0
   };
 
   return {
@@ -879,6 +879,10 @@ export async function addStoryNode(
   const thread = await pool.query("SELECT id FROM story_threads WHERE id=$1", [threadId]);
   if (!thread.rowCount) throw new Error("Story thread not found");
 
+  if (args.evidenceId) {
+    await enrichEvidenceDocument(args.evidenceId);
+  }
+
   const id = randomUUID();
   const result = await pool.query(
     `INSERT INTO story_nodes(

src/documents.ts

@@ -2,12 +2,85 @@ import { createHash } from "node:crypto";
 import { load } from "cheerio";
 
 const DOCUMENT_UA =
-  "Mozilla/5.0 (compatible; ArrowstackMediaIntelligence/0.1; +https://media.arrowstack.dev)";
+  "Mozilla/5.0 (compatible; ArrowstackMediaIntelligenceBot/0.1; +https://media.arrowstack.dev)";
 
 function normaliseText(value: string) {
   return value.replace(/\u00a0/g, " ").replace(/\s+/g, " ").trim();
 }
 
+function collectJsonStrings(value: unknown, out: string[], depth = 0) {
+  if (depth > 14 || out.length > 4000) return;
+  if (typeof value === "string") {
+    const clean = normaliseText(value);
+    if (clean.length >= 18 && !/^https?:\/\//i.test(clean)) out.push(clean);
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) collectJsonStrings(item, out, depth + 1);
+    return;
+  }
+  if (value && typeof value === "object") {
+    for (const item of Object.values(value as Record<string, unknown>)) {
+      collectJsonStrings(item, out, depth + 1);
+    }
+  }
+}
+
+function extractHtmlDocument(html: string, baseUrl: string) {
+  const $ = load(html);
+  const title = normaliseText($("h1").first().text() || $("title").first().text());
+  let canonicalUrl = baseUrl;
+  const canonical = $('link[rel="canonical"]').attr("href");
+  if (canonical) {
+    try { canonicalUrl = new URL(canonical, baseUrl).toString(); } catch {}
+  }
+
+  const structured: string[] = [];
+  $('script[type="application/json"],script[type="application/ld+json"],script#__NEXT_DATA__').each((_i, el) => {
+    const raw = $(el).html() ?? "";
+    if (!raw.trim()) return;
+    try {
+      collectJsonStrings(JSON.parse(raw), structured);
+    } catch {
+      // Some frameworks emit escaped JSON fragments rather than strict JSON.
+      const matches = raw.match(/"([^"\\]{18,})"/g) ?? [];
+      for (const match of matches.slice(0, 1000)) {
+        structured.push(normaliseText(match.slice(1, -1).replace(/\\n/g, " ")));
+      }
+    }
+  });
+
+  $("script,style,noscript,svg,nav,footer,header").remove();
+  const candidates = [
+    $("main").first(),
+    $("article").first(),
+    $('[role="main"]').first(),
+    $("#content").first(),
+    $(".main-content").first(),
+    $("body").first()
+  ].filter((node) => node.length);
+
+  let text = candidates
+    .map((node) => normaliseText(node.text()))
+    .sort((a, b) => b.length - a.length)[0] ?? "";
+
+  if (text.length < 300 && structured.length) {
+    const deduped = Array.from(new Set(structured));
+    text = deduped.join(" ").slice(0, 120000);
+  }
+
+  const description = normaliseText(
+    $('meta[name="description"]').attr("content") ??
+    $('meta[property="og:description"]').attr("content") ??
+    ""
+  );
+  if (text.length < 300 && description) {
+    text = [title, description].filter(Boolean).join(". ");
+  }
+
+  return { title, canonicalUrl, text };
+}
+
 export interface SourceDocumentSnapshot {
   requestedUrl: string;
   canonicalUrl: string;
@@ -52,27 +125,10 @@ export async function fetchSourceDocument(url: string): Promise<SourceDocumentSn
       throw new Error(`Source document blocked by upstream WAF for ${url}`);
     }
 
-    const $ = load(html);
+    const extracted = extractHtmlDocument(html, canonicalUrl);
-    $("script,style,noscript,svg,nav,footer,header").remove();
+    title = extracted.title;
-    title = normaliseText($("h1").first().text() || $("title").first().text());
+    canonicalUrl = extracted.canonicalUrl;
-
+    text = extracted.text;
-    const canonical = $('link[rel="canonical"]').attr("href");
-    if (canonical) {
-      try { canonicalUrl = new URL(canonical, canonicalUrl).toString(); } catch {}
-    }
-
-    const candidates = [
-      $("main").first(),
-      $("article").first(),
-      $('[role="main"]').first(),
-      $("#content").first(),
-      $(".main-content").first(),
-      $("body").first()
-    ].filter((node) => node.length);
-
-    text = candidates
-      .map((node) => normaliseText(node.text()))
-      .sort((a, b) => b.length - a.length)[0] ?? "";
   } else if (
     contentType.includes("json") ||
     contentType.startsWith("text/") ||
@@ -81,8 +137,10 @@ export async function fetchSourceDocument(url: string): Promise<SourceDocumentSn
     text = normaliseText(bytes.toString("utf8"));
   }
 
-  if (!text) {
+  if (!text || text.length < 80) {
-    throw new Error(`Source document is not text-extractable (${contentType}) for ${url}`);
+    throw new Error(
+      `Source document produced insufficient extractable text (${contentType}, bytes=${bytes.length}, extracted=${text.length}) for ${url}`
+    );
   }
 
   return {