diff --git a/src/db.ts b/src/db.ts index 38af0f2..45f97fa 100644 --- a/src/db.ts +++ b/src/db.ts @@ -713,19 +713,19 @@ function buildStorySourceAudit(story: any) { Number(n.provenance_authority_tier) === 1 ).length; - const evidentiaryTypes = new Set(["evidence","actual","forecast","trend","signal","metric"]); + const directEvidenceTypes = new Set(["evidence","actual","signal"]); const sourceTaggedButUnbacked = nodes - .filter((n: any) => evidentiaryTypes.has(n.node_type) && n.source_id && !n.evidence_id) + .filter((n: any) => directEvidenceTypes.has(n.node_type) && n.source_id && !n.evidence_id) .map((n: any) => ({ id: n.id, title: n.title, node_type: n.node_type, source_id: n.source_id })); - const claimTypes = new Set(["claim","counterclaim"]); + const assertionTypes = new Set(["claim","counterclaim","forecast","trend","metric"]); const evidenceNodeIds = new Set(evidenceBacked.map((n: any) => n.id)); const supportingRelationships = new Set([ "supports","contradicts","qualifies","forecast_for","actual_for","regional_breakdown_of" ]); const unsupportedClaims = nodes - .filter((n: any) => claimTypes.has(n.node_type)) + .filter((n: any) => assertionTypes.has(n.node_type)) .filter((claim: any) => !edges.some((edge: any) => edge.to_node_id === claim.id && evidenceNodeIds.has(edge.from_node_id) && @@ -746,8 +746,8 @@ function buildStorySourceAudit(story: any) { at_least_two_independent_sources: distinctGroups.length >= 2, at_least_two_evidence_backed_nodes: evidenceBacked.length >= 2, at_least_one_tier_one_source: tierOneCount >= 1, - all_claims_have_direct_evidence: unsupportedClaims.length === 0, - no_source_tag_only_evidence_nodes: sourceTaggedButUnbacked.length === 0 + all_assertions_have_direct_evidence: unsupportedClaims.length === 0, + no_source_tag_only_direct_evidence_nodes: sourceTaggedButUnbacked.length === 0 }; return { @@ -879,6 +879,10 @@ export async function addStoryNode( const thread = await pool.query("SELECT id FROM story_threads WHERE id=$1", [threadId]); if (!thread.rowCount) throw new Error("Story thread not found"); + if (args.evidenceId) { + await enrichEvidenceDocument(args.evidenceId); + } + const id = randomUUID(); const result = await pool.query( `INSERT INTO story_nodes( diff --git a/src/documents.ts b/src/documents.ts index 312c28b..538eca8 100644 --- a/src/documents.ts +++ b/src/documents.ts @@ -2,12 +2,85 @@ import { createHash } from "node:crypto"; import { load } from "cheerio"; const DOCUMENT_UA = - "Mozilla/5.0 (compatible; ArrowstackMediaIntelligence/0.1; +https://media.arrowstack.dev)"; + "Mozilla/5.0 (compatible; ArrowstackMediaIntelligenceBot/0.1; +https://media.arrowstack.dev)"; function normaliseText(value: string) { return value.replace(/\u00a0/g, " ").replace(/\s+/g, " ").trim(); } +function collectJsonStrings(value: unknown, out: string[], depth = 0) { + if (depth > 14 || out.length > 4000) return; + if (typeof value === "string") { + const clean = normaliseText(value); + if (clean.length >= 18 && !/^https?:\/\//i.test(clean)) out.push(clean); + return; + } + if (Array.isArray(value)) { + for (const item of value) collectJsonStrings(item, out, depth + 1); + return; + } + if (value && typeof value === "object") { + for (const item of Object.values(value as Record)) { + collectJsonStrings(item, out, depth + 1); + } + } +} + +function extractHtmlDocument(html: string, baseUrl: string) { + const $ = load(html); + const title = normaliseText($("h1").first().text() || $("title").first().text()); + let canonicalUrl = baseUrl; + const canonical = $('link[rel="canonical"]').attr("href"); + if (canonical) { + try { canonicalUrl = new URL(canonical, baseUrl).toString(); } catch {} + } + + const structured: string[] = []; + $('script[type="application/json"],script[type="application/ld+json"],script#__NEXT_DATA__').each((_i, el) => { + const raw = $(el).html() ?? ""; + if (!raw.trim()) return; + try { + collectJsonStrings(JSON.parse(raw), structured); + } catch { + // Some frameworks emit escaped JSON fragments rather than strict JSON. + const matches = raw.match(/"([^"\\]{18,})"/g) ?? []; + for (const match of matches.slice(0, 1000)) { + structured.push(normaliseText(match.slice(1, -1).replace(/\\n/g, " "))); + } + } + }); + + $("script,style,noscript,svg,nav,footer,header").remove(); + const candidates = [ + $("main").first(), + $("article").first(), + $('[role="main"]').first(), + $("#content").first(), + $(".main-content").first(), + $("body").first() + ].filter((node) => node.length); + + let text = candidates + .map((node) => normaliseText(node.text())) + .sort((a, b) => b.length - a.length)[0] ?? ""; + + if (text.length < 300 && structured.length) { + const deduped = Array.from(new Set(structured)); + text = deduped.join(" ").slice(0, 120000); + } + + const description = normaliseText( + $('meta[name="description"]').attr("content") ?? + $('meta[property="og:description"]').attr("content") ?? + "" + ); + if (text.length < 300 && description) { + text = [title, description].filter(Boolean).join(". "); + } + + return { title, canonicalUrl, text }; +} + export interface SourceDocumentSnapshot { requestedUrl: string; canonicalUrl: string; @@ -52,27 +125,10 @@ export async function fetchSourceDocument(url: string): Promise node.length); - - text = candidates - .map((node) => normaliseText(node.text())) - .sort((a, b) => b.length - a.length)[0] ?? ""; + const extracted = extractHtmlDocument(html, canonicalUrl); + title = extracted.title; + canonicalUrl = extracted.canonicalUrl; + text = extracted.text; } else if ( contentType.includes("json") || contentType.startsWith("text/") || @@ -81,8 +137,10 @@ export async function fetchSourceDocument(url: string): Promise