diff --git a/src/db.ts b/src/db.ts index 45f97fa..e42a1e0 100644 --- a/src/db.ts +++ b/src/db.ts @@ -236,6 +236,36 @@ export async function migrate() { await pool.query("ALTER TABLE source_snapshots ADD COLUMN IF NOT EXISTS document_text TEXT"); await pool.query("ALTER TABLE source_snapshots ADD COLUMN IF NOT EXISTS http_status INTEGER"); + const payloadMarker = await pool.query( + `INSERT INTO app_meta(key,value) + VALUES('evidence_payload_quality_v1','{"applied":true}'::jsonb) + ON CONFLICT(key) DO NOTHING + RETURNING key` + ); + if (payloadMarker.rowCount) { + await pool.query(` + UPDATE source_snapshots ss + SET snapshot_kind='source_payload' + FROM sources src + WHERE ss.source_id=src.id + AND src.source_type IN ('rss','geojson') + AND src.authority_tier=1 + AND ss.snapshot_kind='normalized' + `); + + await pool.query(` + UPDATE evidence e + SET evidence_kind='source_payload', + locator=COALESCE(locator,'official feed/API item') + FROM source_snapshots ss, sources src + WHERE e.snapshot_id=ss.id + AND ss.source_id=src.id + AND ss.snapshot_kind='source_payload' + AND src.authority_tier=1 + AND e.evidence_kind='normalized_excerpt' + `); + } + await pool.query(` INSERT INTO source_snapshots( id,source_id,signal_id,source_url,retrieved_at,published_at,content_hash, @@ -382,6 +412,17 @@ export async function storeItem( recentEnough && scored.score >= minScore; + const sourceMeta = await pool.query( + "SELECT source_type,authority_tier FROM sources WHERE id=$1", + [item.sourceId] + ); + const directPayload = + Number(sourceMeta.rows[0]?.authority_tier) === 1 && + new Set(["rss","geojson"]).has(String(sourceMeta.rows[0]?.source_type ?? "")); + const snapshotKind = directPayload ? "source_payload" : "normalized"; + const evidenceKind = directPayload ? "source_payload" : "normalized_excerpt"; + const locator = directPayload ? "official feed/API item" : "title+summary"; + await pool.query("BEGIN"); try { await pool.query( @@ -418,9 +459,9 @@ export async function storeItem( await pool.query( `INSERT INTO source_snapshots( id,source_id,signal_id,source_url,published_at,content_hash, - content_type,byte_length,snapshot_json + content_type,byte_length,snapshot_json,snapshot_kind ) - VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9::jsonb)`, + VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9::jsonb,$10)`, [ snapshotId, item.sourceId, @@ -428,9 +469,12 @@ export async function storeItem( item.url, item.publishedAt, hash, - "application/vnd.media-intelligence.normalized+json", + directPayload + ? "application/vnd.media-intelligence.source-payload+json" + : "application/vnd.media-intelligence.normalized+json", Buffer.byteLength(snapshotText), - snapshotText + snapshotText, + snapshotKind ] ); @@ -438,14 +482,16 @@ export async function storeItem( `INSERT INTO evidence( id,signal_id,source_url,content_hash,excerpt,snapshot_id,evidence_kind,locator ) - VALUES($1,$2,$3,$4,$5,$6,'normalized_excerpt','title+summary')`, + VALUES($1,$2,$3,$4,$5,$6,$7,$8)`, [ evidenceId, signalId, item.url, hash, `${item.title}\n${item.summary}`.slice(0, 2000), - snapshotId + snapshotId, + evidenceKind, + locator ] ); @@ -690,15 +736,23 @@ function buildStorySourceAudit(story: any) { const nodes = Array.isArray(story.nodes) ? story.nodes : []; const edges = Array.isArray(story.edges) ? story.edges : []; - const evidenceBacked = nodes.filter((n: any) => - Boolean( - n.evidence_id && - n.snapshot_id && - n.provenance_source_id && + const evidenceBacked = nodes.filter((n: any) => { + if (!n.evidence_id || !n.snapshot_id || !n.provenance_source_id) return false; + + const documentBacked = n.snapshot_kind === "source_document" && - n.evidence_kind === "source_document" - ) - ); + n.evidence_kind === "source_document" && + Number(n.snapshot_text_length ?? 0) >= 500; + + const payloadBacked = + n.snapshot_kind === "source_payload" && + n.evidence_kind === "source_payload" && + Number(n.provenance_authority_tier) === 1 && + Number(n.snapshot_payload_length ?? 0) >= 350 && + Number(n.evidence_excerpt_length ?? 0) >= 160; + + return documentBacked || payloadBacked; + }); const distinctSourceIds = Array.from(new Set( evidenceBacked.map((n: any) => String(n.provenance_source_id)).filter(Boolean) @@ -776,6 +830,11 @@ function buildStorySourceAudit(story: any) { snapshot_id: n.snapshot_id, snapshot_retrieved_at: n.snapshot_retrieved_at, snapshot_published_at: n.snapshot_published_at, + evidence_kind: n.evidence_kind, + snapshot_kind: n.snapshot_kind, + snapshot_text_length: n.snapshot_text_length, + snapshot_payload_length: n.snapshot_payload_length, + evidence_excerpt_length: n.evidence_excerpt_length, url: n.evidence_url ?? n.signal_url })), unsupported_claims: unsupportedClaims, @@ -812,7 +871,10 @@ export async function getStoryThread(id: string) { ss.id AS snapshot_id, ss.snapshot_kind AS snapshot_kind, ss.retrieved_at AS snapshot_retrieved_at, - ss.published_at AS snapshot_published_at + ss.published_at AS snapshot_published_at, + LENGTH(COALESCE(ss.document_text,''))::int AS snapshot_text_length, + LENGTH(COALESCE(ss.snapshot_json::text,''))::int AS snapshot_payload_length, + LENGTH(COALESCE(e.excerpt,''))::int AS evidence_excerpt_length FROM story_nodes n LEFT JOIN evidence e ON e.id=n.evidence_id LEFT JOIN signals s ON s.id=COALESCE(n.signal_id,e.signal_id) @@ -1058,7 +1120,10 @@ export async function enrichEvidenceDocument(evidenceId: string) { if (!current.rowCount) throw new Error("Evidence not found"); const row = current.rows[0]; - if (row.evidence_kind === "source_document" && row.snapshot_id) { + if ( + (row.evidence_kind === "source_document" || row.evidence_kind === "source_payload") && + row.snapshot_id + ) { return getEvidence(evidenceId); }