Sync robust evidence quality tiers

src/db.ts

@@ -236,6 +236,36 @@ export async function migrate() {
   await pool.query("ALTER TABLE source_snapshots ADD COLUMN IF NOT EXISTS document_text TEXT");
   await pool.query("ALTER TABLE source_snapshots ADD COLUMN IF NOT EXISTS http_status INTEGER");
 
+  const payloadMarker = await pool.query(
+    `INSERT INTO app_meta(key,value)
+     VALUES('evidence_payload_quality_v1','{"applied":true}'::jsonb)
+     ON CONFLICT(key) DO NOTHING
+     RETURNING key`
+  );
+  if (payloadMarker.rowCount) {
+    await pool.query(`
+      UPDATE source_snapshots ss
+      SET snapshot_kind='source_payload'
+      FROM sources src
+      WHERE ss.source_id=src.id
+        AND src.source_type IN ('rss','geojson')
+        AND src.authority_tier=1
+        AND ss.snapshot_kind='normalized'
+    `);
+
+    await pool.query(`
+      UPDATE evidence e
+      SET evidence_kind='source_payload',
+          locator=COALESCE(locator,'official feed/API item')
+      FROM source_snapshots ss, sources src
+      WHERE e.snapshot_id=ss.id
+        AND ss.source_id=src.id
+        AND ss.snapshot_kind='source_payload'
+        AND src.authority_tier=1
+        AND e.evidence_kind='normalized_excerpt'
+    `);
+  }
+
   await pool.query(`
     INSERT INTO source_snapshots(
       id,source_id,signal_id,source_url,retrieved_at,published_at,content_hash,
@@ -382,6 +412,17 @@ export async function storeItem(
     recentEnough &&
     scored.score >= minScore;
 
+  const sourceMeta = await pool.query(
+    "SELECT source_type,authority_tier FROM sources WHERE id=$1",
+    [item.sourceId]
+  );
+  const directPayload =
+    Number(sourceMeta.rows[0]?.authority_tier) === 1 &&
+    new Set(["rss","geojson"]).has(String(sourceMeta.rows[0]?.source_type ?? ""));
+  const snapshotKind = directPayload ? "source_payload" : "normalized";
+  const evidenceKind = directPayload ? "source_payload" : "normalized_excerpt";
+  const locator = directPayload ? "official feed/API item" : "title+summary";
+
   await pool.query("BEGIN");
   try {
     await pool.query(
@@ -418,9 +459,9 @@ export async function storeItem(
     await pool.query(
       `INSERT INTO source_snapshots(
          id,source_id,signal_id,source_url,published_at,content_hash,
-         content_type,byte_length,snapshot_json
+         content_type,byte_length,snapshot_json,snapshot_kind
        )
-       VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9::jsonb)`,
+       VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9::jsonb,$10)`,
       [
         snapshotId,
         item.sourceId,
@@ -428,9 +469,12 @@ export async function storeItem(
         item.url,
         item.publishedAt,
         hash,
-        "application/vnd.media-intelligence.normalized+json",
+        directPayload
+          ? "application/vnd.media-intelligence.source-payload+json"
+          : "application/vnd.media-intelligence.normalized+json",
         Buffer.byteLength(snapshotText),
-        snapshotText
+        snapshotText,
+        snapshotKind
       ]
     );
 
@@ -438,14 +482,16 @@ export async function storeItem(
       `INSERT INTO evidence(
          id,signal_id,source_url,content_hash,excerpt,snapshot_id,evidence_kind,locator
        )
-       VALUES($1,$2,$3,$4,$5,$6,'normalized_excerpt','title+summary')`,
+       VALUES($1,$2,$3,$4,$5,$6,$7,$8)`,
       [
         evidenceId,
         signalId,
         item.url,
         hash,
         `${item.title}\n${item.summary}`.slice(0, 2000),
-        snapshotId
+        snapshotId,
+        evidenceKind,
+        locator
       ]
     );
 
@@ -690,15 +736,23 @@ function buildStorySourceAudit(story: any) {
   const nodes = Array.isArray(story.nodes) ? story.nodes : [];
   const edges = Array.isArray(story.edges) ? story.edges : [];
 
-  const evidenceBacked = nodes.filter((n: any) =>
+  const evidenceBacked = nodes.filter((n: any) => {
-    Boolean(
+    if (!n.evidence_id || !n.snapshot_id || !n.provenance_source_id) return false;
-      n.evidence_id &&
+
-      n.snapshot_id &&
+    const documentBacked =
-      n.provenance_source_id &&
       n.snapshot_kind === "source_document" &&
-      n.evidence_kind === "source_document"
+      n.evidence_kind === "source_document" &&
-    )
+      Number(n.snapshot_text_length ?? 0) >= 500;
-  );
+
+    const payloadBacked =
+      n.snapshot_kind === "source_payload" &&
+      n.evidence_kind === "source_payload" &&
+      Number(n.provenance_authority_tier) === 1 &&
+      Number(n.snapshot_payload_length ?? 0) >= 350 &&
+      Number(n.evidence_excerpt_length ?? 0) >= 160;
+
+    return documentBacked || payloadBacked;
+  });
 
   const distinctSourceIds = Array.from(new Set(
     evidenceBacked.map((n: any) => String(n.provenance_source_id)).filter(Boolean)
@@ -776,6 +830,11 @@ function buildStorySourceAudit(story: any) {
       snapshot_id: n.snapshot_id,
       snapshot_retrieved_at: n.snapshot_retrieved_at,
       snapshot_published_at: n.snapshot_published_at,
+      evidence_kind: n.evidence_kind,
+      snapshot_kind: n.snapshot_kind,
+      snapshot_text_length: n.snapshot_text_length,
+      snapshot_payload_length: n.snapshot_payload_length,
+      evidence_excerpt_length: n.evidence_excerpt_length,
       url: n.evidence_url ?? n.signal_url
     })),
     unsupported_claims: unsupportedClaims,
@@ -812,7 +871,10 @@ export async function getStoryThread(id: string) {
               ss.id AS snapshot_id,
               ss.snapshot_kind AS snapshot_kind,
               ss.retrieved_at AS snapshot_retrieved_at,
-              ss.published_at AS snapshot_published_at
+              ss.published_at AS snapshot_published_at,
+              LENGTH(COALESCE(ss.document_text,''))::int AS snapshot_text_length,
+              LENGTH(COALESCE(ss.snapshot_json::text,''))::int AS snapshot_payload_length,
+              LENGTH(COALESCE(e.excerpt,''))::int AS evidence_excerpt_length
        FROM story_nodes n
        LEFT JOIN evidence e ON e.id=n.evidence_id
        LEFT JOIN signals s ON s.id=COALESCE(n.signal_id,e.signal_id)
@@ -1058,7 +1120,10 @@ export async function enrichEvidenceDocument(evidenceId: string) {
   if (!current.rowCount) throw new Error("Evidence not found");
 
   const row = current.rows[0];
-  if (row.evidence_kind === "source_document" && row.snapshot_id) {
+  if (
+    (row.evidence_kind === "source_document" || row.evidence_kind === "source_payload") &&
+    row.snapshot_id
+  ) {
     return getEvidence(evidenceId);
   }