Sync robust evidence quality tiers
This commit is contained in:
parent
a17d5d926e
commit
b2d653482e
97
src/db.ts
97
src/db.ts
@ -236,6 +236,36 @@ export async function migrate() {
|
||||
await pool.query("ALTER TABLE source_snapshots ADD COLUMN IF NOT EXISTS document_text TEXT");
|
||||
await pool.query("ALTER TABLE source_snapshots ADD COLUMN IF NOT EXISTS http_status INTEGER");
|
||||
|
||||
const payloadMarker = await pool.query(
|
||||
`INSERT INTO app_meta(key,value)
|
||||
VALUES('evidence_payload_quality_v1','{"applied":true}'::jsonb)
|
||||
ON CONFLICT(key) DO NOTHING
|
||||
RETURNING key`
|
||||
);
|
||||
if (payloadMarker.rowCount) {
|
||||
await pool.query(`
|
||||
UPDATE source_snapshots ss
|
||||
SET snapshot_kind='source_payload'
|
||||
FROM sources src
|
||||
WHERE ss.source_id=src.id
|
||||
AND src.source_type IN ('rss','geojson')
|
||||
AND src.authority_tier=1
|
||||
AND ss.snapshot_kind='normalized'
|
||||
`);
|
||||
|
||||
await pool.query(`
|
||||
UPDATE evidence e
|
||||
SET evidence_kind='source_payload',
|
||||
locator=COALESCE(locator,'official feed/API item')
|
||||
FROM source_snapshots ss, sources src
|
||||
WHERE e.snapshot_id=ss.id
|
||||
AND ss.source_id=src.id
|
||||
AND ss.snapshot_kind='source_payload'
|
||||
AND src.authority_tier=1
|
||||
AND e.evidence_kind='normalized_excerpt'
|
||||
`);
|
||||
}
|
||||
|
||||
await pool.query(`
|
||||
INSERT INTO source_snapshots(
|
||||
id,source_id,signal_id,source_url,retrieved_at,published_at,content_hash,
|
||||
@ -382,6 +412,17 @@ export async function storeItem(
|
||||
recentEnough &&
|
||||
scored.score >= minScore;
|
||||
|
||||
const sourceMeta = await pool.query(
|
||||
"SELECT source_type,authority_tier FROM sources WHERE id=$1",
|
||||
[item.sourceId]
|
||||
);
|
||||
const directPayload =
|
||||
Number(sourceMeta.rows[0]?.authority_tier) === 1 &&
|
||||
new Set(["rss","geojson"]).has(String(sourceMeta.rows[0]?.source_type ?? ""));
|
||||
const snapshotKind = directPayload ? "source_payload" : "normalized";
|
||||
const evidenceKind = directPayload ? "source_payload" : "normalized_excerpt";
|
||||
const locator = directPayload ? "official feed/API item" : "title+summary";
|
||||
|
||||
await pool.query("BEGIN");
|
||||
try {
|
||||
await pool.query(
|
||||
@ -418,9 +459,9 @@ export async function storeItem(
|
||||
await pool.query(
|
||||
`INSERT INTO source_snapshots(
|
||||
id,source_id,signal_id,source_url,published_at,content_hash,
|
||||
content_type,byte_length,snapshot_json
|
||||
content_type,byte_length,snapshot_json,snapshot_kind
|
||||
)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9::jsonb)`,
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9::jsonb,$10)`,
|
||||
[
|
||||
snapshotId,
|
||||
item.sourceId,
|
||||
@ -428,9 +469,12 @@ export async function storeItem(
|
||||
item.url,
|
||||
item.publishedAt,
|
||||
hash,
|
||||
"application/vnd.media-intelligence.normalized+json",
|
||||
directPayload
|
||||
? "application/vnd.media-intelligence.source-payload+json"
|
||||
: "application/vnd.media-intelligence.normalized+json",
|
||||
Buffer.byteLength(snapshotText),
|
||||
snapshotText
|
||||
snapshotText,
|
||||
snapshotKind
|
||||
]
|
||||
);
|
||||
|
||||
@ -438,14 +482,16 @@ export async function storeItem(
|
||||
`INSERT INTO evidence(
|
||||
id,signal_id,source_url,content_hash,excerpt,snapshot_id,evidence_kind,locator
|
||||
)
|
||||
VALUES($1,$2,$3,$4,$5,$6,'normalized_excerpt','title+summary')`,
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8)`,
|
||||
[
|
||||
evidenceId,
|
||||
signalId,
|
||||
item.url,
|
||||
hash,
|
||||
`${item.title}\n${item.summary}`.slice(0, 2000),
|
||||
snapshotId
|
||||
snapshotId,
|
||||
evidenceKind,
|
||||
locator
|
||||
]
|
||||
);
|
||||
|
||||
@ -690,15 +736,23 @@ function buildStorySourceAudit(story: any) {
|
||||
const nodes = Array.isArray(story.nodes) ? story.nodes : [];
|
||||
const edges = Array.isArray(story.edges) ? story.edges : [];
|
||||
|
||||
const evidenceBacked = nodes.filter((n: any) =>
|
||||
Boolean(
|
||||
n.evidence_id &&
|
||||
n.snapshot_id &&
|
||||
n.provenance_source_id &&
|
||||
const evidenceBacked = nodes.filter((n: any) => {
|
||||
if (!n.evidence_id || !n.snapshot_id || !n.provenance_source_id) return false;
|
||||
|
||||
const documentBacked =
|
||||
n.snapshot_kind === "source_document" &&
|
||||
n.evidence_kind === "source_document"
|
||||
)
|
||||
);
|
||||
n.evidence_kind === "source_document" &&
|
||||
Number(n.snapshot_text_length ?? 0) >= 500;
|
||||
|
||||
const payloadBacked =
|
||||
n.snapshot_kind === "source_payload" &&
|
||||
n.evidence_kind === "source_payload" &&
|
||||
Number(n.provenance_authority_tier) === 1 &&
|
||||
Number(n.snapshot_payload_length ?? 0) >= 350 &&
|
||||
Number(n.evidence_excerpt_length ?? 0) >= 160;
|
||||
|
||||
return documentBacked || payloadBacked;
|
||||
});
|
||||
|
||||
const distinctSourceIds = Array.from(new Set(
|
||||
evidenceBacked.map((n: any) => String(n.provenance_source_id)).filter(Boolean)
|
||||
@ -776,6 +830,11 @@ function buildStorySourceAudit(story: any) {
|
||||
snapshot_id: n.snapshot_id,
|
||||
snapshot_retrieved_at: n.snapshot_retrieved_at,
|
||||
snapshot_published_at: n.snapshot_published_at,
|
||||
evidence_kind: n.evidence_kind,
|
||||
snapshot_kind: n.snapshot_kind,
|
||||
snapshot_text_length: n.snapshot_text_length,
|
||||
snapshot_payload_length: n.snapshot_payload_length,
|
||||
evidence_excerpt_length: n.evidence_excerpt_length,
|
||||
url: n.evidence_url ?? n.signal_url
|
||||
})),
|
||||
unsupported_claims: unsupportedClaims,
|
||||
@ -812,7 +871,10 @@ export async function getStoryThread(id: string) {
|
||||
ss.id AS snapshot_id,
|
||||
ss.snapshot_kind AS snapshot_kind,
|
||||
ss.retrieved_at AS snapshot_retrieved_at,
|
||||
ss.published_at AS snapshot_published_at
|
||||
ss.published_at AS snapshot_published_at,
|
||||
LENGTH(COALESCE(ss.document_text,''))::int AS snapshot_text_length,
|
||||
LENGTH(COALESCE(ss.snapshot_json::text,''))::int AS snapshot_payload_length,
|
||||
LENGTH(COALESCE(e.excerpt,''))::int AS evidence_excerpt_length
|
||||
FROM story_nodes n
|
||||
LEFT JOIN evidence e ON e.id=n.evidence_id
|
||||
LEFT JOIN signals s ON s.id=COALESCE(n.signal_id,e.signal_id)
|
||||
@ -1058,7 +1120,10 @@ export async function enrichEvidenceDocument(evidenceId: string) {
|
||||
if (!current.rowCount) throw new Error("Evidence not found");
|
||||
|
||||
const row = current.rows[0];
|
||||
if (row.evidence_kind === "source_document" && row.snapshot_id) {
|
||||
if (
|
||||
(row.evidence_kind === "source_document" || row.evidence_kind === "source_payload") &&
|
||||
row.snapshot_id
|
||||
) {
|
||||
return getEvidence(evidenceId);
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user