Admin/media-intelligence-deploy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync corporate parser polish and opportunity noise suppression

Browse Source
This commit is contained in:
Admin 2026-09-29 20:15:46 +00:00
parent 3da352e66b
commit 27330207c5
2 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/db.ts
View File

@ -672,15 +672,21 @@ export async function storeItem(
} }
} }
const sourceMeta = await pool.query(
"SELECT source_type,authority_tier,source_class FROM sources WHERE id=$1",
[item.sourceId]
);
const sourceClass = String(sourceMeta.rows[0]?.source_class ?? "");
const suppressNormalizationUpdate =
changeKind === "updated" &&
new Set(["corporate_primary","official_market_notice"]).has(sourceClass);
const shouldCreateOpportunity = const shouldCreateOpportunity =
!options.baseline && !options.baseline &&
!suppressNormalizationUpdate &&
recentEnough && recentEnough &&
scored.score >= minScore; scored.score >= minScore;
const sourceMeta = await pool.query(
"SELECT source_type,authority_tier FROM sources WHERE id=$1",
[item.sourceId]
);
const directPayload = const directPayload =
Number(sourceMeta.rows[0]?.authority_tier) === 1 && Number(sourceMeta.rows[0]?.authority_tier) === 1 &&
new Set(["rss","geojson"]).has(String(sourceMeta.rows[0]?.source_type ?? "")); new Set(["rss","geojson"]).has(String(sourceMeta.rows[0]?.source_type ?? ""));

18
src/sources.ts
View File

@ -298,7 +298,7 @@ function parseNewsDate(text: string): string | null {
const direct = parseParliamentDate(text); const direct = parseParliamentDate(text);
if (direct) return direct; if (direct) return direct;
const match = text.match(/\b(January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\s+(\d{1,2}),?\s+(20\d{2})\b/i); const match = text.match(/\b(January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\s+(\d{1,2}),?\s+(20\d{2})/i);
if (!match) return null; if (!match) return null;
const months: Record<string, number> = { const months: Record<string, number> = {
january:0, jan:0, february:1, feb:1, march:2, mar:2, april:3, apr:3, january:0, jan:0, february:1, feb:1, march:2, mar:2, april:3, apr:3,
@ -444,15 +444,21 @@ async function fetchCorporateNewsroom(
const anchorText = $(element).text().replace(/\s+/g, " ").trim(); const anchorText = $(element).text().replace(/\s+/g, " ").trim();
const container = $(element).closest("article,li,section,div"); const container = $(element).closest("article,li,section,div");
const context = container.text().replace(/\s+/g, " ").trim(); const context = container.text().replace(/\s+/g, " ").trim();
const title = anchorText.length >= 8 const heading = container.find("h1,h2,h3,h4").first().text().replace(/\s+/g, " ").trim();
? anchorText let title = heading.length >= 8 ? heading : anchorText;
: (container.find("h1,h2,h3,h4").first().text().replace(/\s+/g, " ").trim() || anchorText);
if (sourceId === "anthropic-news") {
title = title
.replace(/^(January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\s+\d{1,2},?\s+20\d{2}\s*/i, "")
.replace(/^(Announcements|Science|Features|Product|Policy|Societal Impacts|Interpretability|Education|Event)\s*/i, "")
.trim();
}
if (!title || title.length < 8) return; if (!title || title.length < 8) return;
seen.add(url); seen.add(url);
const publishedAt = parseNewsDate(context); const publishedAt = parseNewsDate(context) ?? parseNewsDate(anchorText);
const summary = context const summary = context
.replace(title, "") .replace(anchorText, "")
.replace(/\s+/g, " ") .replace(/\s+/g, " ")
.trim() .trim()
.slice(0, 3000); .slice(0, 3000);