Admin/media-intelligence-deploy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync corporate parser polish and opportunity noise suppression

Browse Source
This commit is contained in:
Admin 2026-09-29 20:15:46 +00:00
parent 3da352e66b
commit 27330207c5
2 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/db.ts
View File

@ -672,15 +672,21 @@ export async function storeItem(
}
}
const sourceMeta = await pool.query(
"SELECT source_type,authority_tier,source_class FROM sources WHERE id=$1",
[item.sourceId]
);
const sourceClass = String(sourceMeta.rows[0]?.source_class ?? "");
const suppressNormalizationUpdate =
changeKind === "updated" &&
new Set(["corporate_primary","official_market_notice"]).has(sourceClass);
const shouldCreateOpportunity =
!options.baseline &&
!suppressNormalizationUpdate &&
recentEnough &&
scored.score >= minScore;
const sourceMeta = await pool.query(
"SELECT source_type,authority_tier FROM sources WHERE id=$1",
[item.sourceId]
);
const directPayload =
Number(sourceMeta.rows[0]?.authority_tier) === 1 &&
new Set(["rss","geojson"]).has(String(sourceMeta.rows[0]?.source_type ?? ""));

18
src/sources.ts
View File

@ -298,7 +298,7 @@ function parseNewsDate(text: string): string | null {
const direct = parseParliamentDate(text);
if (direct) return direct;
const match = text.match(/\b(January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\s+(\d{1,2}),?\s+(20\d{2})\b/i);
const match = text.match(/\b(January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\s+(\d{1,2}),?\s+(20\d{2})/i);
if (!match) return null;
const months: Record<string, number> = {
january:0, jan:0, february:1, feb:1, march:2, mar:2, april:3, apr:3,
@ -444,15 +444,21 @@ async function fetchCorporateNewsroom(
const anchorText = $(element).text().replace(/\s+/g, " ").trim();
const container = $(element).closest("article,li,section,div");
const context = container.text().replace(/\s+/g, " ").trim();
const title = anchorText.length >= 8
? anchorText
: (container.find("h1,h2,h3,h4").first().text().replace(/\s+/g, " ").trim() || anchorText);
const heading = container.find("h1,h2,h3,h4").first().text().replace(/\s+/g, " ").trim();
let title = heading.length >= 8 ? heading : anchorText;
if (sourceId === "anthropic-news") {
title = title
.replace(/^(January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\s+\d{1,2},?\s+20\d{2}\s*/i, "")
.replace(/^(Announcements|Science|Features|Product|Policy|Societal Impacts|Interpretability|Education|Event)\s*/i, "")
.trim();
}
if (!title || title.length < 8) return;
seen.add(url);
const publishedAt = parseNewsDate(context);
const publishedAt = parseNewsDate(context) ?? parseNewsDate(anchorText);
const summary = context
.replace(title, "")
.replace(anchorText, "")
.replace(/\s+/g, " ")
.trim()
.slice(0, 3000);