diff --git a/src/db.ts b/src/db.ts index 428578f..6e18a60 100644 --- a/src/db.ts +++ b/src/db.ts @@ -672,15 +672,21 @@ export async function storeItem( } } + const sourceMeta = await pool.query( + "SELECT source_type,authority_tier,source_class FROM sources WHERE id=$1", + [item.sourceId] + ); + const sourceClass = String(sourceMeta.rows[0]?.source_class ?? ""); + const suppressNormalizationUpdate = + changeKind === "updated" && + new Set(["corporate_primary","official_market_notice"]).has(sourceClass); + const shouldCreateOpportunity = !options.baseline && + !suppressNormalizationUpdate && recentEnough && scored.score >= minScore; - const sourceMeta = await pool.query( - "SELECT source_type,authority_tier FROM sources WHERE id=$1", - [item.sourceId] - ); const directPayload = Number(sourceMeta.rows[0]?.authority_tier) === 1 && new Set(["rss","geojson"]).has(String(sourceMeta.rows[0]?.source_type ?? "")); diff --git a/src/sources.ts b/src/sources.ts index 5d9c3c0..cba7fdd 100644 --- a/src/sources.ts +++ b/src/sources.ts @@ -298,7 +298,7 @@ function parseNewsDate(text: string): string | null { const direct = parseParliamentDate(text); if (direct) return direct; - const match = text.match(/\b(January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\s+(\d{1,2}),?\s+(20\d{2})\b/i); + const match = text.match(/\b(January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\s+(\d{1,2}),?\s+(20\d{2})/i); if (!match) return null; const months: Record = { january:0, jan:0, february:1, feb:1, march:2, mar:2, april:3, apr:3, @@ -444,15 +444,21 @@ async function fetchCorporateNewsroom( const anchorText = $(element).text().replace(/\s+/g, " ").trim(); const container = $(element).closest("article,li,section,div"); const context = container.text().replace(/\s+/g, " ").trim(); - const title = anchorText.length >= 8 - ? anchorText - : (container.find("h1,h2,h3,h4").first().text().replace(/\s+/g, " ").trim() || anchorText); + const heading = container.find("h1,h2,h3,h4").first().text().replace(/\s+/g, " ").trim(); + let title = heading.length >= 8 ? heading : anchorText; + + if (sourceId === "anthropic-news") { + title = title + .replace(/^(January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\s+\d{1,2},?\s+20\d{2}\s*/i, "") + .replace(/^(Announcements|Science|Features|Product|Policy|Societal Impacts|Interpretability|Education|Event)\s*/i, "") + .trim(); + } if (!title || title.length < 8) return; seen.add(url); - const publishedAt = parseNewsDate(context); + const publishedAt = parseNewsDate(context) ?? parseNewsDate(anchorText); const summary = context - .replace(title, "") + .replace(anchorText, "") .replace(/\s+/g, " ") .trim() .slice(0, 3000);