Admin/media-intelligence-deploy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync MBIE WAF detection

Browse Source
This commit is contained in:
Admin 2026-09-29 07:24:37 +00:00
parent 07f6dc54e2
commit 78235ec5d1
1 changed files with 21 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/sources.ts
View File

@ -663,22 +663,40 @@ async function fetchRbnz(): Promise<NormalizedItem[]> {
async function fetchMbieJobsOnline(): Promise<NormalizedItem[]> { async function fetchMbieJobsOnline(): Promise<NormalizedItem[]> {
const source = SOURCES.find((s) => s.id === "mbie-jobs-online")!; const source = SOURCES.find((s) => s.id === "mbie-jobs-online")!;
const html = await getText(source.url, "text/html,application/xhtml+xml"); const response = await fetch(source.url, {
headers: {
accept: "text/html,application/xhtml+xml",
"accept-language": "en-NZ,en;q=0.9",
"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/130 Safari/537.36"
},
redirect: "follow",
signal: AbortSignal.timeout(20000)
});
if (!response.ok) {
throw new Error(`MBIE Jobs Online HTTP ${response.status} ${response.statusText}`);
}
const html = await response.text();
if (/incapsula incident id/i.test(html) || /request unsuccessful/i.test(html)) {
throw new Error(`MBIE Jobs Online blocked by Incapsula (bytes=${html.length})`);
}
const text = pageText(html); const text = pageText(html);
const updated = const updated =
text.match(/Last updated:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ?? text.match(/Last updated:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
text.match(/updated\s+(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ?? text.match(/updated\s+(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
""; "";
const summary = excerptAround(text, "Jobs Online monthly data release", 2600);
return [{ return [{
sourceId: "mbie-jobs-online", sourceId: "mbie-jobs-online",
externalId: "jobs-online", externalId: "jobs-online",
title: "MBIE Jobs Online labour demand", title: "MBIE Jobs Online labour demand",
summary: excerptAround(text, "Jobs Online monthly data release", 2600), summary,
url: source.url, url: source.url,
publishedAt: parseParliamentDate(updated), publishedAt: parseParliamentDate(updated),
topics: ["employment", "jobs", "vacancies", "labour-market", "mbie", "new-zealand"], topics: ["employment", "jobs", "vacancies", "labour-market", "mbie", "new-zealand"],
raw: { updated, text: text.slice(0, 9000) } raw: { updated, summary }
}]; }];
} }