From 78235ec5d1530c573fc899ec8519c375cc39d6f6 Mon Sep 17 00:00:00 2001
From: Admin <dmparrow@gmail.com>
Date: Tue, 29 Sep 2026 07:24:37 +0000
Subject: [PATCH] Sync MBIE WAF detection

---
 src/sources.ts | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/sources.ts b/src/sources.ts
index 5f1f65f..d505691 100644
--- a/src/sources.ts
+++ b/src/sources.ts
@@ -663,22 +663,40 @@ async function fetchRbnz(): Promise<NormalizedItem[]> {
 
 async function fetchMbieJobsOnline(): Promise<NormalizedItem[]> {
   const source = SOURCES.find((s) => s.id === "mbie-jobs-online")!;
-  const html = await getText(source.url, "text/html,application/xhtml+xml");
+  const response = await fetch(source.url, {
+    headers: {
+      accept: "text/html,application/xhtml+xml",
+      "accept-language": "en-NZ,en;q=0.9",
+      "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/130 Safari/537.36"
+    },
+    redirect: "follow",
+    signal: AbortSignal.timeout(20000)
+  });
+  if (!response.ok) {
+    throw new Error(`MBIE Jobs Online HTTP ${response.status} ${response.statusText}`);
+  }
+
+  const html = await response.text();
+  if (/incapsula incident id/i.test(html) || /request unsuccessful/i.test(html)) {
+    throw new Error(`MBIE Jobs Online blocked by Incapsula (bytes=${html.length})`);
+  }
+
   const text = pageText(html);
   const updated =
     text.match(/Last updated:\s*(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
     text.match(/updated\s+(\d{1,2}\s+[A-Za-z]+\s+20\d{2})/i)?.[1] ??
     "";
+  const summary = excerptAround(text, "Jobs Online monthly data release", 2600);
 
   return [{
     sourceId: "mbie-jobs-online",
     externalId: "jobs-online",
     title: "MBIE Jobs Online labour demand",
-    summary: excerptAround(text, "Jobs Online monthly data release", 2600),
+    summary,
     url: source.url,
     publishedAt: parseParliamentDate(updated),
     topics: ["employment", "jobs", "vacancies", "labour-market", "mbie", "new-zealand"],
-    raw: { updated, text: text.slice(0, 9000) }
+    raw: { updated, summary }
   }];
 }