import * as pdfjsLib from 'pdfjs-dist' import pdfWorker from 'pdfjs-dist/build/pdf.worker.min.mjs?url' import type { Worker } from 'tesseract.js' pdfjsLib.GlobalWorkerOptions.workerSrc = pdfWorker type RegionType = 'ocr' | 'barcode' | 'ignore' type RegionTarget = | 'palletCard' | 'origin' | 'variety' | 'brand' | 'class' | 'base' | 'pack' | 'grower' | 'packrun' | 'size' | 'qty' | 'pickDate' | 'packDate' | 'total' | 'custom' type Box = { x: number; y: number; w: number; h: number } type RegionSettings = { psm: '3' | '6' | '7' | '11' grayscale: boolean contrast: number threshold: number whitelist: string barcodeFormats: string[] firstMatch: boolean } type RefinementRegion = { id: string name: string type: RegionType target: RegionTarget enabled: boolean box: Box settings: RegionSettings lastText?: string confidence?: number } type RefinementProfile = { schemaVersion: 1 id: string name: string revision: number savedAt?: string page: { rotation: 0 | 90 | 180 | 270 renderDpi: number zoom: number } regions: RefinementRegion[] } type RecentCapture = { name: string mime: string dataBase64: string } async function loadPdf(data: ArrayBuffer) { return pdfjsLib.getDocument({ data }).promise } type LoadedPdf = Awaited> type DocumentSource = | { kind: 'pdf'; name: string; document: LoadedPdf } | { kind: 'image'; name: string; bitmap: ImageBitmap } const TARGET_OPTIONS: Array<{ value: RegionTarget; label: string }> = [ { value: 'palletCard', label: 'Pallet card block' }, { value: 'origin', label: 'Origin' }, { value: 'variety', label: 'Variety' }, { value: 'brand', label: 'Brand' }, { value: 'class', label: 'Class' }, { value: 'base', label: 'Base' }, { value: 'pack', label: 'Pack' }, { value: 'grower', label: 'Grower' }, { value: 'packrun', label: 'Packrun' }, { value: 'size', label: 'Size' }, { value: 'qty', label: 'Qty' }, { value: 'pickDate', label: 'Pick date' }, { value: 'packDate', label: 'Pack date' }, { value: 'total', label: 'Total' }, { value: 'custom', label: 'Custom text' }, ] const DEFAULT_SETTINGS: RegionSettings = { psm: '6', grayscale: true, contrast: 1.5, threshold: 0, whitelist: '', barcodeFormats: ['CODE_128'], firstMatch: true, } function defaultProfile(): RefinementProfile { return { schemaVersion: 1, id: 'avocado-pallet-card', name: 'Avocado pallet card', revision: 0, page: { rotation: 0, renderDpi: 200, zoom: 0.82 }, regions: [], } } function cloneSettings(): RegionSettings { return { ...DEFAULT_SETTINGS, barcodeFormats: [...DEFAULT_SETTINGS.barcodeFormats], } } function clamp(value: number, min = 0, max = 1) { return Math.max(min, Math.min(max, value)) } function decodeBase64(value: string) { const binary = atob(value) const bytes = new Uint8Array(binary.length) for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index) return bytes } function escapeHtml(value: string) { return value.replace(/[&<>"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[char]!)) } let ocrWorkerPromise: Promise | null = null async function getOcrWorker(updateStatus: (message: string) => void) { if (!ocrWorkerPromise) { ocrWorkerPromise = (async () => { const { createWorker, OEM } = await import('tesseract.js') return createWorker('eng', OEM.LSTM_ONLY, { logger: (message) => { if (typeof message.progress === 'number') { updateStatus((message.status || 'OCR') + ' · ' + Math.round(message.progress * 100) + '%') } else if (message.status) { updateStatus(message.status) } }, }) })().catch((error) => { ocrWorkerPromise = null throw error }) } return ocrWorkerPromise } export function initOcrRefinement(root: HTMLElement, invoke: (command: string, args?: Record) => Promise) { root.innerHTML = [ '
', '
', '
', '
', 'OCR REFINEMENT', '
', '', 'v1 · unsaved', '
', '
', '
', '', '', '', '', '', '', '
', '
', '
', 'No document loaded', '', '', 'Load a PDF or image, then draw target regions.', '
', '
', '
', '', '
', '
', '
DOCUMENT

Preview & box drawer

', 'Select “+ OCR area” to draw', '
', '
', '
No document loadedLoad the production PDF, a screenshot, or the most recent captured document.
', '', '
', '
', '', '
', '
', ].join('') const byId = (id: string) => root.querySelector('#' + id)! const canvas = byId('ocr-document-canvas') const overlay = byId('ocr-overlay') const pageFrame = byId('ocr-page-frame') const previewEmpty = byId('ocr-preview-empty') const cropCanvas = byId('ocr-region-preview') const profileNameInput = byId('ocr-profile-name') const regionTargetSelect = byId('ocr-region-target') regionTargetSelect.innerHTML = TARGET_OPTIONS.map((option) => '').join('') let profile = defaultProfile() let source: DocumentSource | null = null let selectedId: string | null = null let pendingDrawType: RegionType | null = null let dragState: { kind: 'draw' | 'move' | 'resize' regionId: string startX: number startY: number initial: Box } | null = null let renderGeneration = 0 function selectedRegion() { return profile.regions.find((region) => region.id === selectedId) ?? null } function setStatus(message: string) { byId('ocr-status').textContent = message } function normalizeProfile(value: unknown): RefinementProfile { if (!value || typeof value !== 'object') return defaultProfile() const candidate = value as Partial if (candidate.schemaVersion !== 1 || !Array.isArray(candidate.regions)) return defaultProfile() return { ...defaultProfile(), ...candidate, page: { ...defaultProfile().page, ...candidate.page }, regions: candidate.regions.map((region) => ({ ...region, settings: { ...cloneSettings(), ...region.settings, barcodeFormats: [...(region.settings?.barcodeFormats ?? DEFAULT_SETTINGS.barcodeFormats)], }, })), } as RefinementProfile } function updateProfileHeader() { profileNameInput.value = profile.name const version = byId('ocr-profile-version') version.textContent = profile.revision > 0 ? 'v1 · revision ' + profile.revision : 'v1 · unsaved' byId('ocr-zoom').value = String(profile.page.zoom) byId('ocr-dpi').value = String(profile.page.renderDpi) } async function renderDocument() { if (!source) return const generation = ++renderGeneration const context = canvas.getContext('2d') if (!context) return if (source.kind === 'pdf') { const page = await source.document.getPage(1) if (generation !== renderGeneration) return const pointsPerInch = 72 const scale = profile.page.renderDpi / pointsPerInch const viewport = page.getViewport({ scale, rotation: profile.page.rotation }) const maxDimension = 1600 const renderScale = Math.min(1, maxDimension / Math.max(viewport.width, viewport.height)) const finalViewport = page.getViewport({ scale: scale * renderScale, rotation: profile.page.rotation, }) canvas.width = Math.max(1, Math.round(finalViewport.width)) canvas.height = Math.max(1, Math.round(finalViewport.height)) await page.render({ canvasContext: context, viewport: finalViewport }).promise } else { const bitmap = source.bitmap const rotated = profile.page.rotation === 90 || profile.page.rotation === 270 const maxDimension = 1600 const scale = Math.min(1, maxDimension / Math.max(bitmap.width, bitmap.height)) const sourceWidth = bitmap.width * scale const sourceHeight = bitmap.height * scale canvas.width = Math.max(1, Math.round(rotated ? sourceHeight : sourceWidth)) canvas.height = Math.max(1, Math.round(rotated ? sourceWidth : sourceHeight)) context.clearRect(0, 0, canvas.width, canvas.height) context.save() if (profile.page.rotation === 90) { context.translate(canvas.width, 0) context.rotate(Math.PI / 2) } else if (profile.page.rotation === 180) { context.translate(canvas.width, canvas.height) context.rotate(Math.PI) } else if (profile.page.rotation === 270) { context.translate(0, canvas.height) context.rotate(-Math.PI / 2) } context.drawImage(bitmap, 0, 0, sourceWidth, sourceHeight) context.restore() } pageFrame.hidden = false previewEmpty.hidden = true applyZoom() renderRegions() updateCropPreview() } function applyZoom() { if (!canvas.width || !canvas.height) return const width = canvas.width * profile.page.zoom const height = canvas.height * profile.page.zoom canvas.style.width = width + 'px' canvas.style.height = height + 'px' overlay.style.width = width + 'px' overlay.style.height = height + 'px' pageFrame.style.width = width + 'px' pageFrame.style.height = height + 'px' } async function loadBlob(blob: Blob, name: string) { setStatus('Loading ' + name + '…') selectedId = null if (source?.kind === 'image') source.bitmap.close() if (blob.type === 'application/pdf' || name.toLowerCase().endsWith('.pdf')) { source = { kind: 'pdf', name, document: await loadPdf(await blob.arrayBuffer()) } } else if (blob.type.startsWith('image/')) { source = { kind: 'image', name, bitmap: await createImageBitmap(blob) } } else { throw new Error('Use a PDF, PNG, JPEG or WebP document.') } byId('ocr-document-name').textContent = name setStatus('Document loaded. Draw or adjust target regions.') await renderDocument() } function overlayPoint(event: PointerEvent) { const rect = overlay.getBoundingClientRect() return { x: clamp((event.clientX - rect.left) / rect.width), y: clamp((event.clientY - rect.top) / rect.height), } } function newRegion(type: RegionType, box: Box): RefinementRegion { const number = profile.regions.filter((region) => region.type === type).length + 1 return { id: crypto.randomUUID(), name: type === 'ocr' ? 'OCR area ' + number : type === 'barcode' ? 'Barcode area ' + number : 'Ignore area ' + number, type, target: type === 'ocr' ? 'palletCard' : 'custom', enabled: true, box, settings: cloneSettings(), } } function beginDraw(type: RegionType) { if (!source) { setStatus('Load a document before drawing regions.') return } pendingDrawType = type byId('ocr-draw-mode').textContent = 'Drawing ' + type + ' region · drag on document' overlay.classList.add('drawing') } function renderRegions() { overlay.innerHTML = '' for (const region of profile.regions) { if (!region.enabled) continue const node = document.createElement('div') node.className = 'ocr-region-box type-' + region.type + (region.id === selectedId ? ' selected' : '') node.dataset.regionId = region.id node.style.left = (region.box.x * 100) + '%' node.style.top = (region.box.y * 100) + '%' node.style.width = (region.box.w * 100) + '%' node.style.height = (region.box.h * 100) + '%' const label = document.createElement('span') label.textContent = region.name node.append(label) const handle = document.createElement('i') handle.className = 'ocr-resize-handle' node.append(handle) node.addEventListener('pointerdown', (event) => { if (pendingDrawType) return event.preventDefault() event.stopPropagation() selectedId = region.id const point = overlayPoint(event) dragState = { kind: event.target === handle ? 'resize' : 'move', regionId: region.id, startX: point.x, startY: point.y, initial: { ...region.box }, } overlay.setPointerCapture(event.pointerId) renderRegions() renderRegionList() renderSelectedSettings() }) overlay.append(node) } } function renderRegionList() { const list = byId('ocr-region-list') byId('ocr-region-count').textContent = String(profile.regions.length) if (!profile.regions.length) { list.innerHTML = '
No regions yet.
' return } list.innerHTML = profile.regions.map((region) => { const result = region.lastText ? '' + Math.round(region.confidence ?? 0) + '%' : '' return [ '', ].join('') }).join('') list.querySelectorAll('[data-region-select]').forEach((button) => { button.addEventListener('click', () => { selectedId = button.dataset.regionSelect ?? null renderRegions() renderRegionList() renderSelectedSettings() }) }) } function renderSelectedSettings() { const region = selectedRegion() byId('ocr-delete-region').disabled = !region byId('ocr-no-selection').hidden = Boolean(region) byId('ocr-selection-settings').hidden = !region if (!region) { cropCanvas.width = 1 cropCanvas.height = 1 return } byId('ocr-region-name').value = region.name byId('ocr-region-type').value = region.type regionTargetSelect.value = region.target byId('ocr-psm').value = region.settings.psm byId('ocr-grayscale').checked = region.settings.grayscale byId('ocr-contrast').value = String(region.settings.contrast) byId('ocr-threshold').value = String(region.settings.threshold) byId('ocr-whitelist').value = region.settings.whitelist byId('ocr-barcode-formats').value = region.settings.barcodeFormats.join(', ') byId('ocr-first-match').checked = region.settings.firstMatch byId('ocr-contrast-value').textContent = region.settings.contrast.toFixed(1) + '×' byId('ocr-threshold-value').textContent = region.settings.threshold > 0 ? String(region.settings.threshold) : 'Off' byId('ocr-ocr-settings').hidden = region.type !== 'ocr' byId('ocr-barcode-settings').hidden = region.type !== 'barcode' regionTargetSelect.closest('label')!.toggleAttribute('hidden', region.type !== 'ocr') byId('ocr-coordinates').textContent = 'x ' + (region.box.x * 100).toFixed(1) + '% · y ' + (region.box.y * 100).toFixed(1) + '% · w ' + (region.box.w * 100).toFixed(1) + '% · h ' + (region.box.h * 100).toFixed(1) + '%' byId('ocr-confidence').textContent = region.confidence === undefined ? '' : Math.round(region.confidence) + '% confidence' byId('ocr-raw-output').textContent = region.lastText || '—' byId('ocr-region-status').textContent = region.lastText ? 'Last result' : 'Not run' updateCropPreview() } function updateCropPreview() { const region = selectedRegion() if (!region || !canvas.width || !canvas.height) return const sx = Math.max(0, Math.floor(region.box.x * canvas.width)) const sy = Math.max(0, Math.floor(region.box.y * canvas.height)) const sw = Math.max(1, Math.floor(region.box.w * canvas.width)) const sh = Math.max(1, Math.floor(region.box.h * canvas.height)) const maxWidth = 520 const scale = Math.min(1, maxWidth / sw) cropCanvas.width = Math.max(1, Math.round(sw * scale)) cropCanvas.height = Math.max(1, Math.round(sh * scale)) const context = cropCanvas.getContext('2d', { willReadFrequently: true }) if (!context) return context.clearRect(0, 0, cropCanvas.width, cropCanvas.height) context.filter = (region.settings.grayscale ? 'grayscale(1) ' : '') + 'contrast(' + region.settings.contrast + ')' context.drawImage(canvas, sx, sy, sw, sh, 0, 0, cropCanvas.width, cropCanvas.height) context.filter = 'none' if (region.settings.threshold > 0) { const image = context.getImageData(0, 0, cropCanvas.width, cropCanvas.height) for (let index = 0; index < image.data.length; index += 4) { const luminance = image.data[index] * 0.299 + image.data[index + 1] * 0.587 + image.data[index + 2] * 0.114 const value = luminance >= region.settings.threshold ? 255 : 0 image.data[index] = value image.data[index + 1] = value image.data[index + 2] = value } context.putImageData(image, 0, 0) } } function updateRegionFromControls() { const region = selectedRegion() if (!region) return region.name = byId('ocr-region-name').value.trim() || region.name region.type = byId('ocr-region-type').value as RegionType region.target = regionTargetSelect.value as RegionTarget region.settings.psm = byId('ocr-psm').value as RegionSettings['psm'] region.settings.grayscale = byId('ocr-grayscale').checked region.settings.contrast = Number(byId('ocr-contrast').value) region.settings.threshold = Number(byId('ocr-threshold').value) region.settings.whitelist = byId('ocr-whitelist').value region.settings.barcodeFormats = byId('ocr-barcode-formats').value .split(',') .map((value) => value.trim().toUpperCase()) .filter(Boolean) region.settings.firstMatch = byId('ocr-first-match').checked renderRegions() renderRegionList() renderSelectedSettings() } async function runRegion(regionId: string) { const region = profile.regions.find((candidate) => candidate.id === regionId) if (!region || !region.enabled) return selectedId = region.id renderRegions() renderRegionList() renderSelectedSettings() updateCropPreview() if (region.type !== 'ocr') { byId('ocr-region-status').textContent = region.type === 'barcode' ? 'Preview ready · production barcode decoder handles this region' : 'Ignored region' return } const status = byId('ocr-region-status') status.textContent = 'Preparing OCR…' try { const worker = await getOcrWorker((message) => { status.textContent = message }) const { PSM } = await import('tesseract.js') const psm = { '3': PSM.AUTO, '6': PSM.SINGLE_BLOCK, '7': PSM.SINGLE_LINE, '11': PSM.SPARSE_TEXT, }[region.settings.psm] await worker.setParameters({ tessedit_pageseg_mode: psm, tessedit_char_whitelist: region.settings.whitelist, preserve_interword_spaces: '1', }) const result = await worker.recognize(cropCanvas) region.lastText = result.data.text.trim() region.confidence = result.data.confidence status.textContent = 'OCR complete' renderRegionList() renderSelectedSettings() } catch (error) { status.textContent = 'OCR failed' setStatus('OCR failed: ' + String(error)) } } async function loadSavedProfile() { try { const saved = await invoke('load_ocr_profile') if (saved) profile = normalizeProfile(saved) } catch { profile = defaultProfile() } updateProfileHeader() renderRegionList() renderSelectedSettings() } byId('ocr-load-document').addEventListener('click', () => byId('ocr-file-input').click()) byId('ocr-file-input').addEventListener('change', (event) => { const file = (event.currentTarget as HTMLInputElement).files?.[0] if (!file) return void loadBlob(file, file.name).catch((error) => setStatus('Document load failed: ' + String(error))) }) byId('ocr-load-recent').addEventListener('click', () => { setStatus('Loading most recent captured document…') void invoke('get_recent_capture') .then((capture) => loadBlob(new Blob([decodeBase64(capture.dataBase64)], { type: capture.mime }), capture.name)) .catch((error) => setStatus('Recent capture unavailable: ' + String(error))) }) byId('ocr-rotate').addEventListener('click', () => { profile.page.rotation = ((profile.page.rotation + 90) % 360) as RefinementProfile['page']['rotation'] void renderDocument() }) byId('ocr-zoom').addEventListener('input', (event) => { profile.page.zoom = Number((event.currentTarget as HTMLInputElement).value) applyZoom() }) byId('ocr-dpi').addEventListener('change', (event) => { profile.page.renderDpi = Number((event.currentTarget as HTMLSelectElement).value) void renderDocument() }) root.querySelectorAll('[data-add-region]').forEach((button) => { button.addEventListener('click', () => beginDraw(button.dataset.addRegion as RegionType)) }) overlay.addEventListener('pointerdown', (event) => { if (!pendingDrawType) { if (event.target === overlay) { selectedId = null renderRegions() renderRegionList() renderSelectedSettings() } return } event.preventDefault() event.stopPropagation() const point = overlayPoint(event) const region = newRegion(pendingDrawType, { x: point.x, y: point.y, w: 0.001, h: 0.001 }) profile.regions.push(region) selectedId = region.id dragState = { kind: 'draw', regionId: region.id, startX: point.x, startY: point.y, initial: { ...region.box }, } pendingDrawType = null overlay.classList.remove('drawing') byId('ocr-draw-mode').textContent = 'Drag to move · lower-right handle to resize' overlay.setPointerCapture(event.pointerId) renderRegions() renderRegionList() renderSelectedSettings() }, true) overlay.addEventListener('pointermove', (event) => { if (!dragState) return const region = profile.regions.find((candidate) => candidate.id === dragState?.regionId) if (!region) return const point = overlayPoint(event) const dx = point.x - dragState.startX const dy = point.y - dragState.startY if (dragState.kind === 'draw') { region.box.x = Math.min(dragState.startX, point.x) region.box.y = Math.min(dragState.startY, point.y) region.box.w = Math.max(0.005, Math.abs(point.x - dragState.startX)) region.box.h = Math.max(0.005, Math.abs(point.y - dragState.startY)) } else if (dragState.kind === 'move') { region.box.x = clamp(dragState.initial.x + dx, 0, 1 - region.box.w) region.box.y = clamp(dragState.initial.y + dy, 0, 1 - region.box.h) } else { region.box.w = clamp(dragState.initial.w + dx, 0.01, 1 - region.box.x) region.box.h = clamp(dragState.initial.h + dy, 0.01, 1 - region.box.y) } renderRegions() renderSelectedSettings() }) overlay.addEventListener('pointerup', () => { dragState = null updateCropPreview() }) overlay.addEventListener('pointercancel', () => { dragState = null }) const controlIds = [ 'ocr-region-name', 'ocr-region-type', 'ocr-region-target', 'ocr-psm', 'ocr-grayscale', 'ocr-contrast', 'ocr-threshold', 'ocr-whitelist', 'ocr-barcode-formats', 'ocr-first-match', ] controlIds.forEach((id) => { byId(id).addEventListener('input', updateRegionFromControls) byId(id).addEventListener('change', updateRegionFromControls) }) byId('ocr-delete-region').addEventListener('click', () => { if (!selectedId) return profile.regions = profile.regions.filter((region) => region.id !== selectedId) selectedId = null renderRegions() renderRegionList() renderSelectedSettings() }) byId('ocr-copy-region').addEventListener('click', () => { const region = selectedRegion() if (!region) return const copy: RefinementRegion = { ...region, id: crypto.randomUUID(), name: region.name + ' copy', box: { x: clamp(region.box.x + 0.02, 0, 1 - region.box.w), y: clamp(region.box.y + 0.02, 0, 1 - region.box.h), w: region.box.w, h: region.box.h, }, settings: { ...region.settings, barcodeFormats: [...region.settings.barcodeFormats] }, lastText: undefined, confidence: undefined, } profile.regions.push(copy) selectedId = copy.id renderRegions() renderRegionList() renderSelectedSettings() }) byId('ocr-run-region').addEventListener('click', () => { if (selectedId) void runRegion(selectedId) }) byId('ocr-run-all').addEventListener('click', async () => { const runnable = profile.regions.filter((region) => region.enabled && region.type === 'ocr') if (!source) { setStatus('Load a document before running OCR.') return } if (!runnable.length) { setStatus('Draw at least one OCR region first.') return } setStatus('Running ' + runnable.length + ' OCR region' + (runnable.length === 1 ? '' : 's') + '…') for (const region of runnable) await runRegion(region.id) setStatus('OCR pass complete. Review each region result, then save the profile.') }) byId('ocr-save-profile').addEventListener('click', async () => { profile.name = profileNameInput.value.trim() || 'OCR profile' const next = { ...profile, revision: profile.revision + 1, savedAt: new Date().toISOString(), } setStatus('Saving OCR profile…') try { const saved = await invoke('save_ocr_profile', { profile: next }) profile = normalizeProfile(saved) updateProfileHeader() setStatus('Profile saved. Region coordinates and OCR settings are persisted locally.') } catch (error) { setStatus('Profile save failed: ' + String(error)) } }) void loadSavedProfile() }